View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver.wal;
21  
22  import java.io.IOException;
23  import java.util.NavigableSet;
24  import java.util.TreeSet;
25  import java.util.concurrent.atomic.AtomicLong;
26  import java.util.regex.Matcher;
27  import java.util.regex.Pattern;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.classification.InterfaceAudience;
32  import org.apache.hadoop.conf.Configuration;
33  import org.apache.hadoop.fs.FileStatus;
34  import org.apache.hadoop.fs.FileSystem;
35  import org.apache.hadoop.fs.Path;
36  import org.apache.hadoop.fs.PathFilter;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.HRegionInfo;
39  import org.apache.hadoop.hbase.HTableDescriptor;
40  import org.apache.hadoop.hbase.ServerName;
41  import org.apache.hadoop.hbase.TableName;
42  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
43  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
44  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
45  import org.apache.hadoop.hbase.util.FSUtils;
46  
47  import com.google.protobuf.TextFormat;
48  
49  @InterfaceAudience.Private
50  public class HLogUtil {
51    static final Log LOG = LogFactory.getLog(HLogUtil.class);
52  
53    /**
54     * Pattern used to validate a HLog file name
55     */
56    private static final Pattern pattern =
57        Pattern.compile(".*\\.\\d*("+HLog.META_HLOG_FILE_EXTN+")*");
58  
59    /**
60     * @param filename
61     *          name of the file to validate
62     * @return <tt>true</tt> if the filename matches an HLog, <tt>false</tt>
63     *         otherwise
64     */
65    public static boolean validateHLogFilename(String filename) {
66      return pattern.matcher(filename).matches();
67    }
68  
69    /**
70     * Construct the HLog directory name
71     *
72     * @param serverName
73     *          Server name formatted as described in {@link ServerName}
74     * @return the relative HLog directory name, e.g.
75     *         <code>.logs/1.example.org,60030,12345</code> if
76     *         <code>serverName</code> passed is
77     *         <code>1.example.org,60030,12345</code>
78     */
79    public static String getHLogDirectoryName(final String serverName) {
80      StringBuilder dirName = new StringBuilder(HConstants.HREGION_LOGDIR_NAME);
81      dirName.append("/");
82      dirName.append(serverName);
83      return dirName.toString();
84    }
85  
86    /**
87     * @param regiondir
88     *          This regions directory in the filesystem.
89     * @return The directory that holds recovered edits files for the region
90     *         <code>regiondir</code>
91     */
92    public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {
93      return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);
94    }
95  
96    /**
97     * Move aside a bad edits file.
98     *
99     * @param fs
100    * @param edits
101    *          Edits file to move aside.
102    * @return The name of the moved aside file.
103    * @throws IOException
104    */
105   public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)
106       throws IOException {
107     Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."
108         + System.currentTimeMillis());
109     if (!fs.rename(edits, moveAsideName)) {
110       LOG.warn("Rename failed from " + edits + " to " + moveAsideName);
111     }
112     return moveAsideName;
113   }
114 
115   /**
116    * @param path
117    *          - the path to analyze. Expected format, if it's in hlog directory:
118    *          / [base directory for hbase] / hbase / .logs / ServerName /
119    *          logfile
120    * @return null if it's not a log file. Returns the ServerName of the region
121    *         server that created this log file otherwise.
122    */
123   public static ServerName getServerNameFromHLogDirectoryName(
124       Configuration conf, String path) throws IOException {
125     if (path == null
126         || path.length() <= HConstants.HREGION_LOGDIR_NAME.length()) {
127       return null;
128     }
129 
130     if (conf == null) {
131       throw new IllegalArgumentException("parameter conf must be set");
132     }
133 
134     final String rootDir = conf.get(HConstants.HBASE_DIR);
135     if (rootDir == null || rootDir.isEmpty()) {
136       throw new IllegalArgumentException(HConstants.HBASE_DIR
137           + " key not found in conf.");
138     }
139 
140     final StringBuilder startPathSB = new StringBuilder(rootDir);
141     if (!rootDir.endsWith("/"))
142       startPathSB.append('/');
143     startPathSB.append(HConstants.HREGION_LOGDIR_NAME);
144     if (!HConstants.HREGION_LOGDIR_NAME.endsWith("/"))
145       startPathSB.append('/');
146     final String startPath = startPathSB.toString();
147 
148     String fullPath;
149     try {
150       fullPath = FileSystem.get(conf).makeQualified(new Path(path)).toString();
151     } catch (IllegalArgumentException e) {
152       LOG.info("Call to makeQualified failed on " + path + " " + e.getMessage());
153       return null;
154     }
155 
156     if (!fullPath.startsWith(startPath)) {
157       return null;
158     }
159 
160     final String serverNameAndFile = fullPath.substring(startPath.length());
161 
162     if (serverNameAndFile.indexOf('/') < "a,0,0".length()) {
163       // Either it's a file (not a directory) or it's not a ServerName format
164       return null;
165     }
166 
167     Path p = new Path(path);
168     return getServerNameFromHLogDirectoryName(p);
169   }
170 
171   /**
172    * This function returns region server name from a log file name which is in either format:
173    * hdfs://<name node>/hbase/.logs/<server name>-splitting/... or hdfs://<name
174    * node>/hbase/.logs/<server name>/...
175    * @param logFile
176    * @return null if the passed in logFile isn't a valid HLog file path
177    */
178   public static ServerName getServerNameFromHLogDirectoryName(Path logFile) {
179     Path logDir = logFile.getParent();
180     String logDirName = logDir.getName();
181     if (logDirName.equals(HConstants.HREGION_LOGDIR_NAME)) {
182       logDir = logFile;
183       logDirName = logDir.getName();
184     }
185     ServerName serverName = null;
186     if (logDirName.endsWith(HLog.SPLITTING_EXT)) {
187       logDirName = logDirName.substring(0, logDirName.length() - HLog.SPLITTING_EXT.length());
188     }
189     try {
190       serverName = ServerName.parseServerName(logDirName);
191     } catch (IllegalArgumentException ex) {
192       serverName = null;
193       LOG.warn("Cannot parse a server name from path=" + logFile + "; " + ex.getMessage());
194     }
195     if (serverName != null && serverName.getStartcode() < 0) {
196       LOG.warn("Invalid log file path=" + logFile);
197       return null;
198     }
199     return serverName;
200   }
201 
202   /**
203    * Returns sorted set of edit files made by wal-log splitter, excluding files
204    * with '.temp' suffix.
205    *
206    * @param fs
207    * @param regiondir
208    * @return Files in passed <code>regiondir</code> as a sorted set.
209    * @throws IOException
210    */
211   public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,
212       final Path regiondir) throws IOException {
213     NavigableSet<Path> filesSorted = new TreeSet<Path>();
214     Path editsdir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
215     if (!fs.exists(editsdir))
216       return filesSorted;
217     FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {
218       @Override
219       public boolean accept(Path p) {
220         boolean result = false;
221         try {
222           // Return files and only files that match the editfile names pattern.
223           // There can be other files in this directory other than edit files.
224           // In particular, on error, we'll move aside the bad edit file giving
225           // it a timestamp suffix. See moveAsideBadEditsFile.
226           Matcher m = HLog.EDITFILES_NAME_PATTERN.matcher(p.getName());
227           result = fs.isFile(p) && m.matches();
228           // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,
229           // because it means splithlog thread is writting this file.
230           if (p.getName().endsWith(HLog.RECOVERED_LOG_TMPFILE_SUFFIX)) {
231             result = false;
232           }
233         } catch (IOException e) {
234           LOG.warn("Failed isFile check on " + p);
235         }
236         return result;
237       }
238     });
239     if (files == null)
240       return filesSorted;
241     for (FileStatus status : files) {
242       filesSorted.add(status.getPath());
243     }
244     return filesSorted;
245   }
246 
247   public static boolean isMetaFile(Path p) {
248     return isMetaFile(p.getName());
249   }
250 
251   public static boolean isMetaFile(String p) {
252     if (p != null && p.endsWith(HLog.META_HLOG_FILE_EXTN)) {
253       return true;
254     }
255     return false;
256   }
257 
258   /**
259    * Write the marker that a compaction has succeeded and is about to be committed.
260    * This provides info to the HMaster to allow it to recover the compaction if
261    * this regionserver dies in the middle (This part is not yet implemented). It also prevents
262    * the compaction from finishing if this regionserver has already lost its lease on the log.
263    * @param sequenceId Used by HLog to get sequence Id for the waledit.
264    */
265   public static void writeCompactionMarker(HLog log, HTableDescriptor htd, HRegionInfo info,
266       final CompactionDescriptor c, AtomicLong sequenceId) throws IOException {
267     TableName tn = TableName.valueOf(c.getTableName().toByteArray());
268     HLogKey key = new HLogKey(info.getEncodedNameAsBytes(), tn);
269     log.appendNoSync(htd, info, key, WALEdit.createCompaction(info, c), sequenceId, false, null);
270     log.sync();
271     if (LOG.isTraceEnabled()) {
272       LOG.trace("Appended compaction marker " + TextFormat.shortDebugString(c));
273     }
274   }
275 
276   /**
277    * Write a flush marker indicating a start / abort or a complete of a region flush
278    */
279   public static long writeFlushMarker(HLog log, HTableDescriptor htd, HRegionInfo info,
280       final FlushDescriptor f, AtomicLong sequenceId, boolean sync) throws IOException {
281     TableName tn = TableName.valueOf(f.getTableName().toByteArray());
282     HLogKey key = new HLogKey(info.getEncodedNameAsBytes(), tn);
283     long trx = log.appendNoSync(htd, info, key, WALEdit.createFlushWALEdit(info, f), sequenceId, false, null);
284     if (sync) log.sync(trx);
285     if (LOG.isTraceEnabled()) {
286       LOG.trace("Appended flush marker " + TextFormat.shortDebugString(f));
287     }
288     return trx;
289   }
290 
291   /**
292    * Write a region open marker indicating that the region is opened
293    */
294   public static long writeRegionEventMarker(HLog log, HTableDescriptor htd, HRegionInfo info,
295       final RegionEventDescriptor r, AtomicLong sequenceId) throws IOException {
296     TableName tn = TableName.valueOf(r.getTableName().toByteArray());
297     HLogKey key = new HLogKey(info.getEncodedNameAsBytes(), tn);
298     long trx = log.appendNoSync(htd, info, key, WALEdit.createRegionEventWALEdit(info, r),
299       sequenceId, false, null);
300     log.sync(trx);
301     if (LOG.isTraceEnabled()) {
302       LOG.trace("Appended region event marker " + TextFormat.shortDebugString(r));
303     }
304     return trx;
305   }
306 }