View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver.wal;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.List;
25  import java.util.NavigableSet;
26  import java.util.TreeSet;
27  import java.util.concurrent.atomic.AtomicLong;
28  import java.util.regex.Matcher;
29  import java.util.regex.Pattern;
30  
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  import org.apache.hadoop.hbase.classification.InterfaceAudience;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.fs.FileStatus;
36  import org.apache.hadoop.fs.FileSystem;
37  import org.apache.hadoop.fs.Path;
38  import org.apache.hadoop.fs.PathFilter;
39  import org.apache.hadoop.hbase.HConstants;
40  import org.apache.hadoop.hbase.HRegionInfo;
41  import org.apache.hadoop.hbase.HTableDescriptor;
42  import org.apache.hadoop.hbase.ServerName;
43  import org.apache.hadoop.hbase.TableName;
44  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
45  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
46  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
47  import org.apache.hadoop.hbase.util.FSUtils;
48  
49  import com.google.protobuf.TextFormat;
50  
51  @InterfaceAudience.Private
52  public class HLogUtil {
53    static final Log LOG = LogFactory.getLog(HLogUtil.class);
54  
55    /**
56     * Pattern used to validate a HLog file name
57     */
58    private static final Pattern pattern =
59        Pattern.compile(".*\\.\\d*("+HLog.META_HLOG_FILE_EXTN+")*");
60  
61    /**
62     * @param filename
63     *          name of the file to validate
64     * @return <tt>true</tt> if the filename matches an HLog, <tt>false</tt>
65     *         otherwise
66     */
67    public static boolean validateHLogFilename(String filename) {
68      return pattern.matcher(filename).matches();
69    }
70  
71    /**
72     * Construct the HLog directory name
73     *
74     * @param serverName
75     *          Server name formatted as described in {@link ServerName}
76     * @return the relative HLog directory name, e.g.
77     *         <code>.logs/1.example.org,60030,12345</code> if
78     *         <code>serverName</code> passed is
79     *         <code>1.example.org,60030,12345</code>
80     */
81    public static String getHLogDirectoryName(final String serverName) {
82      StringBuilder dirName = new StringBuilder(HConstants.HREGION_LOGDIR_NAME);
83      dirName.append("/");
84      dirName.append(serverName);
85      return dirName.toString();
86    }
87  
88    /**
89     * @param regiondir
90     *          This regions directory in the filesystem.
91     * @return The directory that holds recovered edits files for the region
92     *         <code>regiondir</code>
93     */
94    public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {
95      return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);
96    }
97    
98    /**
99     * Move aside a bad edits file.
100    *
101    * @param fs
102    * @param edits
103    *          Edits file to move aside.
104    * @return The name of the moved aside file.
105    * @throws IOException
106    */
107   public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)
108       throws IOException {
109     Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."
110         + System.currentTimeMillis());
111     if (!fs.rename(edits, moveAsideName)) {
112       LOG.warn("Rename failed from " + edits + " to " + moveAsideName);
113     }
114     return moveAsideName;
115   }
116 
117   /**
118    * @param path
119    *          - the path to analyze. Expected format, if it's in hlog directory:
120    *          / [base directory for hbase] / hbase / .logs / ServerName /
121    *          logfile
122    * @return null if it's not a log file. Returns the ServerName of the region
123    *         server that created this log file otherwise.
124    */
125   public static ServerName getServerNameFromHLogDirectoryName(
126       Configuration conf, String path) throws IOException {
127     if (path == null
128         || path.length() <= HConstants.HREGION_LOGDIR_NAME.length()) {
129       return null;
130     }
131 
132     if (conf == null) {
133       throw new IllegalArgumentException("parameter conf must be set");
134     }
135 
136     final String rootDir = conf.get(HConstants.HBASE_DIR);
137     if (rootDir == null || rootDir.isEmpty()) {
138       throw new IllegalArgumentException(HConstants.HBASE_DIR
139           + " key not found in conf.");
140     }
141 
142     final StringBuilder startPathSB = new StringBuilder(rootDir);
143     if (!rootDir.endsWith("/"))
144       startPathSB.append('/');
145     startPathSB.append(HConstants.HREGION_LOGDIR_NAME);
146     if (!HConstants.HREGION_LOGDIR_NAME.endsWith("/"))
147       startPathSB.append('/');
148     final String startPath = startPathSB.toString();
149 
150     String fullPath;
151     try {
152       fullPath = FileSystem.get(conf).makeQualified(new Path(path)).toString();
153     } catch (IllegalArgumentException e) {
154       LOG.info("Call to makeQualified failed on " + path + " " + e.getMessage());
155       return null;
156     }
157 
158     if (!fullPath.startsWith(startPath)) {
159       return null;
160     }
161 
162     final String serverNameAndFile = fullPath.substring(startPath.length());
163 
164     if (serverNameAndFile.indexOf('/') < "a,0,0".length()) {
165       // Either it's a file (not a directory) or it's not a ServerName format
166       return null;
167     }
168 
169     Path p = new Path(path);
170     return getServerNameFromHLogDirectoryName(p);
171   }
172 
173   /**
174    * This function returns region server name from a log file name which is in either format:
175    * hdfs://<name node>/hbase/.logs/<server name>-splitting/... or hdfs://<name
176    * node>/hbase/.logs/<server name>/...
177    * @param logFile
178    * @return null if the passed in logFile isn't a valid HLog file path
179    */
180   public static ServerName getServerNameFromHLogDirectoryName(Path logFile) {
181     Path logDir = logFile.getParent();
182     String logDirName = logDir.getName();
183     if (logDirName.equals(HConstants.HREGION_LOGDIR_NAME)) {
184       logDir = logFile;
185       logDirName = logDir.getName();
186     }
187     ServerName serverName = null;
188     if (logDirName.endsWith(HLog.SPLITTING_EXT)) {
189       logDirName = logDirName.substring(0, logDirName.length() - HLog.SPLITTING_EXT.length());
190     }
191     try {
192       serverName = ServerName.parseServerName(logDirName);
193     } catch (IllegalArgumentException ex) {
194       serverName = null;
195       LOG.warn("Cannot parse a server name from path=" + logFile + "; " + ex.getMessage());
196     }
197     if (serverName != null && serverName.getStartcode() < 0) {
198       LOG.warn("Invalid log file path=" + logFile);
199       return null;
200     }
201     return serverName;
202   }
203 
204   /**
205    * Returns sorted set of edit files made by wal-log splitter, excluding files
206    * with '.temp' suffix.
207    *
208    * @param fs
209    * @param regiondir
210    * @return Files in passed <code>regiondir</code> as a sorted set.
211    * @throws IOException
212    */
213   public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,
214       final Path regiondir) throws IOException {
215     NavigableSet<Path> filesSorted = new TreeSet<Path>();
216     Path editsdir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
217     if (!fs.exists(editsdir))
218       return filesSorted;
219     FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {
220       @Override
221       public boolean accept(Path p) {
222         boolean result = false;
223         try {
224           // Return files and only files that match the editfile names pattern.
225           // There can be other files in this directory other than edit files.
226           // In particular, on error, we'll move aside the bad edit file giving
227           // it a timestamp suffix. See moveAsideBadEditsFile.
228           Matcher m = HLog.EDITFILES_NAME_PATTERN.matcher(p.getName());
229           result = fs.isFile(p) && m.matches();
230           // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,
231           // because it means splithlog thread is writting this file.
232           if (p.getName().endsWith(HLog.RECOVERED_LOG_TMPFILE_SUFFIX)) {
233             result = false;
234           }
235         } catch (IOException e) {
236           LOG.warn("Failed isFile check on " + p);
237         }
238         return result;
239       }
240     });
241     if (files == null)
242       return filesSorted;
243     for (FileStatus status : files) {
244       filesSorted.add(status.getPath());
245     }
246     return filesSorted;
247   }
248 
249   public static boolean isMetaFile(Path p) {
250     return isMetaFile(p.getName());
251   }
252 
253   public static boolean isMetaFile(String p) {
254     if (p != null && p.endsWith(HLog.META_HLOG_FILE_EXTN)) {
255       return true;
256     }
257     return false;
258   }
259 
260   /**
261    * Write the marker that a compaction has succeeded and is about to be committed.
262    * This provides info to the HMaster to allow it to recover the compaction if
263    * this regionserver dies in the middle (This part is not yet implemented). It also prevents
264    * the compaction from finishing if this regionserver has already lost its lease on the log.
265    * @param sequenceId Used by HLog to get sequence Id for the waledit.
266    */
267   public static void writeCompactionMarker(HLog log, HTableDescriptor htd, HRegionInfo info,
268       final CompactionDescriptor c, AtomicLong sequenceId) throws IOException {
269     TableName tn = TableName.valueOf(c.getTableName().toByteArray());
270     HLogKey key = new HLogKey(info.getEncodedNameAsBytes(), tn);
271     log.appendNoSync(htd, info, key, WALEdit.createCompaction(info, c), sequenceId, false, null);
272     log.sync();
273     if (LOG.isTraceEnabled()) {
274       LOG.trace("Appended compaction marker " + TextFormat.shortDebugString(c));
275     }
276   }
277 
278   /**
279    * Write a flush marker indicating a start / abort or a complete of a region flush
280    */
281   public static long writeFlushMarker(HLog log, HTableDescriptor htd, HRegionInfo info,
282       final FlushDescriptor f, AtomicLong sequenceId, boolean sync) throws IOException {
283     TableName tn = TableName.valueOf(f.getTableName().toByteArray());
284     HLogKey key = new HLogKey(info.getEncodedNameAsBytes(), tn);
285     long trx = log.appendNoSync(htd, info, key, WALEdit.createFlushWALEdit(info, f), sequenceId, false, null);
286     if (sync) log.sync(trx);
287     if (LOG.isTraceEnabled()) {
288       LOG.trace("Appended flush marker " + TextFormat.shortDebugString(f));
289     }
290     return trx;
291   }
292 
293   /**
294    * Write a region open marker indicating that the region is opened
295    */
296   public static long writeRegionEventMarker(HLog log, HTableDescriptor htd, HRegionInfo info,
297       final RegionEventDescriptor r, AtomicLong sequenceId) throws IOException {
298     TableName tn = TableName.valueOf(r.getTableName().toByteArray());
299     HLogKey key = new HLogKey(info.getEncodedNameAsBytes(), tn);
300     long trx = log.appendNoSync(htd, info, key, WALEdit.createRegionEventWALEdit(info, r),
301       sequenceId, false, null);
302     log.sync(trx);
303     if (LOG.isTraceEnabled()) {
304       LOG.trace("Appended region event marker " + TextFormat.shortDebugString(r));
305     }
306     return trx;
307   }
308   
309   /**
310    * Create a file with name as region open sequence id
311    * 
312    * @param fs
313    * @param regiondir
314    * @param newSeqId
315    * @param saftyBumper
316    * @return long new sequence Id value
317    * @throws IOException
318    */
319   public static long writeRegionOpenSequenceIdFile(final FileSystem fs, final Path regiondir,
320       long newSeqId, long saftyBumper) throws IOException {
321 
322     Path editsdir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
323     long maxSeqId = 0;
324     FileStatus[] files = null;
325     if (fs.exists(editsdir)) {
326       files = FSUtils.listStatus(fs, editsdir, new PathFilter() {
327         @Override
328         public boolean accept(Path p) {
329           if (p.getName().endsWith(HLog.SEQUENCE_ID_FILE_SUFFIX)) {
330             return true;
331           }
332           return false;
333         }
334       });
335       if (files != null) {
336         for (FileStatus status : files) {
337           String fileName = status.getPath().getName();
338           try {
339             Long tmpSeqId = Long.parseLong(fileName.substring(0, fileName.length()
340                     - HLog.SEQUENCE_ID_FILE_SUFFIX.length()));
341             maxSeqId = Math.max(tmpSeqId, maxSeqId);
342           } catch (NumberFormatException ex) {
343             LOG.warn("Invalid SeqId File Name=" + fileName);
344           }
345         }
346       }
347     }
348     if (maxSeqId > newSeqId) {
349       newSeqId = maxSeqId;
350     }
351     newSeqId += saftyBumper; // bump up SeqId
352     
353     // write a new seqId file
354     Path newSeqIdFile = new Path(editsdir, newSeqId + HLog.SEQUENCE_ID_FILE_SUFFIX);
355     if (!fs.createNewFile(newSeqIdFile)) {
356       throw new IOException("Failed to create SeqId file:" + newSeqIdFile);
357     }
358     // remove old ones
359     if(files != null) {
360       for (FileStatus status : files) {
361         fs.delete(status.getPath(), false);
362       }
363     }
364     return newSeqId;
365   }
366   
367 }