View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver.wal;
21  
22  import java.io.IOException;
23  import java.util.NavigableSet;
24  import java.util.TreeSet;
25  import java.util.concurrent.atomic.AtomicLong;
26  import java.util.regex.Matcher;
27  import java.util.regex.Pattern;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FileStatus;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.fs.PathFilter;
36  import org.apache.hadoop.hbase.HConstants;
37  import org.apache.hadoop.hbase.HRegionInfo;
38  import org.apache.hadoop.hbase.HTableDescriptor;
39  import org.apache.hadoop.hbase.ServerName;
40  import org.apache.hadoop.hbase.TableName;
41  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
42  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
43  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
44  import org.apache.hadoop.hbase.util.FSUtils;
45  
46  import com.google.protobuf.TextFormat;
47  
48  public class HLogUtil {
49    static final Log LOG = LogFactory.getLog(HLogUtil.class);
50  
51    /**
52     * Pattern used to validate a HLog file name
53     */
54    private static final Pattern pattern =
55        Pattern.compile(".*\\.\\d*("+HLog.META_HLOG_FILE_EXTN+")*");
56  
57    /**
58     * @param filename
59     *          name of the file to validate
60     * @return <tt>true</tt> if the filename matches an HLog, <tt>false</tt>
61     *         otherwise
62     */
63    public static boolean validateHLogFilename(String filename) {
64      return pattern.matcher(filename).matches();
65    }
66  
67    /**
68     * Construct the HLog directory name
69     *
70     * @param serverName
71     *          Server name formatted as described in {@link ServerName}
72     * @return the relative HLog directory name, e.g.
73     *         <code>.logs/1.example.org,60030,12345</code> if
74     *         <code>serverName</code> passed is
75     *         <code>1.example.org,60030,12345</code>
76     */
77    public static String getHLogDirectoryName(final String serverName) {
78      StringBuilder dirName = new StringBuilder(HConstants.HREGION_LOGDIR_NAME);
79      dirName.append("/");
80      dirName.append(serverName);
81      return dirName.toString();
82    }
83  
84    /**
85     * @param regiondir
86     *          This regions directory in the filesystem.
87     * @return The directory that holds recovered edits files for the region
88     *         <code>regiondir</code>
89     */
90    public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {
91      return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);
92    }
93  
94    /**
95     * Move aside a bad edits file.
96     *
97     * @param fs
98     * @param edits
99     *          Edits file to move aside.
100    * @return The name of the moved aside file.
101    * @throws IOException
102    */
103   public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)
104       throws IOException {
105     Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."
106         + System.currentTimeMillis());
107     if (!fs.rename(edits, moveAsideName)) {
108       LOG.warn("Rename failed from " + edits + " to " + moveAsideName);
109     }
110     return moveAsideName;
111   }
112 
113   /**
114    * @param path
115    *          - the path to analyze. Expected format, if it's in hlog directory:
116    *          / [base directory for hbase] / hbase / .logs / ServerName /
117    *          logfile
118    * @return null if it's not a log file. Returns the ServerName of the region
119    *         server that created this log file otherwise.
120    */
121   public static ServerName getServerNameFromHLogDirectoryName(
122       Configuration conf, String path) throws IOException {
123     if (path == null
124         || path.length() <= HConstants.HREGION_LOGDIR_NAME.length()) {
125       return null;
126     }
127 
128     if (conf == null) {
129       throw new IllegalArgumentException("parameter conf must be set");
130     }
131 
132     final String rootDir = conf.get(HConstants.HBASE_DIR);
133     if (rootDir == null || rootDir.isEmpty()) {
134       throw new IllegalArgumentException(HConstants.HBASE_DIR
135           + " key not found in conf.");
136     }
137 
138     final StringBuilder startPathSB = new StringBuilder(rootDir);
139     if (!rootDir.endsWith("/"))
140       startPathSB.append('/');
141     startPathSB.append(HConstants.HREGION_LOGDIR_NAME);
142     if (!HConstants.HREGION_LOGDIR_NAME.endsWith("/"))
143       startPathSB.append('/');
144     final String startPath = startPathSB.toString();
145 
146     String fullPath;
147     try {
148       fullPath = FileSystem.get(conf).makeQualified(new Path(path)).toString();
149     } catch (IllegalArgumentException e) {
150       LOG.info("Call to makeQualified failed on " + path + " " + e.getMessage());
151       return null;
152     }
153 
154     if (!fullPath.startsWith(startPath)) {
155       return null;
156     }
157 
158     final String serverNameAndFile = fullPath.substring(startPath.length());
159 
160     if (serverNameAndFile.indexOf('/') < "a,0,0".length()) {
161       // Either it's a file (not a directory) or it's not a ServerName format
162       return null;
163     }
164 
165     Path p = new Path(path);
166     return getServerNameFromHLogDirectoryName(p);
167   }
168 
169   /**
170    * This function returns region server name from a log file name which is in either format:
171    * hdfs://<name node>/hbase/.logs/<server name>-splitting/... or hdfs://<name
172    * node>/hbase/.logs/<server name>/...
173    * @param logFile
174    * @return null if the passed in logFile isn't a valid HLog file path
175    */
176   public static ServerName getServerNameFromHLogDirectoryName(Path logFile) {
177     Path logDir = logFile.getParent();
178     String logDirName = logDir.getName();
179     if (logDirName.equals(HConstants.HREGION_LOGDIR_NAME)) {
180       logDir = logFile;
181       logDirName = logDir.getName();
182     }
183     ServerName serverName = null;
184     if (logDirName.endsWith(HLog.SPLITTING_EXT)) {
185       logDirName = logDirName.substring(0, logDirName.length() - HLog.SPLITTING_EXT.length());
186     }
187     try {
188       serverName = ServerName.parseServerName(logDirName);
189     } catch (IllegalArgumentException ex) {
190       serverName = null;
191       LOG.warn("Cannot parse a server name from path=" + logFile + "; " + ex.getMessage());
192     }
193     if (serverName != null && serverName.getStartcode() < 0) {
194       LOG.warn("Invalid log file path=" + logFile);
195       return null;
196     }
197     return serverName;
198   }
199 
200   /**
201    * Returns sorted set of edit files made by wal-log splitter, excluding files
202    * with '.temp' suffix.
203    *
204    * @param fs
205    * @param regiondir
206    * @return Files in passed <code>regiondir</code> as a sorted set.
207    * @throws IOException
208    */
209   public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,
210       final Path regiondir) throws IOException {
211     NavigableSet<Path> filesSorted = new TreeSet<Path>();
212     Path editsdir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
213     if (!fs.exists(editsdir))
214       return filesSorted;
215     FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {
216       @Override
217       public boolean accept(Path p) {
218         boolean result = false;
219         try {
220           // Return files and only files that match the editfile names pattern.
221           // There can be other files in this directory other than edit files.
222           // In particular, on error, we'll move aside the bad edit file giving
223           // it a timestamp suffix. See moveAsideBadEditsFile.
224           Matcher m = HLog.EDITFILES_NAME_PATTERN.matcher(p.getName());
225           result = fs.isFile(p) && m.matches();
226           // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,
227           // because it means splithlog thread is writting this file.
228           if (p.getName().endsWith(HLog.RECOVERED_LOG_TMPFILE_SUFFIX)) {
229             result = false;
230           }
231         } catch (IOException e) {
232           LOG.warn("Failed isFile check on " + p);
233         }
234         return result;
235       }
236     });
237     if (files == null)
238       return filesSorted;
239     for (FileStatus status : files) {
240       filesSorted.add(status.getPath());
241     }
242     return filesSorted;
243   }
244 
245   public static boolean isMetaFile(Path p) {
246     return isMetaFile(p.getName());
247   }
248 
249   public static boolean isMetaFile(String p) {
250     if (p != null && p.endsWith(HLog.META_HLOG_FILE_EXTN)) {
251       return true;
252     }
253     return false;
254   }
255 
256   /**
257    * Write the marker that a compaction has succeeded and is about to be committed.
258    * This provides info to the HMaster to allow it to recover the compaction if
259    * this regionserver dies in the middle (This part is not yet implemented). It also prevents
260    * the compaction from finishing if this regionserver has already lost its lease on the log.
261    * @param sequenceId Used by HLog to get sequence Id for the waledit.
262    */
263   public static void writeCompactionMarker(HLog log, HTableDescriptor htd, HRegionInfo info,
264       final CompactionDescriptor c, AtomicLong sequenceId) throws IOException {
265     TableName tn = TableName.valueOf(c.getTableName().toByteArray());
266     HLogKey key = new HLogKey(info.getEncodedNameAsBytes(), tn);
267     log.appendNoSync(htd, info, key, WALEdit.createCompaction(info, c), sequenceId, false, null);
268     log.sync();
269     if (LOG.isTraceEnabled()) {
270       LOG.trace("Appended compaction marker " + TextFormat.shortDebugString(c));
271     }
272   }
273 
274   /**
275    * Write a flush marker indicating a start / abort or a complete of a region flush
276    */
277   public static long writeFlushMarker(HLog log, HTableDescriptor htd, HRegionInfo info,
278       final FlushDescriptor f, AtomicLong sequenceId, boolean sync) throws IOException {
279     TableName tn = TableName.valueOf(f.getTableName().toByteArray());
280     HLogKey key = new HLogKey(info.getEncodedNameAsBytes(), tn);
281     long trx = log.appendNoSync(htd, info, key, WALEdit.createFlushWALEdit(info, f), sequenceId, false, null);
282     if (sync) log.sync(trx);
283     if (LOG.isTraceEnabled()) {
284       LOG.trace("Appended flush marker " + TextFormat.shortDebugString(f));
285     }
286     return trx;
287   }
288 
289   /**
290    * Write a region open marker indicating that the region is opened
291    */
292   public static long writeRegionEventMarker(HLog log, HTableDescriptor htd, HRegionInfo info,
293       final RegionEventDescriptor r, AtomicLong sequenceId) throws IOException {
294     TableName tn = TableName.valueOf(r.getTableName().toByteArray());
295     HLogKey key = new HLogKey(info.getEncodedNameAsBytes(), tn);
296     long trx = log.appendNoSync(htd, info, key, WALEdit.createRegionEventWALEdit(info, r),
297       sequenceId, false, null);
298     log.sync(trx);
299     if (LOG.isTraceEnabled()) {
300       LOG.trace("Appended region event marker " + TextFormat.shortDebugString(r));
301     }
302     return trx;
303   }
304 }