View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver.wal;
21  
22  import java.io.IOException;
23  import java.util.NavigableSet;
24  import java.util.TreeSet;
25  import java.util.concurrent.atomic.AtomicLong;
26  import java.util.regex.Matcher;
27  import java.util.regex.Pattern;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FileStatus;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.fs.PathFilter;
36  import org.apache.hadoop.hbase.HConstants;
37  import org.apache.hadoop.hbase.HRegionInfo;
38  import org.apache.hadoop.hbase.HTableDescriptor;
39  import org.apache.hadoop.hbase.ServerName;
40  import org.apache.hadoop.hbase.TableName;
41  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
42  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
43  import org.apache.hadoop.hbase.util.FSUtils;
44  
45  import com.google.protobuf.TextFormat;
46  
47  public class HLogUtil {
48    static final Log LOG = LogFactory.getLog(HLogUtil.class);
49  
50    /**
51     * Pattern used to validate a HLog file name
52     */
53    private static final Pattern pattern =
54        Pattern.compile(".*\\.\\d*("+HLog.META_HLOG_FILE_EXTN+")*");
55  
56    /**
57     * @param filename
58     *          name of the file to validate
59     * @return <tt>true</tt> if the filename matches an HLog, <tt>false</tt>
60     *         otherwise
61     */
62    public static boolean validateHLogFilename(String filename) {
63      return pattern.matcher(filename).matches();
64    }
65  
66    /**
67     * Construct the HLog directory name
68     *
69     * @param serverName
70     *          Server name formatted as described in {@link ServerName}
71     * @return the relative HLog directory name, e.g.
72     *         <code>.logs/1.example.org,60030,12345</code> if
73     *         <code>serverName</code> passed is
74     *         <code>1.example.org,60030,12345</code>
75     */
76    public static String getHLogDirectoryName(final String serverName) {
77      StringBuilder dirName = new StringBuilder(HConstants.HREGION_LOGDIR_NAME);
78      dirName.append("/");
79      dirName.append(serverName);
80      return dirName.toString();
81    }
82  
83    /**
84     * @param regiondir
85     *          This regions directory in the filesystem.
86     * @return The directory that holds recovered edits files for the region
87     *         <code>regiondir</code>
88     */
89    public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {
90      return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);
91    }
92  
93    /**
94     * Move aside a bad edits file.
95     *
96     * @param fs
97     * @param edits
98     *          Edits file to move aside.
99     * @return The name of the moved aside file.
100    * @throws IOException
101    */
102   public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)
103       throws IOException {
104     Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."
105         + System.currentTimeMillis());
106     if (!fs.rename(edits, moveAsideName)) {
107       LOG.warn("Rename failed from " + edits + " to " + moveAsideName);
108     }
109     return moveAsideName;
110   }
111 
112   /**
113    * @param path
114    *          - the path to analyze. Expected format, if it's in hlog directory:
115    *          / [base directory for hbase] / hbase / .logs / ServerName /
116    *          logfile
117    * @return null if it's not a log file. Returns the ServerName of the region
118    *         server that created this log file otherwise.
119    */
120   public static ServerName getServerNameFromHLogDirectoryName(
121       Configuration conf, String path) throws IOException {
122     if (path == null
123         || path.length() <= HConstants.HREGION_LOGDIR_NAME.length()) {
124       return null;
125     }
126 
127     if (conf == null) {
128       throw new IllegalArgumentException("parameter conf must be set");
129     }
130 
131     final String rootDir = conf.get(HConstants.HBASE_DIR);
132     if (rootDir == null || rootDir.isEmpty()) {
133       throw new IllegalArgumentException(HConstants.HBASE_DIR
134           + " key not found in conf.");
135     }
136 
137     final StringBuilder startPathSB = new StringBuilder(rootDir);
138     if (!rootDir.endsWith("/"))
139       startPathSB.append('/');
140     startPathSB.append(HConstants.HREGION_LOGDIR_NAME);
141     if (!HConstants.HREGION_LOGDIR_NAME.endsWith("/"))
142       startPathSB.append('/');
143     final String startPath = startPathSB.toString();
144 
145     String fullPath;
146     try {
147       fullPath = FileSystem.get(conf).makeQualified(new Path(path)).toString();
148     } catch (IllegalArgumentException e) {
149       LOG.info("Call to makeQualified failed on " + path + " " + e.getMessage());
150       return null;
151     }
152 
153     if (!fullPath.startsWith(startPath)) {
154       return null;
155     }
156 
157     final String serverNameAndFile = fullPath.substring(startPath.length());
158 
159     if (serverNameAndFile.indexOf('/') < "a,0,0".length()) {
160       // Either it's a file (not a directory) or it's not a ServerName format
161       return null;
162     }
163 
164     Path p = new Path(path);
165     return getServerNameFromHLogDirectoryName(p);
166   }
167 
168   /**
169    * This function returns region server name from a log file name which is in either format:
170    * hdfs://<name node>/hbase/.logs/<server name>-splitting/... or hdfs://<name
171    * node>/hbase/.logs/<server name>/...
172    * @param logFile
173    * @return null if the passed in logFile isn't a valid HLog file path
174    */
175   public static ServerName getServerNameFromHLogDirectoryName(Path logFile) {
176     Path logDir = logFile.getParent();
177     String logDirName = logDir.getName();
178     if (logDirName.equals(HConstants.HREGION_LOGDIR_NAME)) {
179       logDir = logFile;
180       logDirName = logDir.getName();
181     }
182     ServerName serverName = null;
183     if (logDirName.endsWith(HLog.SPLITTING_EXT)) {
184       logDirName = logDirName.substring(0, logDirName.length() - HLog.SPLITTING_EXT.length());
185     }
186     try {
187       serverName = ServerName.parseServerName(logDirName);
188     } catch (IllegalArgumentException ex) {
189       serverName = null;
190       LOG.warn("Cannot parse a server name from path=" + logFile + "; " + ex.getMessage());
191     }
192     if (serverName != null && serverName.getStartcode() < 0) {
193       LOG.warn("Invalid log file path=" + logFile);
194       return null;
195     }
196     return serverName;
197   }
198 
199   /**
200    * Returns sorted set of edit files made by wal-log splitter, excluding files
201    * with '.temp' suffix.
202    *
203    * @param fs
204    * @param regiondir
205    * @return Files in passed <code>regiondir</code> as a sorted set.
206    * @throws IOException
207    */
208   public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,
209       final Path regiondir) throws IOException {
210     NavigableSet<Path> filesSorted = new TreeSet<Path>();
211     Path editsdir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
212     if (!fs.exists(editsdir))
213       return filesSorted;
214     FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {
215       @Override
216       public boolean accept(Path p) {
217         boolean result = false;
218         try {
219           // Return files and only files that match the editfile names pattern.
220           // There can be other files in this directory other than edit files.
221           // In particular, on error, we'll move aside the bad edit file giving
222           // it a timestamp suffix. See moveAsideBadEditsFile.
223           Matcher m = HLog.EDITFILES_NAME_PATTERN.matcher(p.getName());
224           result = fs.isFile(p) && m.matches();
225           // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,
226           // because it means splithlog thread is writting this file.
227           if (p.getName().endsWith(HLog.RECOVERED_LOG_TMPFILE_SUFFIX)) {
228             result = false;
229           }
230         } catch (IOException e) {
231           LOG.warn("Failed isFile check on " + p);
232         }
233         return result;
234       }
235     });
236     if (files == null)
237       return filesSorted;
238     for (FileStatus status : files) {
239       filesSorted.add(status.getPath());
240     }
241     return filesSorted;
242   }
243 
244   public static boolean isMetaFile(Path p) {
245     return isMetaFile(p.getName());
246   }
247 
248   public static boolean isMetaFile(String p) {
249     if (p != null && p.endsWith(HLog.META_HLOG_FILE_EXTN)) {
250       return true;
251     }
252     return false;
253   }
254 
255   /**
256    * Write the marker that a compaction has succeeded and is about to be committed.
257    * This provides info to the HMaster to allow it to recover the compaction if
258    * this regionserver dies in the middle (This part is not yet implemented). It also prevents
259    * the compaction from finishing if this regionserver has already lost its lease on the log.
260    * @param sequenceId Used by HLog to get sequence Id for the waledit.
261    */
262   public static void writeCompactionMarker(HLog log, HTableDescriptor htd, HRegionInfo info,
263       final CompactionDescriptor c, AtomicLong sequenceId) throws IOException {
264     TableName tn = TableName.valueOf(c.getTableName().toByteArray());
265     HLogKey key = new HLogKey(info.getEncodedNameAsBytes(), tn);
266     log.appendNoSync(htd, info, key, WALEdit.createCompaction(info, c), sequenceId, false, null);
267     log.sync();
268     if (LOG.isTraceEnabled()) {
269       LOG.trace("Appended compaction marker " + TextFormat.shortDebugString(c));
270     }
271   }
272 
273   /**
274    * Write a flush marker indicating a start / abort or a complete of a region flush
275    */
276   public static long writeFlushMarker(HLog log, HTableDescriptor htd, HRegionInfo info,
277       final FlushDescriptor f, AtomicLong sequenceId, boolean sync) throws IOException {
278     TableName tn = TableName.valueOf(f.getTableName().toByteArray());
279     HLogKey key = new HLogKey(info.getEncodedNameAsBytes(), tn);
280     long trx = log.appendNoSync(htd, info, key, WALEdit.createFlushWALEdit(info, f), sequenceId, false, null);
281     if (sync) log.sync(trx);
282     if (LOG.isTraceEnabled()) {
283       LOG.trace("Appended flush marker " + TextFormat.shortDebugString(f));
284     }
285     return trx;
286   }
287 }