View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver.wal;
21  
22  import java.io.IOException;
23  import java.util.NavigableSet;
24  import java.util.TreeSet;
25  import java.util.concurrent.atomic.AtomicLong;
26  import java.util.regex.Matcher;
27  import java.util.regex.Pattern;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FileStatus;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.fs.PathFilter;
36  import org.apache.hadoop.hbase.HConstants;
37  import org.apache.hadoop.hbase.HRegionInfo;
38  import org.apache.hadoop.hbase.HTableDescriptor;
39  import org.apache.hadoop.hbase.ServerName;
40  import org.apache.hadoop.hbase.TableName;
41  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
42  import org.apache.hadoop.hbase.util.FSUtils;
43  
44  import com.google.protobuf.TextFormat;
45  
46  public class HLogUtil {
47    static final Log LOG = LogFactory.getLog(HLogUtil.class);
48  
49    /**
50     * Pattern used to validate a HLog file name
51     */
52    private static final Pattern pattern =
53        Pattern.compile(".*\\.\\d*("+HLog.META_HLOG_FILE_EXTN+")*");
54  
55    /**
56     * @param filename
57     *          name of the file to validate
58     * @return <tt>true</tt> if the filename matches an HLog, <tt>false</tt>
59     *         otherwise
60     */
61    public static boolean validateHLogFilename(String filename) {
62      return pattern.matcher(filename).matches();
63    }
64  
65    /**
66     * Construct the HLog directory name
67     *
68     * @param serverName
69     *          Server name formatted as described in {@link ServerName}
70     * @return the relative HLog directory name, e.g.
71     *         <code>.logs/1.example.org,60030,12345</code> if
72     *         <code>serverName</code> passed is
73     *         <code>1.example.org,60030,12345</code>
74     */
75    public static String getHLogDirectoryName(final String serverName) {
76      StringBuilder dirName = new StringBuilder(HConstants.HREGION_LOGDIR_NAME);
77      dirName.append("/");
78      dirName.append(serverName);
79      return dirName.toString();
80    }
81  
82    /**
83     * @param regiondir
84     *          This regions directory in the filesystem.
85     * @return The directory that holds recovered edits files for the region
86     *         <code>regiondir</code>
87     */
88    public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {
89      return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);
90    }
91  
92    /**
93     * Move aside a bad edits file.
94     *
95     * @param fs
96     * @param edits
97     *          Edits file to move aside.
98     * @return The name of the moved aside file.
99     * @throws IOException
100    */
101   public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)
102       throws IOException {
103     Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."
104         + System.currentTimeMillis());
105     if (!fs.rename(edits, moveAsideName)) {
106       LOG.warn("Rename failed from " + edits + " to " + moveAsideName);
107     }
108     return moveAsideName;
109   }
110 
111   /**
112    * @param path
113    *          - the path to analyze. Expected format, if it's in hlog directory:
114    *          / [base directory for hbase] / hbase / .logs / ServerName /
115    *          logfile
116    * @return null if it's not a log file. Returns the ServerName of the region
117    *         server that created this log file otherwise.
118    */
119   public static ServerName getServerNameFromHLogDirectoryName(
120       Configuration conf, String path) throws IOException {
121     if (path == null
122         || path.length() <= HConstants.HREGION_LOGDIR_NAME.length()) {
123       return null;
124     }
125 
126     if (conf == null) {
127       throw new IllegalArgumentException("parameter conf must be set");
128     }
129 
130     final String rootDir = conf.get(HConstants.HBASE_DIR);
131     if (rootDir == null || rootDir.isEmpty()) {
132       throw new IllegalArgumentException(HConstants.HBASE_DIR
133           + " key not found in conf.");
134     }
135 
136     final StringBuilder startPathSB = new StringBuilder(rootDir);
137     if (!rootDir.endsWith("/"))
138       startPathSB.append('/');
139     startPathSB.append(HConstants.HREGION_LOGDIR_NAME);
140     if (!HConstants.HREGION_LOGDIR_NAME.endsWith("/"))
141       startPathSB.append('/');
142     final String startPath = startPathSB.toString();
143 
144     String fullPath;
145     try {
146       fullPath = FileSystem.get(conf).makeQualified(new Path(path)).toString();
147     } catch (IllegalArgumentException e) {
148       LOG.info("Call to makeQualified failed on " + path + " " + e.getMessage());
149       return null;
150     }
151 
152     if (!fullPath.startsWith(startPath)) {
153       return null;
154     }
155 
156     final String serverNameAndFile = fullPath.substring(startPath.length());
157 
158     if (serverNameAndFile.indexOf('/') < "a,0,0".length()) {
159       // Either it's a file (not a directory) or it's not a ServerName format
160       return null;
161     }
162 
163     Path p = new Path(path);
164     return getServerNameFromHLogDirectoryName(p);
165   }
166 
167   /**
168    * This function returns region server name from a log file name which is in either format:
169    * hdfs://<name node>/hbase/.logs/<server name>-splitting/... or hdfs://<name
170    * node>/hbase/.logs/<server name>/...
171    * @param logFile
172    * @return null if the passed in logFile isn't a valid HLog file path
173    */
174   public static ServerName getServerNameFromHLogDirectoryName(Path logFile) {
175     Path logDir = logFile.getParent();
176     String logDirName = logDir.getName();
177     if (logDirName.equals(HConstants.HREGION_LOGDIR_NAME)) {
178       logDir = logFile;
179       logDirName = logDir.getName();
180     }
181     ServerName serverName = null;
182     if (logDirName.endsWith(HLog.SPLITTING_EXT)) {
183       logDirName = logDirName.substring(0, logDirName.length() - HLog.SPLITTING_EXT.length());
184     }
185     try {
186       serverName = ServerName.parseServerName(logDirName);
187     } catch (IllegalArgumentException ex) {
188       serverName = null;
189       LOG.warn("Cannot parse a server name from path=" + logFile + "; " + ex.getMessage());
190     }
191     if (serverName != null && serverName.getStartcode() < 0) {
192       LOG.warn("Invalid log file path=" + logFile);
193       return null;
194     }
195     return serverName;
196   }
197 
198   /**
199    * Returns sorted set of edit files made by wal-log splitter, excluding files
200    * with '.temp' suffix.
201    *
202    * @param fs
203    * @param regiondir
204    * @return Files in passed <code>regiondir</code> as a sorted set.
205    * @throws IOException
206    */
207   public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,
208       final Path regiondir) throws IOException {
209     NavigableSet<Path> filesSorted = new TreeSet<Path>();
210     Path editsdir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
211     if (!fs.exists(editsdir))
212       return filesSorted;
213     FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {
214       @Override
215       public boolean accept(Path p) {
216         boolean result = false;
217         try {
218           // Return files and only files that match the editfile names pattern.
219           // There can be other files in this directory other than edit files.
220           // In particular, on error, we'll move aside the bad edit file giving
221           // it a timestamp suffix. See moveAsideBadEditsFile.
222           Matcher m = HLog.EDITFILES_NAME_PATTERN.matcher(p.getName());
223           result = fs.isFile(p) && m.matches();
224           // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,
225           // because it means splithlog thread is writting this file.
226           if (p.getName().endsWith(HLog.RECOVERED_LOG_TMPFILE_SUFFIX)) {
227             result = false;
228           }
229         } catch (IOException e) {
230           LOG.warn("Failed isFile check on " + p);
231         }
232         return result;
233       }
234     });
235     if (files == null)
236       return filesSorted;
237     for (FileStatus status : files) {
238       filesSorted.add(status.getPath());
239     }
240     return filesSorted;
241   }
242 
243   public static boolean isMetaFile(Path p) {
244     return isMetaFile(p.getName());
245   }
246 
247   public static boolean isMetaFile(String p) {
248     if (p != null && p.endsWith(HLog.META_HLOG_FILE_EXTN)) {
249       return true;
250     }
251     return false;
252   }
253 
254   /**
255    * Write the marker that a compaction has succeeded and is about to be committed.
256    * This provides info to the HMaster to allow it to recover the compaction if
257    * this regionserver dies in the middle (This part is not yet implemented). It also prevents
258    * the compaction from finishing if this regionserver has already lost its lease on the log.
259    * @param sequenceId Used by HLog to get sequence Id for the waledit.
260    */
261   public static void writeCompactionMarker(HLog log, HTableDescriptor htd, HRegionInfo info,
262       final CompactionDescriptor c, AtomicLong sequenceId) throws IOException {
263     TableName tn = TableName.valueOf(c.getTableName().toByteArray());
264     HLogKey key = new HLogKey(info.getEncodedNameAsBytes(), tn);
265     log.appendNoSync(htd, info, key, WALEdit.createCompaction(c), sequenceId, false, null);
266     log.sync();
267     if (LOG.isTraceEnabled()) {
268       LOG.trace("Appended compaction marker " + TextFormat.shortDebugString(c));
269     }
270   }
271 }