View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver.wal;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.NavigableSet;
25  import java.util.TreeSet;
26  import java.util.UUID;
27  import java.util.concurrent.atomic.AtomicLong;
28  import java.util.regex.Matcher;
29  import java.util.regex.Pattern;
30  
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  import org.apache.hadoop.conf.Configuration;
34  import org.apache.hadoop.fs.FileStatus;
35  import org.apache.hadoop.fs.FileSystem;
36  import org.apache.hadoop.fs.Path;
37  import org.apache.hadoop.fs.PathFilter;
38  import org.apache.hadoop.hbase.TableName;
39  import org.apache.hadoop.hbase.HConstants;
40  import org.apache.hadoop.hbase.HRegionInfo;
41  import org.apache.hadoop.hbase.HTableDescriptor;
42  import org.apache.hadoop.hbase.ServerName;
43  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
44  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
45  import org.apache.hadoop.hbase.util.FSUtils;
46  
47  import com.google.protobuf.TextFormat;
48  
49  public class HLogUtil {
50    static final Log LOG = LogFactory.getLog(HLogUtil.class);
51  
52    /**
53     * Pattern used to validate a HLog file name
54     */
55    private static final Pattern pattern =
56        Pattern.compile(".*\\.\\d*("+HLog.META_HLOG_FILE_EXTN+")*");
57  
58    /**
59     * @param filename
60     *          name of the file to validate
61     * @return <tt>true</tt> if the filename matches an HLog, <tt>false</tt>
62     *         otherwise
63     */
64    public static boolean validateHLogFilename(String filename) {
65      return pattern.matcher(filename).matches();
66    }
67  
68    /**
69     * Construct the HLog directory name
70     *
71     * @param serverName
72     *          Server name formatted as described in {@link ServerName}
73     * @return the relative HLog directory name, e.g.
74     *         <code>.logs/1.example.org,60030,12345</code> if
75     *         <code>serverName</code> passed is
76     *         <code>1.example.org,60030,12345</code>
77     */
78    public static String getHLogDirectoryName(final String serverName) {
79      StringBuilder dirName = new StringBuilder(HConstants.HREGION_LOGDIR_NAME);
80      dirName.append("/");
81      dirName.append(serverName);
82      return dirName.toString();
83    }
84  
85    /**
86     * @param regiondir
87     *          This regions directory in the filesystem.
88     * @return The directory that holds recovered edits files for the region
89     *         <code>regiondir</code>
90     */
91    public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {
92      return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);
93    }
94  
95    /**
96     * Move aside a bad edits file.
97     *
98     * @param fs
99     * @param edits
100    *          Edits file to move aside.
101    * @return The name of the moved aside file.
102    * @throws IOException
103    */
104   public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)
105       throws IOException {
106     Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."
107         + System.currentTimeMillis());
108     if (!fs.rename(edits, moveAsideName)) {
109       LOG.warn("Rename failed from " + edits + " to " + moveAsideName);
110     }
111     return moveAsideName;
112   }
113 
114   /**
115    * @param path
116    *          - the path to analyze. Expected format, if it's in hlog directory:
117    *          / [base directory for hbase] / hbase / .logs / ServerName /
118    *          logfile
119    * @return null if it's not a log file. Returns the ServerName of the region
120    *         server that created this log file otherwise.
121    */
122   public static ServerName getServerNameFromHLogDirectoryName(
123       Configuration conf, String path) throws IOException {
124     if (path == null
125         || path.length() <= HConstants.HREGION_LOGDIR_NAME.length()) {
126       return null;
127     }
128 
129     if (conf == null) {
130       throw new IllegalArgumentException("parameter conf must be set");
131     }
132 
133     final String rootDir = conf.get(HConstants.HBASE_DIR);
134     if (rootDir == null || rootDir.isEmpty()) {
135       throw new IllegalArgumentException(HConstants.HBASE_DIR
136           + " key not found in conf.");
137     }
138 
139     final StringBuilder startPathSB = new StringBuilder(rootDir);
140     if (!rootDir.endsWith("/"))
141       startPathSB.append('/');
142     startPathSB.append(HConstants.HREGION_LOGDIR_NAME);
143     if (!HConstants.HREGION_LOGDIR_NAME.endsWith("/"))
144       startPathSB.append('/');
145     final String startPath = startPathSB.toString();
146 
147     String fullPath;
148     try {
149       fullPath = FileSystem.get(conf).makeQualified(new Path(path)).toString();
150     } catch (IllegalArgumentException e) {
151       LOG.info("Call to makeQualified failed on " + path + " " + e.getMessage());
152       return null;
153     }
154 
155     if (!fullPath.startsWith(startPath)) {
156       return null;
157     }
158 
159     final String serverNameAndFile = fullPath.substring(startPath.length());
160 
161     if (serverNameAndFile.indexOf('/') < "a,0,0".length()) {
162       // Either it's a file (not a directory) or it's not a ServerName format
163       return null;
164     }
165 
166     Path p = new Path(path);
167     return getServerNameFromHLogDirectoryName(p);
168   }
169 
170   /**
171    * This function returns region server name from a log file name which is in either format:
172    * hdfs://<name node>/hbase/.logs/<server name>-splitting/... or hdfs://<name
173    * node>/hbase/.logs/<server name>/...
174    * @param logFile
175    * @return null if the passed in logFile isn't a valid HLog file path
176    */
177   public static ServerName getServerNameFromHLogDirectoryName(Path logFile) {
178     Path logDir = logFile.getParent();
179     String logDirName = logDir.getName();
180     if (logDirName.equals(HConstants.HREGION_LOGDIR_NAME)) {
181       logDir = logFile;
182       logDirName = logDir.getName();
183     }
184     ServerName serverName = null;
185     if (logDirName.endsWith(HLog.SPLITTING_EXT)) {
186       logDirName = logDirName.substring(0, logDirName.length() - HLog.SPLITTING_EXT.length());
187     }
188     try {
189       serverName = ServerName.parseServerName(logDirName);
190     } catch (IllegalArgumentException ex) {
191       serverName = null;
192       LOG.warn("Cannot parse a server name from path=" + logFile + "; " + ex.getMessage());
193     }
194     if (serverName != null && serverName.getStartcode() < 0) {
195       LOG.warn("Invalid log file path=" + logFile);
196       return null;
197     }
198     return serverName;
199   }
200 
201   /**
202    * Returns sorted set of edit files made by wal-log splitter, excluding files
203    * with '.temp' suffix.
204    *
205    * @param fs
206    * @param regiondir
207    * @return Files in passed <code>regiondir</code> as a sorted set.
208    * @throws IOException
209    */
210   public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,
211       final Path regiondir) throws IOException {
212     NavigableSet<Path> filesSorted = new TreeSet<Path>();
213     Path editsdir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
214     if (!fs.exists(editsdir))
215       return filesSorted;
216     FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {
217       @Override
218       public boolean accept(Path p) {
219         boolean result = false;
220         try {
221           // Return files and only files that match the editfile names pattern.
222           // There can be other files in this directory other than edit files.
223           // In particular, on error, we'll move aside the bad edit file giving
224           // it a timestamp suffix. See moveAsideBadEditsFile.
225           Matcher m = HLog.EDITFILES_NAME_PATTERN.matcher(p.getName());
226           result = fs.isFile(p) && m.matches();
227           // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,
228           // because it means splithlog thread is writting this file.
229           if (p.getName().endsWith(HLog.RECOVERED_LOG_TMPFILE_SUFFIX)) {
230             result = false;
231           }
232         } catch (IOException e) {
233           LOG.warn("Failed isFile check on " + p);
234         }
235         return result;
236       }
237     });
238     if (files == null)
239       return filesSorted;
240     for (FileStatus status : files) {
241       filesSorted.add(status.getPath());
242     }
243     return filesSorted;
244   }
245 
246   public static boolean isMetaFile(Path p) {
247     return isMetaFile(p.getName());
248   }
249 
250   public static boolean isMetaFile(String p) {
251     if (p != null && p.endsWith(HLog.META_HLOG_FILE_EXTN)) {
252       return true;
253     }
254     return false;
255   }
256 
257   /**
258    * Write the marker that a compaction has succeeded and is about to be committed.
259    * This provides info to the HMaster to allow it to recover the compaction if
260    * this regionserver dies in the middle (This part is not yet implemented). It also prevents
261    * the compaction from finishing if this regionserver has already lost its lease on the log.
262    * @param sequenceId Used by HLog to get sequence Id for the waledit.
263    */
264   public static void writeCompactionMarker(HLog log, HTableDescriptor htd, HRegionInfo info,
265       final CompactionDescriptor c, AtomicLong sequenceId) throws IOException {
266     WALEdit e = WALEdit.createCompaction(c);
267     long now = EnvironmentEdgeManager.currentTimeMillis();
268     TableName tn = TableName.valueOf(c.getTableName().toByteArray());
269     log.appendNoSync(info, tn, e, new ArrayList<UUID>(), now, htd, sequenceId, false,
270       HConstants.NO_NONCE, HConstants.NO_NONCE);
271     log.sync();
272 
273     if (LOG.isTraceEnabled()) {
274       LOG.trace("Appended compaction marker " + TextFormat.shortDebugString(c));
275     }
276   }
277 }