View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.wal;
20  
21  import java.io.Closeable;
22  import java.io.DataInput;
23  import java.io.DataOutput;
24  import java.io.IOException;
25  import java.util.List;
26  import java.util.regex.Pattern;
27  
28  import com.google.common.annotations.VisibleForTesting;
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.hbase.classification.InterfaceAudience;
32  import org.apache.hadoop.hbase.classification.InterfaceStability;
33  import org.apache.hadoop.conf.Configuration;
34  import org.apache.hadoop.fs.FSDataInputStream;
35  import org.apache.hadoop.fs.FileSystem;
36  import org.apache.hadoop.fs.Path;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.ServerName;
39  import org.apache.hadoop.hbase.util.FSUtils;
40  
41  // imports for things that haven't moved from regionserver.wal yet.
42  import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
43  import org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter;
44  import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
45  
46  /**
47   * A WAL Provider that returns a single thread safe WAL that writes to HDFS.
48   * By default, this implementation picks a directory in HDFS based on a combination of
49   * <ul>
50   *   <li>the HBase root directory
51   *   <li>HConstants.HREGION_LOGDIR_NAME
52   *   <li>the given factory's factoryId (usually identifying the regionserver by host:port)
53   * </ul>
54   * It also uses the providerId to diffentiate among files.
55   *
56   */
57  @InterfaceAudience.Private
58  @InterfaceStability.Evolving
59  public class DefaultWALProvider implements WALProvider {
60    private static final Log LOG = LogFactory.getLog(DefaultWALProvider.class);
61  
62    // Only public so classes back in regionserver.wal can access
63    public interface Reader extends WAL.Reader {
64      /**
65       * @param fs File system.
66       * @param path Path.
67       * @param c Configuration.
68       * @param s Input stream that may have been pre-opened by the caller; may be null.
69       */
70      void init(FileSystem fs, Path path, Configuration c, FSDataInputStream s) throws IOException;
71    }
72  
73    // Only public so classes back in regionserver.wal can access
74    public interface Writer extends WALProvider.Writer {
75      void init(FileSystem fs, Path path, Configuration c, boolean overwritable) throws IOException;
76    }
77  
78    protected FSHLog log = null;
79  
80    /**
81     * @param factory factory that made us, identity used for FS layout. may not be null
82     * @param conf may not be null
83     * @param listeners may be null
84     * @param providerId differentiate between providers from one facotry, used for FS layout. may be
85     *                   null
86     */
87    @Override
88    public void init(final WALFactory factory, final Configuration conf,
89        final List<WALActionsListener> listeners, String providerId) throws IOException {
90      if (null != log) {
91        throw new IllegalStateException("WALProvider.init should only be called once.");
92      }
93      if (null == providerId) {
94        providerId = DEFAULT_PROVIDER_ID;
95      }
96      final String logPrefix = factory.factoryId + WAL_FILE_NAME_DELIMITER + providerId;
97      log = new FSHLog(FileSystem.get(conf), FSUtils.getRootDir(conf),
98          getWALDirectoryName(factory.factoryId), HConstants.HREGION_OLDLOGDIR_NAME, conf, listeners,
99          true, logPrefix, META_WAL_PROVIDER_ID.equals(providerId) ? META_WAL_PROVIDER_ID : null);
100   }
101 
102   @Override
103   public WAL getWAL(final byte[] identifier) throws IOException {
104    return log;
105   }
106 
107   @Override
108   public void close() throws IOException {
109     log.close();
110   }
111 
112   @Override
113   public void shutdown() throws IOException {
114     log.shutdown();
115   }
116 
117   // should be package private; more visible for use in FSHLog
118   public static final String WAL_FILE_NAME_DELIMITER = ".";
119   /** The hbase:meta region's WAL filename extension */
120   @VisibleForTesting
121   public static final String META_WAL_PROVIDER_ID = ".meta";
122   static final String DEFAULT_PROVIDER_ID = "default";
123 
124   // Implementation details that currently leak in tests or elsewhere follow
125   /** File Extension used while splitting an WAL into regions (HBASE-2312) */
126   public static final String SPLITTING_EXT = "-splitting";
127 
128   /**
129    * iff the given WALFactory is using the DefaultWALProvider for meta and/or non-meta,
130    * count the number of files (rolled and active). if either of them aren't, count 0
131    * for that provider.
132    * @param walFactory may not be null.
133    */
134   public static long getNumLogFiles(WALFactory walFactory) {
135     long result = 0;
136     if (walFactory.provider instanceof DefaultWALProvider) {
137       result += ((FSHLog)((DefaultWALProvider)walFactory.provider).log).getNumLogFiles();
138     }
139     WALProvider meta = walFactory.metaProvider.get();
140     if (meta instanceof DefaultWALProvider) {
141       result += ((FSHLog)((DefaultWALProvider)meta).log).getNumLogFiles();
142     }
143     return result;
144   }
145 
146   /**
147    * iff the given WALFactory is using the DefaultWALProvider for meta and/or non-meta,
148    * count the size of files (rolled and active). if either of them aren't, count 0
149    * for that provider.
150    * @param walFactory may not be null.
151    */
152   public static long getLogFileSize(WALFactory walFactory) {
153     long result = 0;
154     if (walFactory.provider instanceof DefaultWALProvider) {
155       result += ((FSHLog)((DefaultWALProvider)walFactory.provider).log).getLogFileSize();
156     }
157     WALProvider meta = walFactory.metaProvider.get();
158     if (meta instanceof DefaultWALProvider) {
159       result += ((FSHLog)((DefaultWALProvider)meta).log).getLogFileSize();
160     }
161     return result;
162   }
163 
164   /**
165    * returns the number of rolled WAL files.
166    */
167   @VisibleForTesting
168   public static int getNumRolledLogFiles(WAL wal) {
169     return ((FSHLog)wal).getNumRolledLogFiles();
170   }
171 
172   /**
173    * return the current filename from the current wal.
174    */
175   @VisibleForTesting
176   public static Path getCurrentFileName(final WAL wal) {
177     return ((FSHLog)wal).getCurrentFileName();
178   }
179 
180   /**
181    * request a log roll, but don't actually do it.
182    */
183   @VisibleForTesting
184   static void requestLogRoll(final WAL wal) {
185     ((FSHLog)wal).requestLogRoll();
186   }
187 
188   /**
189    * It returns the file create timestamp from the file name.
190    * For name format see {@link #validateWALFilename(String)}
191    * public until remaining tests move to o.a.h.h.wal
192    * @param wal must not be null
193    * @return the file number that is part of the WAL file name
194    */
195   @VisibleForTesting
196   public static long extractFileNumFromWAL(final WAL wal) {
197     final Path walName = ((FSHLog)wal).getCurrentFileName();
198     if (walName == null) {
199       throw new IllegalArgumentException("The WAL path couldn't be null");
200     }
201     final String[] walPathStrs = walName.toString().split("\\" + WAL_FILE_NAME_DELIMITER);
202     return Long.parseLong(walPathStrs[walPathStrs.length - (isMetaFile(walName) ? 2:1)]);
203   }
204 
205   /**
206    * Pattern used to validate a WAL file name
207    * see {@link #validateWALFilename(String)} for description.
208    */
209   private static final Pattern pattern = Pattern.compile(".*\\.\\d*("+META_WAL_PROVIDER_ID+")*");
210 
211   /**
212    * A WAL file name is of the format:
213    * &lt;wal-name&gt;{@link #WAL_FILE_NAME_DELIMITER}&lt;file-creation-timestamp&gt;[.meta].
214    *
215    * provider-name is usually made up of a server-name and a provider-id
216    *
217    * @param filename name of the file to validate
218    * @return <tt>true</tt> if the filename matches an WAL, <tt>false</tt>
219    *         otherwise
220    */
221   public static boolean validateWALFilename(String filename) {
222     return pattern.matcher(filename).matches();
223   }
224 
225   /**
226    * Construct the directory name for all WALs on a given server.
227    *
228    * @param serverName
229    *          Server name formatted as described in {@link ServerName}
230    * @return the relative WAL directory name, e.g.
231    *         <code>.logs/1.example.org,60030,12345</code> if
232    *         <code>serverName</code> passed is
233    *         <code>1.example.org,60030,12345</code>
234    */
235   public static String getWALDirectoryName(final String serverName) {
236     StringBuilder dirName = new StringBuilder(HConstants.HREGION_LOGDIR_NAME);
237     dirName.append("/");
238     dirName.append(serverName);
239     return dirName.toString();
240   }
241 
242   /**
243    * Pulls a ServerName out of a Path generated according to our layout rules.
244    *
245    * In the below layouts, this method ignores the format of the logfile component.
246    *
247    * Current format:
248    *
249    * [base directory for hbase]/hbase/.logs/ServerName/logfile
250    *      or
251    * [base directory for hbase]/hbase/.logs/ServerName-splitting/logfile
252    *
253    * Expected to work for individual log files and server-specific directories.
254    *
255    * @return null if it's not a log file. Returns the ServerName of the region
256    *         server that created this log file otherwise.
257    */
258   public static ServerName getServerNameFromWALDirectoryName(Configuration conf, String path)
259       throws IOException {
260     if (path == null
261         || path.length() <= HConstants.HREGION_LOGDIR_NAME.length()) {
262       return null;
263     }
264 
265     if (conf == null) {
266       throw new IllegalArgumentException("parameter conf must be set");
267     }
268 
269     final String rootDir = conf.get(HConstants.HBASE_DIR);
270     if (rootDir == null || rootDir.isEmpty()) {
271       throw new IllegalArgumentException(HConstants.HBASE_DIR
272           + " key not found in conf.");
273     }
274 
275     final StringBuilder startPathSB = new StringBuilder(rootDir);
276     if (!rootDir.endsWith("/"))
277       startPathSB.append('/');
278     startPathSB.append(HConstants.HREGION_LOGDIR_NAME);
279     if (!HConstants.HREGION_LOGDIR_NAME.endsWith("/"))
280       startPathSB.append('/');
281     final String startPath = startPathSB.toString();
282 
283     String fullPath;
284     try {
285       fullPath = FileSystem.get(conf).makeQualified(new Path(path)).toString();
286     } catch (IllegalArgumentException e) {
287       LOG.info("Call to makeQualified failed on " + path + " " + e.getMessage());
288       return null;
289     }
290 
291     if (!fullPath.startsWith(startPath)) {
292       return null;
293     }
294 
295     final String serverNameAndFile = fullPath.substring(startPath.length());
296 
297     if (serverNameAndFile.indexOf('/') < "a,0,0".length()) {
298       // Either it's a file (not a directory) or it's not a ServerName format
299       return null;
300     }
301 
302     Path p = new Path(path);
303     return getServerNameFromWALDirectoryName(p);
304   }
305 
306   /**
307    * This function returns region server name from a log file name which is in one of the following
308    * formats:
309    * <ul>
310    *   <li>hdfs://&lt;name node&gt;/hbase/.logs/&lt;server name&gt;-splitting/...</li>
311    *   <li>hdfs://&lt;name node&gt;/hbase/.logs/&lt;server name&gt;/...</li>
312    * </ul>
313    * @param logFile
314    * @return null if the passed in logFile isn't a valid WAL file path
315    */
316   public static ServerName getServerNameFromWALDirectoryName(Path logFile) {
317     String logDirName = logFile.getParent().getName();
318     // We were passed the directory and not a file in it.
319     if (logDirName.equals(HConstants.HREGION_LOGDIR_NAME)) {
320       logDirName = logFile.getName();
321     }
322     ServerName serverName = null;
323     if (logDirName.endsWith(SPLITTING_EXT)) {
324       logDirName = logDirName.substring(0, logDirName.length() - SPLITTING_EXT.length());
325     }
326     try {
327       serverName = ServerName.parseServerName(logDirName);
328     } catch (IllegalArgumentException ex) {
329       serverName = null;
330       LOG.warn("Cannot parse a server name from path=" + logFile + "; " + ex.getMessage());
331     }
332     if (serverName != null && serverName.getStartcode() < 0) {
333       LOG.warn("Invalid log file path=" + logFile);
334       serverName = null;
335     }
336     return serverName;
337   }
338 
339   public static boolean isMetaFile(Path p) {
340     return isMetaFile(p.getName());
341   }
342 
343   public static boolean isMetaFile(String p) {
344     if (p != null && p.endsWith(META_WAL_PROVIDER_ID)) {
345       return true;
346     }
347     return false;
348   }
349 
350   /**
351    * public because of FSHLog. Should be package-private
352    */
353   public static Writer createWriter(final Configuration conf, final FileSystem fs, final Path path,
354       final boolean overwritable)
355       throws IOException {
356     // Configuration already does caching for the Class lookup.
357     Class<? extends Writer> logWriterClass = conf.getClass("hbase.regionserver.hlog.writer.impl",
358         ProtobufLogWriter.class, Writer.class);
359     Writer writer = null;
360     try {
361       writer = logWriterClass.newInstance();
362       writer.init(fs, path, conf, overwritable);
363       return writer;
364     } catch (Exception e) {
365       LOG.debug("Error instantiating log writer.", e);
366       if (writer != null) {
367         try{
368           writer.close();
369         } catch(IOException ee){
370           LOG.error("cannot close log writer", ee);
371         }
372       }
373       throw new IOException("cannot get log writer", e);
374     }
375   }
376 
377   /**
378    * Get prefix of the log from its name, assuming WAL name in format of
379    * log_prefix.filenumber.log_suffix @see {@link FSHLog#getCurrentFileName()}
380    * @param name Name of the WAL to parse
381    * @return prefix of the log
382    */
383   public static String getWALPrefixFromWALName(String name) {
384     int endIndex = name.replaceAll(META_WAL_PROVIDER_ID, "").lastIndexOf(".");
385     return name.substring(0, endIndex);
386   }
387 
388 }