View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import java.io.FileNotFoundException;
23  import java.io.IOException;
24  import java.util.regex.Matcher;
25  import java.util.regex.Pattern;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.hbase.classification.InterfaceAudience;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.fs.FileStatus;
32  import org.apache.hadoop.fs.FileSystem;
33  import org.apache.hadoop.fs.Path;
34  import org.apache.hadoop.hbase.HDFSBlocksDistribution;
35  import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
36  import org.apache.hadoop.hbase.io.HFileLink;
37  import org.apache.hadoop.hbase.io.HalfStoreFileReader;
38  import org.apache.hadoop.hbase.io.Reference;
39  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
40  import org.apache.hadoop.hbase.util.FSUtils;
41  
42  /**
43   * Describe a StoreFile (hfile, reference, link)
44   */
45  @InterfaceAudience.Private
46  public class StoreFileInfo implements Comparable<StoreFileInfo> {
47    public static final Log LOG = LogFactory.getLog(StoreFileInfo.class);
48  
49    /**
50     * A non-capture group, for hfiles, so that this can be embedded.
51     * HFiles are uuid ([0-9a-z]+). Bulk loaded hfiles has (_SeqId_[0-9]+_) has suffix.
52     */
53    public static final String HFILE_NAME_REGEX = "[0-9a-f]+(?:_SeqId_[0-9]+_)?";
54  
55    /** Regex that will work for hfiles */
56    private static final Pattern HFILE_NAME_PATTERN =
57      Pattern.compile("^(" + HFILE_NAME_REGEX + ")");
58  
59    /**
60     * Regex that will work for straight reference names (<hfile>.<parentEncRegion>)
61     * and hfilelink reference names (<table>=<region>-<hfile>.<parentEncRegion>)
62     * If reference, then the regex has more than just one group.
63     * Group 1, hfile/hfilelink pattern, is this file's id.
64     * Group 2 '(.+)' is the reference's parent region name.
65     */
66    private static final Pattern REF_NAME_PATTERN =
67      Pattern.compile(String.format("^(%s|%s)\\.(.+)$",
68        HFILE_NAME_REGEX, HFileLink.LINK_NAME_REGEX));
69  
70    // Configuration
71    private Configuration conf;
72  
73    // HDFS blocks distribution information
74    private HDFSBlocksDistribution hdfsBlocksDistribution = null;
75  
76    // If this storefile references another, this is the reference instance.
77    private final Reference reference;
78  
79    // If this storefile is a link to another, this is the link instance.
80    private final HFileLink link;
81  
82    // FileSystem information for the file.
83    private final FileStatus fileStatus;
84  
85    private RegionCoprocessorHost coprocessorHost;
86  
87    /**
88     * Create a Store File Info
89     * @param conf the {@link Configuration} to use
90     * @param fs The current file system to use.
91     * @param path The {@link Path} of the file
92     */
93    public StoreFileInfo(final Configuration conf, final FileSystem fs, final Path path)
94        throws IOException {
95      this(conf, fs, fs.getFileStatus(path));
96    }
97  
98    /**
99     * Create a Store File Info
100    * @param conf the {@link Configuration} to use
101    * @param fs The current file system to use.
102    * @param fileStatus The {@link FileStatus} of the file
103    */
104   public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus)
105       throws IOException {
106     this.conf = conf;
107     this.fileStatus = fileStatus;
108     Path p = fileStatus.getPath();
109     if (HFileLink.isHFileLink(p)) {
110       // HFileLink
111       this.reference = null;
112       this.link = new HFileLink(conf, p);
113       if (LOG.isTraceEnabled()) LOG.trace(p + " is a link");
114     } else if (isReference(p)) {
115       this.reference = Reference.read(fs, p);
116       Path referencePath = getReferredToFile(p);
117       if (HFileLink.isHFileLink(referencePath)) {
118         // HFileLink Reference
119         this.link = new HFileLink(conf, referencePath);
120       } else {
121         // Reference
122         this.link = null;
123       }
124       if (LOG.isTraceEnabled()) LOG.trace(p + " is a " + reference.getFileRegion() +
125         " reference to " + referencePath);
126     } else if (isHFile(p)) {
127       // HFile
128       this.reference = null;
129       this.link = null;
130     } else {
131       throw new IOException("path=" + p + " doesn't look like a valid StoreFile");
132     }
133   }
134 
135   /**
136    * Create a Store File Info from an HFileLink
137    * @param conf the {@link Configuration} to use
138    * @param fs The current file system to use.
139    * @param fileStatus The {@link FileStatus} of the file
140    */
141   public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus,
142       final HFileLink link)
143       throws IOException {
144     this.conf = conf;
145     this.fileStatus = fileStatus;
146       // HFileLink
147     this.reference = null;
148     this.link = link;
149   }
150 
151   /**
152    * Create a Store File Info from an HFileLink
153    * @param conf
154    * @param fs
155    * @param fileStatus
156    * @param reference
157    * @throws IOException
158    */
159   public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus,
160       final Reference reference)
161       throws IOException {
162     this.conf = conf;
163     this.fileStatus = fileStatus;
164     this.reference = reference;
165     this.link = null;
166   }
167 
168   /**
169    * Sets the region coprocessor env.
170    * @param coprocessorHost
171    */
172   public void setRegionCoprocessorHost(RegionCoprocessorHost coprocessorHost) {
173     this.coprocessorHost = coprocessorHost;
174   }
175 
176   /*
177    * @return the Reference object associated to this StoreFileInfo.
178    *         null if the StoreFile is not a reference.
179    */
180   public Reference getReference() {
181     return this.reference;
182   }
183 
184   /** @return True if the store file is a Reference */
185   public boolean isReference() {
186     return this.reference != null;
187   }
188 
189   /** @return True if the store file is a top Reference */
190   public boolean isTopReference() {
191     return this.reference != null && Reference.isTopFileRegion(this.reference.getFileRegion());
192   }
193 
194   /** @return True if the store file is a link */
195   public boolean isLink() {
196     return this.link != null && this.reference == null;
197   }
198 
199   /** @return the HDFS block distribution */
200   public HDFSBlocksDistribution getHDFSBlockDistribution() {
201     return this.hdfsBlocksDistribution;
202   }
203 
204   /**
205    * Open a Reader for the StoreFile
206    * @param fs The current file system to use.
207    * @param cacheConf The cache configuration and block cache reference.
208    * @return The StoreFile.Reader for the file
209    */
210   public StoreFile.Reader open(final FileSystem fs,
211       final CacheConfig cacheConf) throws IOException {
212     FSDataInputStreamWrapper in;
213     FileStatus status;
214 
215     if (this.link != null) {
216       // HFileLink
217       in = new FSDataInputStreamWrapper(fs, this.link);
218       status = this.link.getFileStatus(fs);
219     } else if (this.reference != null) {
220       // HFile Reference
221       Path referencePath = getReferredToFile(this.getPath());
222       in = new FSDataInputStreamWrapper(fs, referencePath);
223       status = fs.getFileStatus(referencePath);
224     } else {
225       in = new FSDataInputStreamWrapper(fs, this.getPath());
226       status = fileStatus;
227     }
228     long length = status.getLen();
229     hdfsBlocksDistribution = computeHDFSBlocksDistribution(fs);
230 
231     StoreFile.Reader reader = null;
232     if (this.coprocessorHost != null) {
233       reader = this.coprocessorHost.preStoreFileReaderOpen(fs, this.getPath(), in, length,
234         cacheConf, reference);
235     }
236     if (reader == null) {
237       if (this.reference != null) {
238         reader = new HalfStoreFileReader(fs, this.getPath(), in, length, cacheConf, reference,
239           conf);
240       } else {
241         reader = new StoreFile.Reader(fs, this.getPath(), in, length, cacheConf, conf);
242       }
243     }
244     if (this.coprocessorHost != null) {
245       reader = this.coprocessorHost.postStoreFileReaderOpen(fs, this.getPath(), in, length,
246         cacheConf, reference, reader);
247     }
248     return reader;
249   }
250 
251   /**
252    * Compute the HDFS Block Distribution for this StoreFile
253    */
254   public HDFSBlocksDistribution computeHDFSBlocksDistribution(final FileSystem fs)
255       throws IOException {
256 
257     // guard agains the case where we get the FileStatus from link, but by the time we
258     // call compute the file is moved again
259     if (this.link != null) {
260       FileNotFoundException exToThrow = null;
261       for (int i = 0; i < this.link.getLocations().length; i++) {
262         try {
263           return computeHDFSBlocksDistributionInternal(fs);
264         } catch (FileNotFoundException ex) {
265           // try the other location
266           exToThrow = ex;
267         }
268       }
269       throw exToThrow;
270     } else {
271       return computeHDFSBlocksDistributionInternal(fs);
272     }
273   }
274 
275   private HDFSBlocksDistribution computeHDFSBlocksDistributionInternal(final FileSystem fs)
276       throws IOException {
277     FileStatus status = getReferencedFileStatus(fs);
278     if (this.reference != null) {
279       return computeRefFileHDFSBlockDistribution(fs, reference, status);
280     } else {
281       return FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen());
282     }
283   }
284 
285   /**
286    * Get the {@link FileStatus} of the file referenced by this StoreFileInfo
287    * @param fs The current file system to use.
288    * @return The {@link FileStatus} of the file referenced by this StoreFileInfo
289    */
290   public FileStatus getReferencedFileStatus(final FileSystem fs) throws IOException {
291     FileStatus status;
292     if (this.reference != null) {
293       if (this.link != null) {
294         FileNotFoundException exToThrow = null;
295         for (int i = 0; i < this.link.getLocations().length; i++) {
296           // HFileLink Reference
297           try {
298             return link.getFileStatus(fs);
299           } catch (FileNotFoundException ex) {
300             // try the other location
301             exToThrow = ex;
302           }
303         }
304         throw exToThrow;
305       } else {
306         // HFile Reference
307         Path referencePath = getReferredToFile(this.getPath());
308         status = fs.getFileStatus(referencePath);
309       }
310     } else {
311       if (this.link != null) {
312         FileNotFoundException exToThrow = null;
313         for (int i = 0; i < this.link.getLocations().length; i++) {
314           // HFileLink
315           try {
316             return link.getFileStatus(fs);
317           } catch (FileNotFoundException ex) {
318             // try the other location
319             exToThrow = ex;
320           }
321         }
322         throw exToThrow;
323       } else {
324         status = this.fileStatus;
325       }
326     }
327     return status;
328   }
329 
330   /** @return The {@link Path} of the file */
331   public Path getPath() {
332     return this.fileStatus.getPath();
333   }
334 
335   /** @return The {@link FileStatus} of the file */
336   public FileStatus getFileStatus() {
337     return this.fileStatus;
338   }
339 
340   /** @return Get the modification time of the file. */
341   public long getModificationTime() {
342     return this.fileStatus.getModificationTime();
343   }
344 
345   @Override
346   public String toString() {
347     return this.getPath() +
348       (isReference() ? "-" + getReferredToFile(this.getPath()) + "-" + reference : "");
349   }
350 
351   /**
352    * @param path Path to check.
353    * @return True if the path has format of a HFile.
354    */
355   public static boolean isHFile(final Path path) {
356     return isHFile(path.getName());
357   }
358 
359   public static boolean isHFile(final String fileName) {
360     Matcher m = HFILE_NAME_PATTERN.matcher(fileName);
361     return m.matches() && m.groupCount() > 0;
362   }
363 
364   /**
365    * @param path Path to check.
366    * @return True if the path has format of a HStoreFile reference.
367    */
368   public static boolean isReference(final Path path) {
369     return isReference(path.getName());
370   }
371 
372   /**
373    * @param name file name to check.
374    * @return True if the path has format of a HStoreFile reference.
375    */
376   public static boolean isReference(final String name) {
377     Matcher m = REF_NAME_PATTERN.matcher(name);
378     return m.matches() && m.groupCount() > 1;
379   }
380 
381   /*
382    * Return path to the file referred to by a Reference.  Presumes a directory
383    * hierarchy of <code>${hbase.rootdir}/data/${namespace}/tablename/regionname/familyname</code>.
384    * @param p Path to a Reference file.
385    * @return Calculated path to parent region file.
386    * @throws IllegalArgumentException when path regex fails to match.
387    */
388   public static Path getReferredToFile(final Path p) {
389     Matcher m = REF_NAME_PATTERN.matcher(p.getName());
390     if (m == null || !m.matches()) {
391       LOG.warn("Failed match of store file name " + p.toString());
392       throw new IllegalArgumentException("Failed match of store file name " +
393           p.toString());
394     }
395 
396     // Other region name is suffix on the passed Reference file name
397     String otherRegion = m.group(2);
398     // Tabledir is up two directories from where Reference was written.
399     Path tableDir = p.getParent().getParent().getParent();
400     String nameStrippedOfSuffix = m.group(1);
401     LOG.debug("reference '" + p + "' to region=" + otherRegion + " hfile=" + nameStrippedOfSuffix);
402 
403     // Build up new path with the referenced region in place of our current
404     // region in the reference path.  Also strip regionname suffix from name.
405     return new Path(new Path(new Path(tableDir, otherRegion),
406       p.getParent().getName()), nameStrippedOfSuffix);
407   }
408 
409   /**
410    * Validate the store file name.
411    * @param fileName name of the file to validate
412    * @return <tt>true</tt> if the file could be a valid store file, <tt>false</tt> otherwise
413    */
414   public static boolean validateStoreFileName(final String fileName) {
415     if (HFileLink.isHFileLink(fileName) || isReference(fileName))
416       return(true);
417     return !fileName.contains("-");
418   }
419 
420   /**
421    * Return if the specified file is a valid store file or not.
422    * @param fileStatus The {@link FileStatus} of the file
423    * @return <tt>true</tt> if the file is valid
424    */
425   public static boolean isValid(final FileStatus fileStatus)
426       throws IOException {
427     final Path p = fileStatus.getPath();
428 
429     if (fileStatus.isDirectory())
430       return false;
431 
432     // Check for empty hfile. Should never be the case but can happen
433     // after data loss in hdfs for whatever reason (upgrade, etc.): HBASE-646
434     // NOTE: that the HFileLink is just a name, so it's an empty file.
435     if (!HFileLink.isHFileLink(p) && fileStatus.getLen() <= 0) {
436       LOG.warn("Skipping " + p + " because it is empty. HBASE-646 DATA LOSS?");
437       return false;
438     }
439 
440     return validateStoreFileName(p.getName());
441   }
442 
443   /**
444    * helper function to compute HDFS blocks distribution of a given reference
445    * file.For reference file, we don't compute the exact value. We use some
446    * estimate instead given it might be good enough. we assume bottom part
447    * takes the first half of reference file, top part takes the second half
448    * of the reference file. This is just estimate, given
449    * midkey ofregion != midkey of HFile, also the number and size of keys vary.
450    * If this estimate isn't good enough, we can improve it later.
451    * @param fs  The FileSystem
452    * @param reference  The reference
453    * @param status  The reference FileStatus
454    * @return HDFS blocks distribution
455    */
456   private static HDFSBlocksDistribution computeRefFileHDFSBlockDistribution(
457       final FileSystem fs, final Reference reference, final FileStatus status)
458       throws IOException {
459     if (status == null) {
460       return null;
461     }
462 
463     long start = 0;
464     long length = 0;
465 
466     if (Reference.isTopFileRegion(reference.getFileRegion())) {
467       start = status.getLen()/2;
468       length = status.getLen() - status.getLen()/2;
469     } else {
470       start = 0;
471       length = status.getLen()/2;
472     }
473     return FSUtils.computeHDFSBlocksDistribution(fs, status, start, length);
474   }
475 
476   @Override
477   public boolean equals(Object that) {
478     if (that == null) {
479       return false;
480     }
481 
482     if (that instanceof StoreFileInfo) {
483       return this.compareTo((StoreFileInfo)that) == 0;
484     }
485 
486     return false;
487   };
488 
489   @Override
490   public int compareTo(StoreFileInfo o) {
491     return this.fileStatus.compareTo(o.fileStatus);
492   }
493 
494   @Override
495   public int hashCode() {
496     return this.fileStatus.hashCode();
497   }
498 }