001/*
002 *
003 * Licensed to the Apache Software Foundation (ASF) under one
004 * or more contributor license agreements.  See the NOTICE file
005 * distributed with this work for additional information
006 * regarding copyright ownership.  The ASF licenses this file
007 * to you under the Apache License, Version 2.0 (the
008 * "License"); you may not use this file except in compliance
009 * with the License.  You may obtain a copy of the License at
010 *
011 *     http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 */
019package org.apache.hadoop.hbase.mob;
020
021import java.io.FileNotFoundException;
022import java.io.IOException;
023import java.text.ParseException;
024import java.text.SimpleDateFormat;
025import java.util.ArrayList;
026import java.util.Calendar;
027import java.util.Collection;
028import java.util.Collections;
029import java.util.Date;
030import java.util.HashSet;
031import java.util.List;
032import java.util.Optional;
033import java.util.Set;
034import java.util.UUID;
035
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.fs.FileStatus;
038import org.apache.hadoop.fs.FileSystem;
039import org.apache.hadoop.fs.Path;
040import org.apache.hadoop.hbase.Cell;
041import org.apache.hadoop.hbase.HConstants;
042import org.apache.hadoop.hbase.PrivateCellUtil;
043import org.apache.hadoop.hbase.TableName;
044import org.apache.hadoop.hbase.Tag;
045import org.apache.hadoop.hbase.TagType;
046import org.apache.hadoop.hbase.TagUtil;
047import org.apache.hadoop.hbase.backup.HFileArchiver;
048import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
049import org.apache.hadoop.hbase.client.RegionInfo;
050import org.apache.hadoop.hbase.client.RegionInfoBuilder;
051import org.apache.hadoop.hbase.client.Scan;
052import org.apache.hadoop.hbase.client.TableDescriptor;
053import org.apache.hadoop.hbase.io.HFileLink;
054import org.apache.hadoop.hbase.io.compress.Compression;
055import org.apache.hadoop.hbase.io.crypto.Encryption;
056import org.apache.hadoop.hbase.io.hfile.CacheConfig;
057import org.apache.hadoop.hbase.io.hfile.HFile;
058import org.apache.hadoop.hbase.io.hfile.HFileContext;
059import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
060import org.apache.hadoop.hbase.regionserver.BloomType;
061import org.apache.hadoop.hbase.regionserver.HStore;
062import org.apache.hadoop.hbase.regionserver.HStoreFile;
063import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
064import org.apache.hadoop.hbase.util.Bytes;
065import org.apache.hadoop.hbase.util.ChecksumType;
066import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
067import org.apache.hadoop.hbase.util.FSUtils;
068import org.apache.yetus.audience.InterfaceAudience;
069import org.slf4j.Logger;
070import org.slf4j.LoggerFactory;
071
072/**
073 * The mob utilities
074 */
075@InterfaceAudience.Private
076public final class MobUtils {
077
078  private static final Logger LOG = LoggerFactory.getLogger(MobUtils.class);
079  public static final String SEP = "_";
080
081  private static final ThreadLocal<SimpleDateFormat> LOCAL_FORMAT =
082    new ThreadLocal<SimpleDateFormat>() {
083      @Override
084      protected SimpleDateFormat initialValue() {
085        return new SimpleDateFormat("yyyyMMdd");
086      }
087    };
088
089  /**
090   * Private constructor to keep this class from being instantiated.
091   */
092  private MobUtils() {
093  }
094
095  /**
096   * Formats a date to a string.
097   * @param date The date.
098   * @return The string format of the date, it's yyyymmdd.
099   */
100  public static String formatDate(Date date) {
101    return LOCAL_FORMAT.get().format(date);
102  }
103
104  /**
105   * Parses the string to a date.
106   * @param dateString The string format of a date, it's yyyymmdd.
107   * @return A date.
108   */
109  public static Date parseDate(String dateString) throws ParseException {
110    return LOCAL_FORMAT.get().parse(dateString);
111  }
112
113  /**
114   * Whether the current cell is a mob reference cell.
115   * @param cell The current cell.
116   * @return True if the cell has a mob reference tag, false if it doesn't.
117   */
118  public static boolean isMobReferenceCell(Cell cell) {
119    if (cell.getTagsLength() > 0) {
120      Optional<Tag> tag = PrivateCellUtil.getTag(cell, TagType.MOB_REFERENCE_TAG_TYPE);
121      if (tag.isPresent()) {
122        return true;
123      }
124    }
125    return false;
126  }
127
128  /**
129   * Gets the table name tag.
130   * @param cell The current cell.
131   * @return The table name tag.
132   */
133  public static Tag getTableNameTag(Cell cell) {
134    if (cell.getTagsLength() > 0) {
135      Optional<Tag> tag = PrivateCellUtil.getTag(cell, TagType.MOB_TABLE_NAME_TAG_TYPE);
136      if (tag.isPresent()) {
137        return tag.get();
138      }
139    }
140    return null;
141  }
142
143  /**
144   * Whether the tag list has a mob reference tag.
145   * @param tags The tag list.
146   * @return True if the list has a mob reference tag, false if it doesn't.
147   */
148  public static boolean hasMobReferenceTag(List<Tag> tags) {
149    if (!tags.isEmpty()) {
150      for (Tag tag : tags) {
151        if (tag.getType() == TagType.MOB_REFERENCE_TAG_TYPE) {
152          return true;
153        }
154      }
155    }
156    return false;
157  }
158
159  /**
160   * Indicates whether it's a raw scan. The information is set in the attribute "hbase.mob.scan.raw"
161   * of scan. For a mob cell, in a normal scan the scanners retrieves the mob cell from the mob
162   * file. In a raw scan, the scanner directly returns cell in HBase without retrieve the one in the
163   * mob file.
164   * @param scan The current scan.
165   * @return True if it's a raw scan.
166   */
167  public static boolean isRawMobScan(Scan scan) {
168    byte[] raw = scan.getAttribute(MobConstants.MOB_SCAN_RAW);
169    try {
170      return raw != null && Bytes.toBoolean(raw);
171    } catch (IllegalArgumentException e) {
172      return false;
173    }
174  }
175
176  /**
177   * Indicates whether it's a reference only scan. The information is set in the attribute
178   * "hbase.mob.scan.ref.only" of scan. If it's a ref only scan, only the cells with ref tag are
179   * returned.
180   * @param scan The current scan.
181   * @return True if it's a ref only scan.
182   */
183  public static boolean isRefOnlyScan(Scan scan) {
184    byte[] refOnly = scan.getAttribute(MobConstants.MOB_SCAN_REF_ONLY);
185    try {
186      return refOnly != null && Bytes.toBoolean(refOnly);
187    } catch (IllegalArgumentException e) {
188      return false;
189    }
190  }
191
192  /**
193   * Indicates whether the scan contains the information of caching blocks. The information is set
194   * in the attribute "hbase.mob.cache.blocks" of scan.
195   * @param scan The current scan.
196   * @return True when the Scan attribute specifies to cache the MOB blocks.
197   */
198  public static boolean isCacheMobBlocks(Scan scan) {
199    byte[] cache = scan.getAttribute(MobConstants.MOB_CACHE_BLOCKS);
200    try {
201      return cache != null && Bytes.toBoolean(cache);
202    } catch (IllegalArgumentException e) {
203      return false;
204    }
205  }
206
207  /**
208   * Sets the attribute of caching blocks in the scan.
209   * @param scan The current scan.
210   * @param cacheBlocks True, set the attribute of caching blocks into the scan, the scanner with
211   *          this scan caches blocks. False, the scanner doesn't cache blocks for this scan.
212   */
213  public static void setCacheMobBlocks(Scan scan, boolean cacheBlocks) {
214    scan.setAttribute(MobConstants.MOB_CACHE_BLOCKS, Bytes.toBytes(cacheBlocks));
215  }
216
217  /**
218   * Cleans the expired mob files. Cleans the files whose creation date is older than (current -
219   * columnFamily.ttl), and the minVersions of that column family is 0.
220   * @param fs The current file system.
221   * @param conf The current configuration.
222   * @param tableName The current table name.
223   * @param columnDescriptor The descriptor of the current column family.
224   * @param cacheConfig The cacheConfig that disables the block cache.
225   * @param current The current time.
226   */
227  public static void cleanExpiredMobFiles(FileSystem fs, Configuration conf, TableName tableName,
228      ColumnFamilyDescriptor columnDescriptor, CacheConfig cacheConfig, long current)
229      throws IOException {
230    long timeToLive = columnDescriptor.getTimeToLive();
231    if (Integer.MAX_VALUE == timeToLive) {
232      // no need to clean, because the TTL is not set.
233      return;
234    }
235
236    Calendar calendar = Calendar.getInstance();
237    calendar.setTimeInMillis(current - timeToLive * 1000);
238    calendar.set(Calendar.HOUR_OF_DAY, 0);
239    calendar.set(Calendar.MINUTE, 0);
240    calendar.set(Calendar.SECOND, 0);
241
242    Date expireDate = calendar.getTime();
243
244    LOG.info("MOB HFiles older than " + expireDate.toGMTString() + " will be deleted!");
245
246    FileStatus[] stats = null;
247    Path mobTableDir = FSUtils.getTableDir(getMobHome(conf), tableName);
248    Path path = getMobFamilyPath(conf, tableName, columnDescriptor.getNameAsString());
249    try {
250      stats = fs.listStatus(path);
251    } catch (FileNotFoundException e) {
252      LOG.warn("Failed to find the mob file " + path, e);
253    }
254    if (null == stats) {
255      // no file found
256      return;
257    }
258    List<HStoreFile> filesToClean = new ArrayList<>();
259    int deletedFileCount = 0;
260    for (FileStatus file : stats) {
261      String fileName = file.getPath().getName();
262      try {
263        if (HFileLink.isHFileLink(file.getPath())) {
264          HFileLink hfileLink = HFileLink.buildFromHFileLinkPattern(conf, file.getPath());
265          fileName = hfileLink.getOriginPath().getName();
266        }
267
268        Date fileDate = parseDate(MobFileName.getDateFromName(fileName));
269
270        if (LOG.isDebugEnabled()) {
271          LOG.debug("Checking file {}", fileName);
272        }
273        if (fileDate.getTime() < expireDate.getTime()) {
274          if (LOG.isDebugEnabled()) {
275            LOG.debug("{} is an expired file", fileName);
276          }
277          filesToClean
278              .add(new HStoreFile(fs, file.getPath(), conf, cacheConfig, BloomType.NONE, true));
279        }
280      } catch (Exception e) {
281        LOG.error("Cannot parse the fileName " + fileName, e);
282      }
283    }
284    if (!filesToClean.isEmpty()) {
285      try {
286        removeMobFiles(conf, fs, tableName, mobTableDir, columnDescriptor.getName(), filesToClean);
287        deletedFileCount = filesToClean.size();
288      } catch (IOException e) {
289        LOG.error("Failed to delete the mob files " + filesToClean, e);
290      }
291    }
292    LOG.info("{} expired mob files are deleted", deletedFileCount);
293  }
294
295  /**
296   * Gets the root dir of the mob files. It's {HBASE_DIR}/mobdir.
297   * @param conf The current configuration.
298   * @return the root dir of the mob file.
299   */
300  public static Path getMobHome(Configuration conf) {
301    Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR));
302    return getMobHome(hbaseDir);
303  }
304
305  /**
306   * Gets the root dir of the mob files under the qualified HBase root dir. It's {rootDir}/mobdir.
307   * @param rootDir The qualified path of HBase root directory.
308   * @return The root dir of the mob file.
309   */
310  public static Path getMobHome(Path rootDir) {
311    return new Path(rootDir, MobConstants.MOB_DIR_NAME);
312  }
313
314  /**
315   * Gets the qualified root dir of the mob files.
316   * @param conf The current configuration.
317   * @return The qualified root dir.
318   */
319  public static Path getQualifiedMobRootDir(Configuration conf) throws IOException {
320    Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR));
321    Path mobRootDir = new Path(hbaseDir, MobConstants.MOB_DIR_NAME);
322    FileSystem fs = mobRootDir.getFileSystem(conf);
323    return mobRootDir.makeQualified(fs.getUri(), fs.getWorkingDirectory());
324  }
325
326  /**
327   * Gets the table dir of the mob files under the qualified HBase root dir. It's
328   * {rootDir}/mobdir/data/${namespace}/${tableName}
329   * @param rootDir The qualified path of HBase root directory.
330   * @param tableName The name of table.
331   * @return The table dir of the mob file.
332   */
333  public static Path getMobTableDir(Path rootDir, TableName tableName) {
334    return FSUtils.getTableDir(getMobHome(rootDir), tableName);
335  }
336
337  /**
338   * Gets the region dir of the mob files. It's
339   * {HBASE_DIR}/mobdir/data/{namespace}/{tableName}/{regionEncodedName}.
340   * @param conf The current configuration.
341   * @param tableName The current table name.
342   * @return The region dir of the mob files.
343   */
344  public static Path getMobRegionPath(Configuration conf, TableName tableName) {
345    return getMobRegionPath(new Path(conf.get(HConstants.HBASE_DIR)), tableName);
346  }
347
348  /**
349   * Gets the region dir of the mob files under the specified root dir. It's
350   * {rootDir}/mobdir/data/{namespace}/{tableName}/{regionEncodedName}.
351   * @param rootDir The qualified path of HBase root directory.
352   * @param tableName The current table name.
353   * @return The region dir of the mob files.
354   */
355  public static Path getMobRegionPath(Path rootDir, TableName tableName) {
356    Path tablePath = FSUtils.getTableDir(getMobHome(rootDir), tableName);
357    RegionInfo regionInfo = getMobRegionInfo(tableName);
358    return new Path(tablePath, regionInfo.getEncodedName());
359  }
360
361  /**
362   * Gets the family dir of the mob files. It's
363   * {HBASE_DIR}/mobdir/{namespace}/{tableName}/{regionEncodedName}/{columnFamilyName}.
364   * @param conf The current configuration.
365   * @param tableName The current table name.
366   * @param familyName The current family name.
367   * @return The family dir of the mob files.
368   */
369  public static Path getMobFamilyPath(Configuration conf, TableName tableName, String familyName) {
370    return new Path(getMobRegionPath(conf, tableName), familyName);
371  }
372
373  /**
374   * Gets the family dir of the mob files. It's
375   * {HBASE_DIR}/mobdir/{namespace}/{tableName}/{regionEncodedName}/{columnFamilyName}.
376   * @param regionPath The path of mob region which is a dummy one.
377   * @param familyName The current family name.
378   * @return The family dir of the mob files.
379   */
380  public static Path getMobFamilyPath(Path regionPath, String familyName) {
381    return new Path(regionPath, familyName);
382  }
383
384  /**
385   * Gets the RegionInfo of the mob files. This is a dummy region. The mob files are not saved in a
386   * region in HBase. This is only used in mob snapshot. It's internally used only.
387   * @param tableName
388   * @return A dummy mob region info.
389   */
390  public static RegionInfo getMobRegionInfo(TableName tableName) {
391    return RegionInfoBuilder.newBuilder(tableName).setStartKey(MobConstants.MOB_REGION_NAME_BYTES)
392        .setEndKey(HConstants.EMPTY_END_ROW).setSplit(false).setRegionId(0).build();
393  }
394
395  /**
396   * Gets whether the current RegionInfo is a mob one.
397   * @param regionInfo The current RegionInfo.
398   * @return If true, the current RegionInfo is a mob one.
399   */
400  public static boolean isMobRegionInfo(RegionInfo regionInfo) {
401    return regionInfo == null ? false
402        : getMobRegionInfo(regionInfo.getTable()).getEncodedName()
403            .equals(regionInfo.getEncodedName());
404  }
405
406  /**
407   * Gets whether the current region name follows the pattern of a mob region name.
408   * @param tableName The current table name.
409   * @param regionName The current region name.
410   * @return True if the current region name follows the pattern of a mob region name.
411   */
412  public static boolean isMobRegionName(TableName tableName, byte[] regionName) {
413    return Bytes.equals(regionName, getMobRegionInfo(tableName).getRegionName());
414  }
415
416  /**
417   * Archives the mob files.
418   * @param conf The current configuration.
419   * @param fs The current file system.
420   * @param tableName The table name.
421   * @param tableDir The table directory.
422   * @param family The name of the column family.
423   * @param storeFiles The files to be deleted.
424   */
425  public static void removeMobFiles(Configuration conf, FileSystem fs, TableName tableName,
426      Path tableDir, byte[] family, Collection<HStoreFile> storeFiles) throws IOException {
427    HFileArchiver.archiveStoreFiles(conf, fs, getMobRegionInfo(tableName), tableDir, family,
428      storeFiles);
429  }
430
431  /**
432   * Creates a mob reference KeyValue. The value of the mob reference KeyValue is mobCellValueSize +
433   * mobFileName.
434   * @param cell The original Cell.
435   * @param fileName The mob file name where the mob reference KeyValue is written.
436   * @param tableNameTag The tag of the current table name. It's very important in cloning the
437   *          snapshot.
438   * @return The mob reference KeyValue.
439   */
440  public static Cell createMobRefCell(Cell cell, byte[] fileName, Tag tableNameTag) {
441    // Append the tags to the KeyValue.
442    // The key is same, the value is the filename of the mob file
443    List<Tag> tags = new ArrayList<>();
444    // Add the ref tag as the 1st one.
445    tags.add(MobConstants.MOB_REF_TAG);
446    // Add the tag of the source table name, this table is where this mob file is flushed
447    // from.
448    // It's very useful in cloning the snapshot. When reading from the cloning table, we need to
449    // find the original mob files by this table name. For details please see cloning
450    // snapshot for mob files.
451    tags.add(tableNameTag);
452    return createMobRefCell(cell, fileName, TagUtil.fromList(tags));
453  }
454
455  public static Cell createMobRefCell(Cell cell, byte[] fileName, byte[] refCellTags) {
456    byte[] refValue = Bytes.add(Bytes.toBytes(cell.getValueLength()), fileName);
457    return PrivateCellUtil.createCell(cell, refValue, TagUtil.concatTags(refCellTags, cell));
458  }
459
460  /**
461   * Creates a writer for the mob file in temp directory.
462   * @param conf The current configuration.
463   * @param fs The current file system.
464   * @param family The descriptor of the current column family.
465   * @param date The date string, its format is yyyymmmdd.
466   * @param basePath The basic path for a temp directory.
467   * @param maxKeyCount The key count.
468   * @param compression The compression algorithm.
469   * @param startKey The hex string of the start key.
470   * @param cacheConfig The current cache config.
471   * @param cryptoContext The encryption context.
472   * @param isCompaction If the writer is used in compaction.
473   * @return The writer for the mob file.
474   */
475  public static StoreFileWriter createWriter(Configuration conf, FileSystem fs,
476      ColumnFamilyDescriptor family, String date, Path basePath, long maxKeyCount,
477      Compression.Algorithm compression, String startKey, CacheConfig cacheConfig,
478      Encryption.Context cryptoContext, boolean isCompaction, String regionName)
479      throws IOException {
480    MobFileName mobFileName = MobFileName.create(startKey, date,
481      UUID.randomUUID().toString().replaceAll("-", ""), regionName);
482    return createWriter(conf, fs, family, mobFileName, basePath, maxKeyCount, compression,
483      cacheConfig, cryptoContext, isCompaction);
484  }
485
486  /**
487   * Creates a writer for the mob file in temp directory.
488   * @param conf The current configuration.
489   * @param fs The current file system.
490   * @param family The descriptor of the current column family.
491   * @param mobFileName The mob file name.
492   * @param basePath The basic path for a temp directory.
493   * @param maxKeyCount The key count.
494   * @param compression The compression algorithm.
495   * @param cacheConfig The current cache config.
496   * @param cryptoContext The encryption context.
497   * @param isCompaction If the writer is used in compaction.
498   * @return The writer for the mob file.
499   */
500  public static StoreFileWriter createWriter(Configuration conf, FileSystem fs,
501      ColumnFamilyDescriptor family, MobFileName mobFileName, Path basePath, long maxKeyCount,
502      Compression.Algorithm compression, CacheConfig cacheConfig, Encryption.Context cryptoContext,
503      boolean isCompaction) throws IOException {
504    return createWriter(conf, fs, family, new Path(basePath, mobFileName.getFileName()),
505      maxKeyCount, compression, cacheConfig, cryptoContext, HStore.getChecksumType(conf),
506      HStore.getBytesPerChecksum(conf), family.getBlocksize(), BloomType.NONE, isCompaction);
507  }
508
509  /**
510   * Creates a writer for the mob file in temp directory.
511   * @param conf The current configuration.
512   * @param fs The current file system.
513   * @param family The descriptor of the current column family.
514   * @param path The path for a temp directory.
515   * @param maxKeyCount The key count.
516   * @param compression The compression algorithm.
517   * @param cacheConfig The current cache config.
518   * @param cryptoContext The encryption context.
519   * @param checksumType The checksum type.
520   * @param bytesPerChecksum The bytes per checksum.
521   * @param blocksize The HFile block size.
522   * @param bloomType The bloom filter type.
523   * @param isCompaction If the writer is used in compaction.
524   * @return The writer for the mob file.
525   */
526  public static StoreFileWriter createWriter(Configuration conf, FileSystem fs,
527      ColumnFamilyDescriptor family, Path path, long maxKeyCount, Compression.Algorithm compression,
528      CacheConfig cacheConfig, Encryption.Context cryptoContext, ChecksumType checksumType,
529      int bytesPerChecksum, int blocksize, BloomType bloomType, boolean isCompaction)
530      throws IOException {
531    if (compression == null) {
532      compression = HFile.DEFAULT_COMPRESSION_ALGORITHM;
533    }
534    final CacheConfig writerCacheConf;
535    if (isCompaction) {
536      writerCacheConf = new CacheConfig(cacheConfig);
537      writerCacheConf.setCacheDataOnWrite(false);
538    } else {
539      writerCacheConf = cacheConfig;
540    }
541    HFileContext hFileContext = new HFileContextBuilder().withCompression(compression)
542        .withIncludesMvcc(true).withIncludesTags(true).withCompressTags(family.isCompressTags())
543        .withChecksumType(checksumType).withBytesPerCheckSum(bytesPerChecksum)
544        .withBlockSize(blocksize).withHBaseCheckSum(true)
545        .withDataBlockEncoding(family.getDataBlockEncoding()).withEncryptionContext(cryptoContext)
546        .withCreateTime(EnvironmentEdgeManager.currentTime()).build();
547
548    StoreFileWriter w = new StoreFileWriter.Builder(conf, writerCacheConf, fs)
549        .withFilePath(path).withBloomType(bloomType)
550        .withMaxKeyCount(maxKeyCount).withFileContext(hFileContext).build();
551    return w;
552  }
553
554  /**
555   * Indicates whether the current mob ref cell has a valid value. A mob ref cell has a mob
556   * reference tag. The value of a mob ref cell consists of two parts, real mob value length and mob
557   * file name. The real mob value length takes 4 bytes. The remaining part is the mob file name.
558   * @param cell The mob ref cell.
559   * @return True if the cell has a valid value.
560   */
561  public static boolean hasValidMobRefCellValue(Cell cell) {
562    return cell.getValueLength() > Bytes.SIZEOF_INT;
563  }
564
565  /**
566   * Gets the mob value length from the mob ref cell. A mob ref cell has a mob reference tag. The
567   * value of a mob ref cell consists of two parts, real mob value length and mob file name. The
568   * real mob value length takes 4 bytes. The remaining part is the mob file name.
569   * @param cell The mob ref cell.
570   * @return The real mob value length.
571   */
572  public static int getMobValueLength(Cell cell) {
573    return PrivateCellUtil.getValueAsInt(cell);
574  }
575
576  /**
577   * Gets the mob file name from the mob ref cell. A mob ref cell has a mob reference tag. The value
578   * of a mob ref cell consists of two parts, real mob value length and mob file name. The real mob
579   * value length takes 4 bytes. The remaining part is the mob file name.
580   * @param cell The mob ref cell.
581   * @return The mob file name.
582   */
583  public static String getMobFileName(Cell cell) {
584    return Bytes.toString(cell.getValueArray(), cell.getValueOffset() + Bytes.SIZEOF_INT,
585      cell.getValueLength() - Bytes.SIZEOF_INT);
586  }
587
588  /**
589   * Checks whether this table has mob-enabled columns.
590   * @param htd The current table descriptor.
591   * @return Whether this table has mob-enabled columns.
592   */
593  public static boolean hasMobColumns(TableDescriptor htd) {
594    ColumnFamilyDescriptor[] hcds = htd.getColumnFamilies();
595    for (ColumnFamilyDescriptor hcd : hcds) {
596      if (hcd.isMobEnabled()) {
597        return true;
598      }
599    }
600    return false;
601  }
602
603  /**
604   * Get list of Mob column families (if any exists)
605   * @param htd table descriptor
606   * @return list of Mob column families
607   */
608  public static List<ColumnFamilyDescriptor> getMobColumnFamilies(TableDescriptor htd) {
609
610    List<ColumnFamilyDescriptor> fams = new ArrayList<ColumnFamilyDescriptor>();
611    ColumnFamilyDescriptor[] hcds = htd.getColumnFamilies();
612    for (ColumnFamilyDescriptor hcd : hcds) {
613      if (hcd.isMobEnabled()) {
614        fams.add(hcd);
615      }
616    }
617    return fams;
618  }
619
620  /**
621   * Indicates whether return null value when the mob file is missing or corrupt. The information is
622   * set in the attribute "empty.value.on.mobcell.miss" of scan.
623   * @param scan The current scan.
624   * @return True if the readEmptyValueOnMobCellMiss is enabled.
625   */
626  public static boolean isReadEmptyValueOnMobCellMiss(Scan scan) {
627    byte[] readEmptyValueOnMobCellMiss =
628        scan.getAttribute(MobConstants.EMPTY_VALUE_ON_MOBCELL_MISS);
629    try {
630      return readEmptyValueOnMobCellMiss != null && Bytes.toBoolean(readEmptyValueOnMobCellMiss);
631    } catch (IllegalArgumentException e) {
632      return false;
633    }
634  }
635
636  /**
637   * Checks if the mob file is expired.
638   * @param column The descriptor of the current column family.
639   * @param current The current time.
640   * @param fileDate The date string parsed from the mob file name.
641   * @return True if the mob file is expired.
642   */
643  public static boolean isMobFileExpired(ColumnFamilyDescriptor column, long current,
644      String fileDate) {
645    if (column.getMinVersions() > 0) {
646      return false;
647    }
648    long timeToLive = column.getTimeToLive();
649    if (Integer.MAX_VALUE == timeToLive) {
650      return false;
651    }
652
653    Date expireDate = new Date(current - timeToLive * 1000);
654    expireDate = new Date(expireDate.getYear(), expireDate.getMonth(), expireDate.getDate());
655    try {
656      Date date = parseDate(fileDate);
657      if (date.getTime() < expireDate.getTime()) {
658        return true;
659      }
660    } catch (ParseException e) {
661      LOG.warn("Failed to parse the date " + fileDate, e);
662      return false;
663    }
664    return false;
665  }
666
667  /**
668   * Get list of referenced MOB files from a given collection of store files
669   * @param storeFiles store files
670   * @param mobDir MOB file directory
671   * @return list of MOB file paths
672   */
673
674  public static List<Path> getReferencedMobFiles(Collection<HStoreFile> storeFiles, Path mobDir) {
675
676    Set<String> mobSet = new HashSet<String>();
677    for (HStoreFile sf : storeFiles) {
678      byte[] value = sf.getMetadataValue(HStoreFile.MOB_FILE_REFS);
679      if (value != null && value.length > 1) {
680        String s = Bytes.toString(value);
681        String[] all = s.split(",");
682        Collections.addAll(mobSet, all);
683      }
684    }
685    List<Path> retList = new ArrayList<Path>();
686    for (String name : mobSet) {
687      retList.add(new Path(mobDir, name));
688    }
689    return retList;
690  }
691}