View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import java.io.IOException;
22  import java.io.InterruptedIOException;
23  import java.net.InetSocketAddress;
24  import java.security.Key;
25  import java.security.KeyException;
26  import java.util.ArrayList;
27  import java.util.Collection;
28  import java.util.Collections;
29  import java.util.HashMap;
30  import java.util.HashSet;
31  import java.util.Iterator;
32  import java.util.List;
33  import java.util.NavigableSet;
34  import java.util.Set;
35  import java.util.concurrent.Callable;
36  import java.util.concurrent.CompletionService;
37  import java.util.concurrent.ConcurrentHashMap;
38  import java.util.concurrent.ExecutionException;
39  import java.util.concurrent.ExecutorCompletionService;
40  import java.util.concurrent.Future;
41  import java.util.concurrent.ThreadPoolExecutor;
42  import java.util.concurrent.atomic.AtomicBoolean;
43  import java.util.concurrent.locks.ReentrantReadWriteLock;
44  
45  import org.apache.commons.logging.Log;
46  import org.apache.commons.logging.LogFactory;
47  import org.apache.hadoop.conf.Configuration;
48  import org.apache.hadoop.fs.FileSystem;
49  import org.apache.hadoop.fs.Path;
50  import org.apache.hadoop.hbase.Cell;
51  import org.apache.hadoop.hbase.CellComparator;
52  import org.apache.hadoop.hbase.CellUtil;
53  import org.apache.hadoop.hbase.CompoundConfiguration;
54  import org.apache.hadoop.hbase.HColumnDescriptor;
55  import org.apache.hadoop.hbase.HConstants;
56  import org.apache.hadoop.hbase.HRegionInfo;
57  import org.apache.hadoop.hbase.KeyValue;
58  import org.apache.hadoop.hbase.TableName;
59  import org.apache.hadoop.hbase.Tag;
60  import org.apache.hadoop.hbase.TagType;
61  import org.apache.hadoop.hbase.classification.InterfaceAudience;
62  import org.apache.hadoop.hbase.client.Scan;
63  import org.apache.hadoop.hbase.conf.ConfigurationManager;
64  import org.apache.hadoop.hbase.io.compress.Compression;
65  import org.apache.hadoop.hbase.io.crypto.Cipher;
66  import org.apache.hadoop.hbase.io.crypto.Encryption;
67  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
68  import org.apache.hadoop.hbase.io.hfile.HFile;
69  import org.apache.hadoop.hbase.io.hfile.HFileContext;
70  import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
71  import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
72  import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
73  import org.apache.hadoop.hbase.io.hfile.HFileScanner;
74  import org.apache.hadoop.hbase.io.hfile.InvalidHFileException;
75  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
76  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
77  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
78  import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
79  import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
80  import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
81  import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
82  import org.apache.hadoop.hbase.regionserver.compactions.OffPeakHours;
83  import org.apache.hadoop.hbase.regionserver.compactions.CompactionThroughputController;
84  import org.apache.hadoop.hbase.regionserver.wal.WALUtil;
85  import org.apache.hadoop.hbase.security.EncryptionUtil;
86  import org.apache.hadoop.hbase.security.User;
87  import org.apache.hadoop.hbase.util.Bytes;
88  import org.apache.hadoop.hbase.util.ChecksumType;
89  import org.apache.hadoop.hbase.util.ClassSize;
90  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
91  import org.apache.hadoop.hbase.util.Pair;
92  import org.apache.hadoop.hbase.util.ReflectionUtils;
93  import org.apache.hadoop.ipc.RemoteException;
94  import org.apache.hadoop.util.StringUtils;
95  import org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix;
96  
97  import com.google.common.annotations.VisibleForTesting;
98  import com.google.common.base.Preconditions;
99  import com.google.common.collect.ImmutableCollection;
100 import com.google.common.collect.ImmutableList;
101 import com.google.common.collect.Lists;
102 import com.google.common.collect.Sets;
103 
104 /**
105  * A Store holds a column family in a Region.  Its a memstore and a set of zero
106  * or more StoreFiles, which stretch backwards over time.
107  *
108  * <p>There's no reason to consider append-logging at this level; all logging
109  * and locking is handled at the HRegion level.  Store just provides
110  * services to manage sets of StoreFiles.  One of the most important of those
111  * services is compaction services where files are aggregated once they pass
112  * a configurable threshold.
113  *
114  * <p>The only thing having to do with logs that Store needs to deal with is
115  * the reconstructionLog.  This is a segment of an HRegion's log that might
116  * NOT be present upon startup.  If the param is NULL, there's nothing to do.
117  * If the param is non-NULL, we need to process the log to reconstruct
118  * a TreeMap that might not have been written to disk before the process
119  * died.
120  *
121  * <p>It's assumed that after this constructor returns, the reconstructionLog
122  * file will be deleted (by whoever has instantiated the Store).
123  *
124  * <p>Locking and transactions are handled at a higher level.  This API should
125  * not be called directly but by an HRegion manager.
126  */
127 @InterfaceAudience.Private
128 public class HStore implements Store {
129   private static final String MEMSTORE_CLASS_NAME = "hbase.regionserver.memstore.class";
130   public static final String COMPACTCHECKER_INTERVAL_MULTIPLIER_KEY =
131       "hbase.server.compactchecker.interval.multiplier";
132   public static final String BLOCKING_STOREFILES_KEY = "hbase.hstore.blockingStoreFiles";
133   public static final int DEFAULT_COMPACTCHECKER_INTERVAL_MULTIPLIER = 1000;
134   public static final int DEFAULT_BLOCKING_STOREFILE_COUNT = 7;
135 
136   static final Log LOG = LogFactory.getLog(HStore.class);
137 
138   protected final MemStore memstore;
139   // This stores directory in the filesystem.
140   private final HRegion region;
141   private final HColumnDescriptor family;
142   private final HRegionFileSystem fs;
143   private Configuration conf;
144   private final CacheConfig cacheConf;
145   private long lastCompactSize = 0;
146   volatile boolean forceMajor = false;
147   /* how many bytes to write between status checks */
148   static int closeCheckInterval = 0;
149   private volatile long storeSize = 0L;
150   private volatile long totalUncompressedBytes = 0L;
151 
152   /**
153    * RWLock for store operations.
154    * Locked in shared mode when the list of component stores is looked at:
155    *   - all reads/writes to table data
156    *   - checking for split
157    * Locked in exclusive mode when the list of component stores is modified:
158    *   - closing
159    *   - completing a compaction
160    */
161   final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
162   private final boolean verifyBulkLoads;
163 
164   private ScanInfo scanInfo;
165 
166   // TODO: ideally, this should be part of storeFileManager, as we keep passing this to it.
167   final List<StoreFile> filesCompacting = Lists.newArrayList();
168 
169   // All access must be synchronized.
170   private final Set<ChangedReadersObserver> changedReaderObservers =
171     Collections.newSetFromMap(new ConcurrentHashMap<ChangedReadersObserver, Boolean>());
172 
173   private final int blocksize;
174   private HFileDataBlockEncoder dataBlockEncoder;
175 
176   /** Checksum configuration */
177   private ChecksumType checksumType;
178   private int bytesPerChecksum;
179 
180   // Comparing KeyValues
181   private final KeyValue.KVComparator comparator;
182 
183   final StoreEngine<?, ?, ?, ?> storeEngine;
184 
185   private static final AtomicBoolean offPeakCompactionTracker = new AtomicBoolean();
186   private volatile OffPeakHours offPeakHours;
187 
188   private static final int DEFAULT_FLUSH_RETRIES_NUMBER = 10;
189   private int flushRetriesNumber;
190   private int pauseTime;
191 
192   private long blockingFileCount;
193   private int compactionCheckMultiplier;
194 
195   private Encryption.Context cryptoContext = Encryption.Context.NONE;
196 
197   private volatile long flushedCellsCount = 0;
198   private volatile long compactedCellsCount = 0;
199   private volatile long majorCompactedCellsCount = 0;
200   private volatile long flushedCellsSize = 0;
201   private volatile long compactedCellsSize = 0;
202   private volatile long majorCompactedCellsSize = 0;
203 
204   /**
205    * Constructor
206    * @param region
207    * @param family HColumnDescriptor for this column
208    * @param confParam configuration object
209    * failed.  Can be null.
210    * @throws IOException
211    */
212   protected HStore(final HRegion region, final HColumnDescriptor family,
213       final Configuration confParam) throws IOException {
214 
215     HRegionInfo info = region.getRegionInfo();
216     this.fs = region.getRegionFileSystem();
217 
218     // Assemble the store's home directory and Ensure it exists.
219     fs.createStoreDir(family.getNameAsString());
220     this.region = region;
221     this.family = family;
222     // 'conf' renamed to 'confParam' b/c we use this.conf in the constructor
223     // CompoundConfiguration will look for keys in reverse order of addition, so we'd
224     // add global config first, then table and cf overrides, then cf metadata.
225     this.conf = new CompoundConfiguration()
226       .add(confParam)
227       .addStringMap(region.getTableDesc().getConfiguration())
228       .addStringMap(family.getConfiguration())
229       .addBytesMap(family.getValues());
230     this.blocksize = family.getBlocksize();
231 
232     this.dataBlockEncoder =
233         new HFileDataBlockEncoderImpl(family.getDataBlockEncoding());
234 
235     this.comparator = region.getCellCompartor();
236     // used by ScanQueryMatcher
237     long timeToPurgeDeletes =
238         Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0);
239     LOG.trace("Time to purge deletes set to " + timeToPurgeDeletes +
240         "ms in store " + this);
241     // Get TTL
242     long ttl = determineTTLFromFamily(family);
243     // Why not just pass a HColumnDescriptor in here altogether?  Even if have
244     // to clone it?
245     scanInfo = new ScanInfo(family, ttl, timeToPurgeDeletes, this.comparator);
246     String className = conf.get(MEMSTORE_CLASS_NAME, DefaultMemStore.class.getName());
247     this.memstore = ReflectionUtils.instantiateWithCustomCtor(className, new Class[] {
248         Configuration.class, KeyValue.KVComparator.class }, new Object[] { conf, this.comparator });
249     this.offPeakHours = OffPeakHours.getInstance(conf);
250 
251     // Setting up cache configuration for this family
252     this.cacheConf = new CacheConfig(conf, family);
253 
254     this.verifyBulkLoads = conf.getBoolean("hbase.hstore.bulkload.verify", false);
255 
256     this.blockingFileCount =
257         conf.getInt(BLOCKING_STOREFILES_KEY, DEFAULT_BLOCKING_STOREFILE_COUNT);
258     this.compactionCheckMultiplier = conf.getInt(
259         COMPACTCHECKER_INTERVAL_MULTIPLIER_KEY, DEFAULT_COMPACTCHECKER_INTERVAL_MULTIPLIER);
260     if (this.compactionCheckMultiplier <= 0) {
261       LOG.error("Compaction check period multiplier must be positive, setting default: "
262           + DEFAULT_COMPACTCHECKER_INTERVAL_MULTIPLIER);
263       this.compactionCheckMultiplier = DEFAULT_COMPACTCHECKER_INTERVAL_MULTIPLIER;
264     }
265 
266     if (HStore.closeCheckInterval == 0) {
267       HStore.closeCheckInterval = conf.getInt(
268           "hbase.hstore.close.check.interval", 10*1000*1000 /* 10 MB */);
269     }
270 
271     this.storeEngine = StoreEngine.create(this, this.conf, this.comparator);
272     this.storeEngine.getStoreFileManager().loadFiles(loadStoreFiles());
273 
274     // Initialize checksum type from name. The names are CRC32, CRC32C, etc.
275     this.checksumType = getChecksumType(conf);
276     // initilize bytes per checksum
277     this.bytesPerChecksum = getBytesPerChecksum(conf);
278     flushRetriesNumber = conf.getInt(
279         "hbase.hstore.flush.retries.number", DEFAULT_FLUSH_RETRIES_NUMBER);
280     pauseTime = conf.getInt(HConstants.HBASE_SERVER_PAUSE, HConstants.DEFAULT_HBASE_SERVER_PAUSE);
281     if (flushRetriesNumber <= 0) {
282       throw new IllegalArgumentException(
283           "hbase.hstore.flush.retries.number must be > 0, not "
284               + flushRetriesNumber);
285     }
286 
287     // Crypto context for new store files
288     String cipherName = family.getEncryptionType();
289     if (cipherName != null) {
290       Cipher cipher;
291       Key key;
292       byte[] keyBytes = family.getEncryptionKey();
293       if (keyBytes != null) {
294         // Family provides specific key material
295         String masterKeyName = conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY,
296           User.getCurrent().getShortName());
297         try {
298           // First try the master key
299           key = EncryptionUtil.unwrapKey(conf, masterKeyName, keyBytes);
300         } catch (KeyException e) {
301           // If the current master key fails to unwrap, try the alternate, if
302           // one is configured
303           if (LOG.isDebugEnabled()) {
304             LOG.debug("Unable to unwrap key with current master key '" + masterKeyName + "'");
305           }
306           String alternateKeyName =
307             conf.get(HConstants.CRYPTO_MASTERKEY_ALTERNATE_NAME_CONF_KEY);
308           if (alternateKeyName != null) {
309             try {
310               key = EncryptionUtil.unwrapKey(conf, alternateKeyName, keyBytes);
311             } catch (KeyException ex) {
312               throw new IOException(ex);
313             }
314           } else {
315             throw new IOException(e);
316           }
317         }
318         // Use the algorithm the key wants
319         cipher = Encryption.getCipher(conf, key.getAlgorithm());
320         if (cipher == null) {
321           throw new RuntimeException("Cipher '" + key.getAlgorithm() + "' is not available");
322         }
323         // Fail if misconfigured
324         // We use the encryption type specified in the column schema as a sanity check on
325         // what the wrapped key is telling us
326         if (!cipher.getName().equalsIgnoreCase(cipherName)) {
327           throw new RuntimeException("Encryption for family '" + family.getNameAsString() +
328             "' configured with type '" + cipherName +
329             "' but key specifies algorithm '" + cipher.getName() + "'");
330         }
331       } else {
332         // Family does not provide key material, create a random key
333         cipher = Encryption.getCipher(conf, cipherName);
334         if (cipher == null) {
335           throw new RuntimeException("Cipher '" + cipherName + "' is not available");
336         }
337         key = cipher.getRandomKey();
338       }
339       cryptoContext = Encryption.newContext(conf);
340       cryptoContext.setCipher(cipher);
341       cryptoContext.setKey(key);
342     }
343   }
344 
345   /**
346    * @param family
347    * @return TTL in seconds of the specified family
348    */
349   private static long determineTTLFromFamily(final HColumnDescriptor family) {
350     // HCD.getTimeToLive returns ttl in seconds.  Convert to milliseconds.
351     long ttl = family.getTimeToLive();
352     if (ttl == HConstants.FOREVER) {
353       // Default is unlimited ttl.
354       ttl = Long.MAX_VALUE;
355     } else if (ttl == -1) {
356       ttl = Long.MAX_VALUE;
357     } else {
358       // Second -> ms adjust for user data
359       ttl *= 1000;
360     }
361     return ttl;
362   }
363 
364   @Override
365   public String getColumnFamilyName() {
366     return this.family.getNameAsString();
367   }
368 
369   @Override
370   public TableName getTableName() {
371     return this.getRegionInfo().getTable();
372   }
373 
374   @Override
375   public FileSystem getFileSystem() {
376     return this.fs.getFileSystem();
377   }
378 
379   public HRegionFileSystem getRegionFileSystem() {
380     return this.fs;
381   }
382 
383   /* Implementation of StoreConfigInformation */
384   @Override
385   public long getStoreFileTtl() {
386     // TTL only applies if there's no MIN_VERSIONs setting on the column.
387     return (this.scanInfo.getMinVersions() == 0) ? this.scanInfo.getTtl() : Long.MAX_VALUE;
388   }
389 
390   @Override
391   public long getMemstoreFlushSize() {
392     // TODO: Why is this in here?  The flushsize of the region rather than the store?  St.Ack
393     return this.region.memstoreFlushSize;
394   }
395 
396   @Override
397   public long getFlushableSize() {
398     return this.memstore.getFlushableSize();
399   }
400 
401   @Override
402   public long getSnapshotSize() {
403     return this.memstore.getSnapshotSize();
404   }
405 
406   @Override
407   public long getCompactionCheckMultiplier() {
408     return this.compactionCheckMultiplier;
409   }
410 
411   @Override
412   public long getBlockingFileCount() {
413     return blockingFileCount;
414   }
415   /* End implementation of StoreConfigInformation */
416 
417   /**
418    * Returns the configured bytesPerChecksum value.
419    * @param conf The configuration
420    * @return The bytesPerChecksum that is set in the configuration
421    */
422   public static int getBytesPerChecksum(Configuration conf) {
423     return conf.getInt(HConstants.BYTES_PER_CHECKSUM,
424                        HFile.DEFAULT_BYTES_PER_CHECKSUM);
425   }
426 
427   /**
428    * Returns the configured checksum algorithm.
429    * @param conf The configuration
430    * @return The checksum algorithm that is set in the configuration
431    */
432   public static ChecksumType getChecksumType(Configuration conf) {
433     String checksumName = conf.get(HConstants.CHECKSUM_TYPE_NAME);
434     if (checksumName == null) {
435       return HFile.DEFAULT_CHECKSUM_TYPE;
436     } else {
437       return ChecksumType.nameToType(checksumName);
438     }
439   }
440 
441   /**
442    * @return how many bytes to write between status checks
443    */
444   public static int getCloseCheckInterval() {
445     return closeCheckInterval;
446   }
447 
448   @Override
449   public HColumnDescriptor getFamily() {
450     return this.family;
451   }
452 
453   /**
454    * @return The maximum sequence id in all store files. Used for log replay.
455    */
456   @Override
457   public long getMaxSequenceId() {
458     return StoreFile.getMaxSequenceIdInList(this.getStorefiles());
459   }
460 
461   @Override
462   public long getMaxMemstoreTS() {
463     return StoreFile.getMaxMemstoreTSInList(this.getStorefiles());
464   }
465 
466   /**
467    * @param tabledir {@link Path} to where the table is being stored
468    * @param hri {@link HRegionInfo} for the region.
469    * @param family {@link HColumnDescriptor} describing the column family
470    * @return Path to family/Store home directory.
471    */
472   @Deprecated
473   public static Path getStoreHomedir(final Path tabledir,
474       final HRegionInfo hri, final byte[] family) {
475     return getStoreHomedir(tabledir, hri.getEncodedName(), family);
476   }
477 
478   /**
479    * @param tabledir {@link Path} to where the table is being stored
480    * @param encodedName Encoded region name.
481    * @param family {@link HColumnDescriptor} describing the column family
482    * @return Path to family/Store home directory.
483    */
484   @Deprecated
485   public static Path getStoreHomedir(final Path tabledir,
486       final String encodedName, final byte[] family) {
487     return new Path(tabledir, new Path(encodedName, Bytes.toString(family)));
488   }
489 
490   @Override
491   public HFileDataBlockEncoder getDataBlockEncoder() {
492     return dataBlockEncoder;
493   }
494 
495   /**
496    * Should be used only in tests.
497    * @param blockEncoder the block delta encoder to use
498    */
499   void setDataBlockEncoderInTest(HFileDataBlockEncoder blockEncoder) {
500     this.dataBlockEncoder = blockEncoder;
501   }
502 
503   /**
504    * Creates an unsorted list of StoreFile loaded in parallel
505    * from the given directory.
506    * @throws IOException
507    */
508   private List<StoreFile> loadStoreFiles() throws IOException {
509     Collection<StoreFileInfo> files = fs.getStoreFiles(getColumnFamilyName());
510     return openStoreFiles(files);
511   }
512 
513   private List<StoreFile> openStoreFiles(Collection<StoreFileInfo> files) throws IOException {
514     if (files == null || files.size() == 0) {
515       return new ArrayList<StoreFile>();
516     }
517     // initialize the thread pool for opening store files in parallel..
518     ThreadPoolExecutor storeFileOpenerThreadPool =
519       this.region.getStoreFileOpenAndCloseThreadPool("StoreFileOpenerThread-" +
520           this.getColumnFamilyName());
521     CompletionService<StoreFile> completionService =
522       new ExecutorCompletionService<StoreFile>(storeFileOpenerThreadPool);
523 
524     int totalValidStoreFile = 0;
525     for (final StoreFileInfo storeFileInfo: files) {
526       // open each store file in parallel
527       completionService.submit(new Callable<StoreFile>() {
528         @Override
529         public StoreFile call() throws IOException {
530           StoreFile storeFile = createStoreFileAndReader(storeFileInfo);
531           return storeFile;
532         }
533       });
534       totalValidStoreFile++;
535     }
536 
537     ArrayList<StoreFile> results = new ArrayList<StoreFile>(files.size());
538     IOException ioe = null;
539     try {
540       for (int i = 0; i < totalValidStoreFile; i++) {
541         try {
542           Future<StoreFile> future = completionService.take();
543           StoreFile storeFile = future.get();
544           long length = storeFile.getReader().length();
545           this.storeSize += length;
546           this.totalUncompressedBytes +=
547               storeFile.getReader().getTotalUncompressedBytes();
548           if (LOG.isDebugEnabled()) {
549             LOG.debug("loaded " + storeFile.toStringDetailed());
550           }
551           results.add(storeFile);
552         } catch (InterruptedException e) {
553           if (ioe == null) ioe = new InterruptedIOException(e.getMessage());
554         } catch (ExecutionException e) {
555           if (ioe == null) ioe = new IOException(e.getCause());
556         }
557       }
558     } finally {
559       storeFileOpenerThreadPool.shutdownNow();
560     }
561     if (ioe != null) {
562       // close StoreFile readers
563       for (StoreFile file : results) {
564         try {
565           if (file != null) file.closeReader(true);
566         } catch (IOException e) {
567           LOG.warn(e.getMessage());
568         }
569       }
570       throw ioe;
571     }
572 
573     return results;
574   }
575 
576   /**
577    * Checks the underlying store files, and opens the files that  have not
578    * been opened, and removes the store file readers for store files no longer
579    * available. Mainly used by secondary region replicas to keep up to date with
580    * the primary region files.
581    * @throws IOException
582    */
583   @Override
584   public void refreshStoreFiles() throws IOException {
585     Collection<StoreFileInfo> newFiles = fs.getStoreFiles(getColumnFamilyName());
586     refreshStoreFilesInternal(newFiles);
587   }
588 
589   @Override
590   public void refreshStoreFiles(Collection<String> newFiles) throws IOException {
591     List<StoreFileInfo> storeFiles = new ArrayList<StoreFileInfo>(newFiles.size());
592     for (String file : newFiles) {
593       storeFiles.add(fs.getStoreFileInfo(getColumnFamilyName(), file));
594     }
595     refreshStoreFilesInternal(storeFiles);
596   }
597 
598   /**
599    * Checks the underlying store files, and opens the files that  have not
600    * been opened, and removes the store file readers for store files no longer
601    * available. Mainly used by secondary region replicas to keep up to date with
602    * the primary region files.
603    * @throws IOException
604    */
605   private void refreshStoreFilesInternal(Collection<StoreFileInfo> newFiles) throws IOException {
606     StoreFileManager sfm = storeEngine.getStoreFileManager();
607     Collection<StoreFile> currentFiles = sfm.getStorefiles();
608     if (currentFiles == null) currentFiles = new ArrayList<StoreFile>(0);
609 
610     if (newFiles == null) newFiles = new ArrayList<StoreFileInfo>(0);
611 
612     HashMap<StoreFileInfo, StoreFile> currentFilesSet = new HashMap<StoreFileInfo, StoreFile>(currentFiles.size());
613     for (StoreFile sf : currentFiles) {
614       currentFilesSet.put(sf.getFileInfo(), sf);
615     }
616     HashSet<StoreFileInfo> newFilesSet = new HashSet<StoreFileInfo>(newFiles);
617 
618     Set<StoreFileInfo> toBeAddedFiles = Sets.difference(newFilesSet, currentFilesSet.keySet());
619     Set<StoreFileInfo> toBeRemovedFiles = Sets.difference(currentFilesSet.keySet(), newFilesSet);
620 
621     if (toBeAddedFiles.isEmpty() && toBeRemovedFiles.isEmpty()) {
622       return;
623     }
624 
625     LOG.info("Refreshing store files for region " + this.getRegionInfo().getRegionNameAsString()
626       + " files to add: " + toBeAddedFiles + " files to remove: " + toBeRemovedFiles);
627 
628     Set<StoreFile> toBeRemovedStoreFiles = new HashSet<StoreFile>(toBeRemovedFiles.size());
629     for (StoreFileInfo sfi : toBeRemovedFiles) {
630       toBeRemovedStoreFiles.add(currentFilesSet.get(sfi));
631     }
632 
633     // try to open the files
634     List<StoreFile> openedFiles = openStoreFiles(toBeAddedFiles);
635 
636     // propogate the file changes to the underlying store file manager
637     replaceStoreFiles(toBeRemovedStoreFiles, openedFiles); //won't throw an exception
638 
639     // Advance the memstore read point to be at least the new store files seqIds so that
640     // readers might pick it up. This assumes that the store is not getting any writes (otherwise
641     // in-flight transactions might be made visible)
642     if (!toBeAddedFiles.isEmpty()) {
643       region.getMVCC().advanceMemstoreReadPointIfNeeded(this.getMaxSequenceId());
644     }
645 
646     // notify scanners, close file readers, and recompute store size
647     completeCompaction(toBeRemovedStoreFiles, false);
648   }
649 
650   private StoreFile createStoreFileAndReader(final Path p) throws IOException {
651     StoreFileInfo info = new StoreFileInfo(conf, this.getFileSystem(), p);
652     return createStoreFileAndReader(info);
653   }
654 
655   private StoreFile createStoreFileAndReader(final StoreFileInfo info)
656       throws IOException {
657     info.setRegionCoprocessorHost(this.region.getCoprocessorHost());
658     StoreFile storeFile = new StoreFile(this.getFileSystem(), info, this.conf, this.cacheConf,
659       this.family.getBloomFilterType());
660     storeFile.createReader();
661     return storeFile;
662   }
663 
664   @Override
665   public Pair<Long, Cell> add(final Cell cell) {
666     lock.readLock().lock();
667     try {
668        return this.memstore.add(cell);
669     } finally {
670       lock.readLock().unlock();
671     }
672   }
673 
674   @Override
675   public long timeOfOldestEdit() {
676     return memstore.timeOfOldestEdit();
677   }
678 
679   /**
680    * Adds a value to the memstore
681    *
682    * @param kv
683    * @return memstore size delta
684    */
685   protected long delete(final KeyValue kv) {
686     lock.readLock().lock();
687     try {
688       return this.memstore.delete(kv);
689     } finally {
690       lock.readLock().unlock();
691     }
692   }
693 
694   @Override
695   public void rollback(final Cell cell) {
696     lock.readLock().lock();
697     try {
698       this.memstore.rollback(cell);
699     } finally {
700       lock.readLock().unlock();
701     }
702   }
703 
704   /**
705    * @return All store files.
706    */
707   @Override
708   public Collection<StoreFile> getStorefiles() {
709     return this.storeEngine.getStoreFileManager().getStorefiles();
710   }
711 
712   @Override
713   public void assertBulkLoadHFileOk(Path srcPath) throws IOException {
714     HFile.Reader reader  = null;
715     try {
716       LOG.info("Validating hfile at " + srcPath + " for inclusion in "
717           + "store " + this + " region " + this.getRegionInfo().getRegionNameAsString());
718       reader = HFile.createReader(srcPath.getFileSystem(conf),
719           srcPath, cacheConf, conf);
720       reader.loadFileInfo();
721 
722       byte[] firstKey = reader.getFirstRowKey();
723       Preconditions.checkState(firstKey != null, "First key can not be null");
724       byte[] lk = reader.getLastKey();
725       Preconditions.checkState(lk != null, "Last key can not be null");
726       byte[] lastKey =  KeyValue.createKeyValueFromKey(lk).getRow();
727 
728       LOG.debug("HFile bounds: first=" + Bytes.toStringBinary(firstKey) +
729           " last=" + Bytes.toStringBinary(lastKey));
730       LOG.debug("Region bounds: first=" +
731           Bytes.toStringBinary(getRegionInfo().getStartKey()) +
732           " last=" + Bytes.toStringBinary(getRegionInfo().getEndKey()));
733 
734       if (!this.getRegionInfo().containsRange(firstKey, lastKey)) {
735         throw new WrongRegionException(
736             "Bulk load file " + srcPath.toString() + " does not fit inside region "
737             + this.getRegionInfo().getRegionNameAsString());
738       }
739 
740       if(reader.length() > conf.getLong(HConstants.HREGION_MAX_FILESIZE,
741           HConstants.DEFAULT_MAX_FILE_SIZE)) {
742         LOG.warn("Trying to bulk load hfile " + srcPath.toString() + " with size: " +
743             reader.length() + " bytes can be problematic as it may lead to oversplitting.");
744       }
745 
746       if (verifyBulkLoads) {
747         long verificationStartTime = EnvironmentEdgeManager.currentTime();
748         LOG.info("Full verification started for bulk load hfile: " + srcPath.toString());
749         Cell prevCell = null;
750         HFileScanner scanner = reader.getScanner(false, false, false);
751         scanner.seekTo();
752         do {
753           Cell cell = scanner.getKeyValue();
754           if (prevCell != null) {
755             if (CellComparator.compareRows(prevCell, cell) > 0) {
756               throw new InvalidHFileException("Previous row is greater than"
757                   + " current row: path=" + srcPath + " previous="
758                   + CellUtil.getCellKeyAsString(prevCell) + " current="
759                   + CellUtil.getCellKeyAsString(cell));
760             }
761             if (CellComparator.compareFamilies(prevCell, cell) != 0) {
762               throw new InvalidHFileException("Previous key had different"
763                   + " family compared to current key: path=" + srcPath
764                   + " previous="
765                   + Bytes.toStringBinary(prevCell.getFamilyArray(), prevCell.getFamilyOffset(),
766                       prevCell.getFamilyLength())
767                   + " current="
768                   + Bytes.toStringBinary(cell.getFamilyArray(), cell.getFamilyOffset(),
769                       cell.getFamilyLength()));
770             }
771           }
772           prevCell = cell;
773         } while (scanner.next());
774       LOG.info("Full verification complete for bulk load hfile: " + srcPath.toString()
775          + " took " + (EnvironmentEdgeManager.currentTime() - verificationStartTime)
776          + " ms");
777       }
778     } finally {
779       if (reader != null) reader.close();
780     }
781   }
782 
783   @Override
784   public Path bulkLoadHFile(String srcPathStr, long seqNum) throws IOException {
785     Path srcPath = new Path(srcPathStr);
786     Path dstPath = fs.bulkLoadStoreFile(getColumnFamilyName(), srcPath, seqNum);
787 
788     LOG.info("Loaded HFile " + srcPath + " into store '" + getColumnFamilyName() + "' as "
789         + dstPath + " - updating store file list.");
790 
791     StoreFile sf = createStoreFileAndReader(dstPath);
792     bulkLoadHFile(sf);
793 
794     LOG.info("Successfully loaded store file " + srcPath + " into store " + this
795         + " (new location: " + dstPath + ")");
796 
797     return dstPath;
798   }
799 
800   @Override
801   public void bulkLoadHFile(StoreFileInfo fileInfo) throws IOException {
802     StoreFile sf = createStoreFileAndReader(fileInfo);
803     bulkLoadHFile(sf);
804   }
805 
806   private void bulkLoadHFile(StoreFile sf) throws IOException {
807     StoreFile.Reader r = sf.getReader();
808     this.storeSize += r.length();
809     this.totalUncompressedBytes += r.getTotalUncompressedBytes();
810 
811     // Append the new storefile into the list
812     this.lock.writeLock().lock();
813     try {
814       this.storeEngine.getStoreFileManager().insertNewFiles(Lists.newArrayList(sf));
815     } finally {
816       // We need the lock, as long as we are updating the storeFiles
817       // or changing the memstore. Let us release it before calling
818       // notifyChangeReadersObservers. See HBASE-4485 for a possible
819       // deadlock scenario that could have happened if continue to hold
820       // the lock.
821       this.lock.writeLock().unlock();
822     }
823     notifyChangedReadersObservers();
824     LOG.info("Loaded HFile " + sf.getFileInfo() + " into store '" + getColumnFamilyName());
825     if (LOG.isTraceEnabled()) {
826       String traceMessage = "BULK LOAD time,size,store size,store files ["
827           + EnvironmentEdgeManager.currentTime() + "," + r.length() + "," + storeSize
828           + "," + storeEngine.getStoreFileManager().getStorefileCount() + "]";
829       LOG.trace(traceMessage);
830     }
831   }
832 
833   @Override
834   public ImmutableCollection<StoreFile> close() throws IOException {
835     this.lock.writeLock().lock();
836     try {
837       // Clear so metrics doesn't find them.
838       ImmutableCollection<StoreFile> result = storeEngine.getStoreFileManager().clearFiles();
839 
840       if (!result.isEmpty()) {
841         // initialize the thread pool for closing store files in parallel.
842         ThreadPoolExecutor storeFileCloserThreadPool = this.region
843             .getStoreFileOpenAndCloseThreadPool("StoreFileCloserThread-"
844                 + this.getColumnFamilyName());
845 
846         // close each store file in parallel
847         CompletionService<Void> completionService =
848           new ExecutorCompletionService<Void>(storeFileCloserThreadPool);
849         for (final StoreFile f : result) {
850           completionService.submit(new Callable<Void>() {
851             @Override
852             public Void call() throws IOException {
853               f.closeReader(true);
854               return null;
855             }
856           });
857         }
858 
859         IOException ioe = null;
860         try {
861           for (int i = 0; i < result.size(); i++) {
862             try {
863               Future<Void> future = completionService.take();
864               future.get();
865             } catch (InterruptedException e) {
866               if (ioe == null) {
867                 ioe = new InterruptedIOException();
868                 ioe.initCause(e);
869               }
870             } catch (ExecutionException e) {
871               if (ioe == null) ioe = new IOException(e.getCause());
872             }
873           }
874         } finally {
875           storeFileCloserThreadPool.shutdownNow();
876         }
877         if (ioe != null) throw ioe;
878       }
879       LOG.info("Closed " + this);
880       return result;
881     } finally {
882       this.lock.writeLock().unlock();
883     }
884   }
885 
886   /**
887    * Snapshot this stores memstore. Call before running
888    * {@link #flushCache(long, MemStoreSnapshot, MonitoredTask)}
889    *  so it has some work to do.
890    */
891   void snapshot() {
892     this.lock.writeLock().lock();
893     try {
894       this.memstore.snapshot();
895     } finally {
896       this.lock.writeLock().unlock();
897     }
898   }
899 
900   /**
901    * Write out current snapshot.  Presumes {@link #snapshot()} has been called
902    * previously.
903    * @param logCacheFlushId flush sequence number
904    * @param snapshot
905    * @param status
906    * @return The path name of the tmp file to which the store was flushed
907    * @throws IOException
908    */
909   protected List<Path> flushCache(final long logCacheFlushId, MemStoreSnapshot snapshot,
910       MonitoredTask status) throws IOException {
911     // If an exception happens flushing, we let it out without clearing
912     // the memstore snapshot.  The old snapshot will be returned when we say
913     // 'snapshot', the next time flush comes around.
914     // Retry after catching exception when flushing, otherwise server will abort
915     // itself
916     StoreFlusher flusher = storeEngine.getStoreFlusher();
917     IOException lastException = null;
918     for (int i = 0; i < flushRetriesNumber; i++) {
919       try {
920         List<Path> pathNames = flusher.flushSnapshot(snapshot, logCacheFlushId, status);
921         Path lastPathName = null;
922         try {
923           for (Path pathName : pathNames) {
924             lastPathName = pathName;
925             validateStoreFile(pathName);
926           }
927           return pathNames;
928         } catch (Exception e) {
929           LOG.warn("Failed validating store file " + lastPathName + ", retrying num=" + i, e);
930           if (e instanceof IOException) {
931             lastException = (IOException) e;
932           } else {
933             lastException = new IOException(e);
934           }
935         }
936       } catch (IOException e) {
937         LOG.warn("Failed flushing store file, retrying num=" + i, e);
938         lastException = e;
939       }
940       if (lastException != null && i < (flushRetriesNumber - 1)) {
941         try {
942           Thread.sleep(pauseTime);
943         } catch (InterruptedException e) {
944           IOException iie = new InterruptedIOException();
945           iie.initCause(e);
946           throw iie;
947         }
948       }
949     }
950     throw lastException;
951   }
952 
953   /*
954    * @param path The pathname of the tmp file into which the store was flushed
955    * @param logCacheFlushId
956    * @param status
957    * @return StoreFile created.
958    * @throws IOException
959    */
960   private StoreFile commitFile(final Path path, final long logCacheFlushId, MonitoredTask status)
961       throws IOException {
962     // Write-out finished successfully, move into the right spot
963     Path dstPath = fs.commitStoreFile(getColumnFamilyName(), path);
964 
965     status.setStatus("Flushing " + this + ": reopening flushed file");
966     StoreFile sf = createStoreFileAndReader(dstPath);
967 
968     StoreFile.Reader r = sf.getReader();
969     this.storeSize += r.length();
970     this.totalUncompressedBytes += r.getTotalUncompressedBytes();
971 
972     if (LOG.isInfoEnabled()) {
973       LOG.info("Added " + sf + ", entries=" + r.getEntries() +
974         ", sequenceid=" + logCacheFlushId +
975         ", filesize=" + TraditionalBinaryPrefix.long2String(r.length(), "", 1));
976     }
977     return sf;
978   }
979 
980   /*
981    * @param maxKeyCount
982    * @param compression Compression algorithm to use
983    * @param isCompaction whether we are creating a new file in a compaction
984    * @param includesMVCCReadPoint - whether to include MVCC or not
985    * @param includesTag - includesTag or not
986    * @return Writer for a new StoreFile in the tmp dir.
987    */
988   @Override
989   public StoreFile.Writer createWriterInTmp(long maxKeyCount, Compression.Algorithm compression,
990       boolean isCompaction, boolean includeMVCCReadpoint, boolean includesTag)
991   throws IOException {
992     final CacheConfig writerCacheConf;
993     if (isCompaction) {
994       // Don't cache data on write on compactions.
995       writerCacheConf = new CacheConfig(cacheConf);
996       writerCacheConf.setCacheDataOnWrite(false);
997     } else {
998       writerCacheConf = cacheConf;
999     }
1000     InetSocketAddress[] favoredNodes = null;
1001     if (region.getRegionServerServices() != null) {
1002       favoredNodes = region.getRegionServerServices().getFavoredNodesForRegion(
1003           region.getRegionInfo().getEncodedName());
1004     }
1005     HFileContext hFileContext = createFileContext(compression, includeMVCCReadpoint, includesTag,
1006       cryptoContext);
1007     StoreFile.Writer w = new StoreFile.WriterBuilder(conf, writerCacheConf,
1008         this.getFileSystem())
1009             .withFilePath(fs.createTempName())
1010             .withComparator(comparator)
1011             .withBloomType(family.getBloomFilterType())
1012             .withMaxKeyCount(maxKeyCount)
1013             .withFavoredNodes(favoredNodes)
1014             .withFileContext(hFileContext)
1015             .build();
1016     return w;
1017   }
1018 
1019   private HFileContext createFileContext(Compression.Algorithm compression,
1020       boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context cryptoContext) {
1021     if (compression == null) {
1022       compression = HFile.DEFAULT_COMPRESSION_ALGORITHM;
1023     }
1024     HFileContext hFileContext = new HFileContextBuilder()
1025                                 .withIncludesMvcc(includeMVCCReadpoint)
1026                                 .withIncludesTags(includesTag)
1027                                 .withCompression(compression)
1028                                 .withCompressTags(family.isCompressTags())
1029                                 .withChecksumType(checksumType)
1030                                 .withBytesPerCheckSum(bytesPerChecksum)
1031                                 .withBlockSize(blocksize)
1032                                 .withHBaseCheckSum(true)
1033                                 .withDataBlockEncoding(family.getDataBlockEncoding())
1034                                 .withEncryptionContext(cryptoContext)
1035                                 .withCreateTime(EnvironmentEdgeManager.currentTime())
1036                                 .build();
1037     return hFileContext;
1038   }
1039 
1040 
1041   /*
1042    * Change storeFiles adding into place the Reader produced by this new flush.
1043    * @param sfs Store files
1044    * @param snapshotId
1045    * @throws IOException
1046    * @return Whether compaction is required.
1047    */
1048   private boolean updateStorefiles(final List<StoreFile> sfs, final long snapshotId)
1049       throws IOException {
1050     this.lock.writeLock().lock();
1051     try {
1052       this.storeEngine.getStoreFileManager().insertNewFiles(sfs);
1053       if (snapshotId > 0) {
1054         this.memstore.clearSnapshot(snapshotId);
1055       }
1056     } finally {
1057       // We need the lock, as long as we are updating the storeFiles
1058       // or changing the memstore. Let us release it before calling
1059       // notifyChangeReadersObservers. See HBASE-4485 for a possible
1060       // deadlock scenario that could have happened if continue to hold
1061       // the lock.
1062       this.lock.writeLock().unlock();
1063     }
1064 
1065     // Tell listeners of the change in readers.
1066     notifyChangedReadersObservers();
1067 
1068     if (LOG.isTraceEnabled()) {
1069       long totalSize = 0;
1070       for (StoreFile sf : sfs) {
1071         totalSize += sf.getReader().length();
1072       }
1073       String traceMessage = "FLUSH time,count,size,store size,store files ["
1074           + EnvironmentEdgeManager.currentTime() + "," + sfs.size() + "," + totalSize
1075           + "," + storeSize + "," + storeEngine.getStoreFileManager().getStorefileCount() + "]";
1076       LOG.trace(traceMessage);
1077     }
1078     return needsCompaction();
1079   }
1080 
1081   /*
1082    * Notify all observers that set of Readers has changed.
1083    * @throws IOException
1084    */
1085   private void notifyChangedReadersObservers() throws IOException {
1086     for (ChangedReadersObserver o: this.changedReaderObservers) {
1087       o.updateReaders();
1088     }
1089   }
1090 
1091   /**
1092    * Get all scanners with no filtering based on TTL (that happens further down
1093    * the line).
1094    * @return all scanners for this store
1095    */
1096   @Override
1097   public List<KeyValueScanner> getScanners(boolean cacheBlocks, boolean isGet,
1098       boolean usePread, boolean isCompaction, ScanQueryMatcher matcher, byte[] startRow,
1099       byte[] stopRow, long readPt) throws IOException {
1100     Collection<StoreFile> storeFilesToScan;
1101     List<KeyValueScanner> memStoreScanners;
1102     this.lock.readLock().lock();
1103     try {
1104       storeFilesToScan =
1105           this.storeEngine.getStoreFileManager().getFilesForScanOrGet(isGet, startRow, stopRow);
1106       memStoreScanners = this.memstore.getScanners(readPt);
1107     } finally {
1108       this.lock.readLock().unlock();
1109     }
1110 
1111     // First the store file scanners
1112 
1113     // TODO this used to get the store files in descending order,
1114     // but now we get them in ascending order, which I think is
1115     // actually more correct, since memstore get put at the end.
1116     List<StoreFileScanner> sfScanners = StoreFileScanner
1117       .getScannersForStoreFiles(storeFilesToScan, cacheBlocks, usePread, isCompaction, matcher,
1118         readPt);
1119     List<KeyValueScanner> scanners =
1120       new ArrayList<KeyValueScanner>(sfScanners.size()+1);
1121     scanners.addAll(sfScanners);
1122     // Then the memstore scanners
1123     scanners.addAll(memStoreScanners);
1124     return scanners;
1125   }
1126 
1127   @Override
1128   public void addChangedReaderObserver(ChangedReadersObserver o) {
1129     this.changedReaderObservers.add(o);
1130   }
1131 
1132   @Override
1133   public void deleteChangedReaderObserver(ChangedReadersObserver o) {
1134     // We don't check if observer present; it may not be (legitimately)
1135     this.changedReaderObservers.remove(o);
1136   }
1137 
1138   //////////////////////////////////////////////////////////////////////////////
1139   // Compaction
1140   //////////////////////////////////////////////////////////////////////////////
1141 
1142   /**
1143    * Compact the StoreFiles.  This method may take some time, so the calling
1144    * thread must be able to block for long periods.
1145    *
1146    * <p>During this time, the Store can work as usual, getting values from
1147    * StoreFiles and writing new StoreFiles from the memstore.
1148    *
1149    * Existing StoreFiles are not destroyed until the new compacted StoreFile is
1150    * completely written-out to disk.
1151    *
1152    * <p>The compactLock prevents multiple simultaneous compactions.
1153    * The structureLock prevents us from interfering with other write operations.
1154    *
1155    * <p>We don't want to hold the structureLock for the whole time, as a compact()
1156    * can be lengthy and we want to allow cache-flushes during this period.
1157    *
1158    * <p> Compaction event should be idempotent, since there is no IO Fencing for
1159    * the region directory in hdfs. A region server might still try to complete the
1160    * compaction after it lost the region. That is why the following events are carefully
1161    * ordered for a compaction:
1162    *  1. Compaction writes new files under region/.tmp directory (compaction output)
1163    *  2. Compaction atomically moves the temporary file under region directory
1164    *  3. Compaction appends a WAL edit containing the compaction input and output files.
1165    *  Forces sync on WAL.
1166    *  4. Compaction deletes the input files from the region directory.
1167    *
1168    * Failure conditions are handled like this:
1169    *  - If RS fails before 2, compaction wont complete. Even if RS lives on and finishes
1170    *  the compaction later, it will only write the new data file to the region directory.
1171    *  Since we already have this data, this will be idempotent but we will have a redundant
1172    *  copy of the data.
1173    *  - If RS fails between 2 and 3, the region will have a redundant copy of the data. The
1174    *  RS that failed won't be able to finish snyc() for WAL because of lease recovery in WAL.
1175    *  - If RS fails after 3, the region region server who opens the region will pick up the
1176    *  the compaction marker from the WAL and replay it by removing the compaction input files.
1177    *  Failed RS can also attempt to delete those files, but the operation will be idempotent
1178    *
1179    * See HBASE-2231 for details.
1180    *
1181    * @param compaction compaction details obtained from requestCompaction()
1182    * @throws IOException
1183    * @return Storefile we compacted into or null if we failed or opted out early.
1184    */
1185   @Override
1186   public List<StoreFile> compact(CompactionContext compaction,
1187       CompactionThroughputController throughputController) throws IOException {
1188     assert compaction != null;
1189     List<StoreFile> sfs = null;
1190     CompactionRequest cr = compaction.getRequest();;
1191     try {
1192       // Do all sanity checking in here if we have a valid CompactionRequest
1193       // because we need to clean up after it on the way out in a finally
1194       // block below
1195       long compactionStartTime = EnvironmentEdgeManager.currentTime();
1196       assert compaction.hasSelection();
1197       Collection<StoreFile> filesToCompact = cr.getFiles();
1198       assert !filesToCompact.isEmpty();
1199       synchronized (filesCompacting) {
1200         // sanity check: we're compacting files that this store knows about
1201         // TODO: change this to LOG.error() after more debugging
1202         Preconditions.checkArgument(filesCompacting.containsAll(filesToCompact));
1203       }
1204 
1205       // Ready to go. Have list of files to compact.
1206       LOG.info("Starting compaction of " + filesToCompact.size() + " file(s) in "
1207           + this + " of " + this.getRegionInfo().getRegionNameAsString()
1208           + " into tmpdir=" + fs.getTempDir() + ", totalSize="
1209           + TraditionalBinaryPrefix.long2String(cr.getSize(), "", 1));
1210 
1211       // Commence the compaction.
1212       List<Path> newFiles = compaction.compact(throughputController);
1213 
1214       // TODO: get rid of this!
1215       if (!this.conf.getBoolean("hbase.hstore.compaction.complete", true)) {
1216         LOG.warn("hbase.hstore.compaction.complete is set to false");
1217         sfs = new ArrayList<StoreFile>(newFiles.size());
1218         for (Path newFile : newFiles) {
1219           // Create storefile around what we wrote with a reader on it.
1220           StoreFile sf = createStoreFileAndReader(newFile);
1221           sf.closeReader(true);
1222           sfs.add(sf);
1223         }
1224         return sfs;
1225       }
1226       // Do the steps necessary to complete the compaction.
1227       sfs = moveCompatedFilesIntoPlace(cr, newFiles);
1228       writeCompactionWalRecord(filesToCompact, sfs);
1229       replaceStoreFiles(filesToCompact, sfs);
1230       if (cr.isMajor()) {
1231         majorCompactedCellsCount += getCompactionProgress().totalCompactingKVs;
1232         majorCompactedCellsSize += getCompactionProgress().totalCompactedSize;
1233       } else {
1234         compactedCellsCount += getCompactionProgress().totalCompactingKVs;
1235         compactedCellsSize += getCompactionProgress().totalCompactedSize;
1236       }
1237       // At this point the store will use new files for all new scanners.
1238       completeCompaction(filesToCompact, true); // Archive old files & update store size.
1239 
1240       logCompactionEndMessage(cr, sfs, compactionStartTime);
1241       return sfs;
1242     } finally {
1243       finishCompactionRequest(cr);
1244     }
1245   }
1246 
1247   private List<StoreFile> moveCompatedFilesIntoPlace(
1248       CompactionRequest cr, List<Path> newFiles) throws IOException {
1249     List<StoreFile> sfs = new ArrayList<StoreFile>(newFiles.size());
1250     for (Path newFile : newFiles) {
1251       assert newFile != null;
1252       StoreFile sf = moveFileIntoPlace(newFile);
1253       if (this.getCoprocessorHost() != null) {
1254         this.getCoprocessorHost().postCompact(this, sf, cr);
1255       }
1256       assert sf != null;
1257       sfs.add(sf);
1258     }
1259     return sfs;
1260   }
1261 
1262   // Package-visible for tests
1263   StoreFile moveFileIntoPlace(final Path newFile) throws IOException {
1264     validateStoreFile(newFile);
1265     // Move the file into the right spot
1266     Path destPath = fs.commitStoreFile(getColumnFamilyName(), newFile);
1267     return createStoreFileAndReader(destPath);
1268   }
1269 
1270   /**
1271    * Writes the compaction WAL record.
1272    * @param filesCompacted Files compacted (input).
1273    * @param newFiles Files from compaction.
1274    */
1275   private void writeCompactionWalRecord(Collection<StoreFile> filesCompacted,
1276       Collection<StoreFile> newFiles) throws IOException {
1277     if (region.getWAL() == null) return;
1278     List<Path> inputPaths = new ArrayList<Path>(filesCompacted.size());
1279     for (StoreFile f : filesCompacted) {
1280       inputPaths.add(f.getPath());
1281     }
1282     List<Path> outputPaths = new ArrayList<Path>(newFiles.size());
1283     for (StoreFile f : newFiles) {
1284       outputPaths.add(f.getPath());
1285     }
1286     HRegionInfo info = this.region.getRegionInfo();
1287     CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(info,
1288         family.getName(), inputPaths, outputPaths, fs.getStoreDir(getFamily().getNameAsString()));
1289     WALUtil.writeCompactionMarker(region.getWAL(), this.region.getTableDesc(),
1290         this.region.getRegionInfo(), compactionDescriptor, this.region.getSequenceId());
1291   }
1292 
1293   @VisibleForTesting
1294   void replaceStoreFiles(final Collection<StoreFile> compactedFiles,
1295       final Collection<StoreFile> result) throws IOException {
1296     this.lock.writeLock().lock();
1297     try {
1298       this.storeEngine.getStoreFileManager().addCompactionResults(compactedFiles, result);
1299       filesCompacting.removeAll(compactedFiles); // safe bc: lock.writeLock();
1300     } finally {
1301       this.lock.writeLock().unlock();
1302     }
1303   }
1304 
1305   /**
1306    * Log a very elaborate compaction completion message.
1307    * @param cr Request.
1308    * @param sfs Resulting files.
1309    * @param compactionStartTime Start time.
1310    */
1311   private void logCompactionEndMessage(
1312       CompactionRequest cr, List<StoreFile> sfs, long compactionStartTime) {
1313     long now = EnvironmentEdgeManager.currentTime();
1314     StringBuilder message = new StringBuilder(
1315       "Completed" + (cr.isMajor() ? " major" : "") + " compaction of "
1316       + cr.getFiles().size() + (cr.isAllFiles() ? " (all)" : "") + " file(s) in "
1317       + this + " of " + this.getRegionInfo().getRegionNameAsString() + " into ");
1318     if (sfs.isEmpty()) {
1319       message.append("none, ");
1320     } else {
1321       for (StoreFile sf: sfs) {
1322         message.append(sf.getPath().getName());
1323         message.append("(size=");
1324         message.append(TraditionalBinaryPrefix.long2String(sf.getReader().length(), "", 1));
1325         message.append("), ");
1326       }
1327     }
1328     message.append("total size for store is ")
1329       .append(StringUtils.TraditionalBinaryPrefix.long2String(storeSize, "", 1))
1330       .append(". This selection was in queue for ")
1331       .append(StringUtils.formatTimeDiff(compactionStartTime, cr.getSelectionTime()))
1332       .append(", and took ").append(StringUtils.formatTimeDiff(now, compactionStartTime))
1333       .append(" to execute.");
1334     LOG.info(message.toString());
1335     if (LOG.isTraceEnabled()) {
1336       int fileCount = storeEngine.getStoreFileManager().getStorefileCount();
1337       long resultSize = 0;
1338       for (StoreFile sf : sfs) {
1339         resultSize += sf.getReader().length();
1340       }
1341       String traceMessage = "COMPACTION start,end,size out,files in,files out,store size,"
1342         + "store files [" + compactionStartTime + "," + now + "," + resultSize + ","
1343           + cr.getFiles().size() + "," + sfs.size() + "," +  storeSize + "," + fileCount + "]";
1344       LOG.trace(traceMessage);
1345     }
1346   }
1347 
1348   /**
1349    * Call to complete a compaction. Its for the case where we find in the WAL a compaction
1350    * that was not finished.  We could find one recovering a WAL after a regionserver crash.
1351    * See HBASE-2231.
1352    * @param compaction
1353    */
1354   @Override
1355   public void replayCompactionMarker(CompactionDescriptor compaction,
1356       boolean pickCompactionFiles, boolean removeFiles)
1357       throws IOException {
1358     LOG.debug("Completing compaction from the WAL marker");
1359     List<String> compactionInputs = compaction.getCompactionInputList();
1360     List<String> compactionOutputs = Lists.newArrayList(compaction.getCompactionOutputList());
1361 
1362     // The Compaction Marker is written after the compaction is completed,
1363     // and the files moved into the region/family folder.
1364     //
1365     // If we crash after the entry is written, we may not have removed the
1366     // input files, but the output file is present.
1367     // (The unremoved input files will be removed by this function)
1368     //
1369     // If we scan the directory and the file is not present, it can mean that:
1370     //   - The file was manually removed by the user
1371     //   - The file was removed as consequence of subsequent compaction
1372     // so, we can't do anything with the "compaction output list" because those
1373     // files have already been loaded when opening the region (by virtue of
1374     // being in the store's folder) or they may be missing due to a compaction.
1375 
1376     String familyName = this.getColumnFamilyName();
1377     List<String> inputFiles = new ArrayList<String>(compactionInputs.size());
1378     for (String compactionInput : compactionInputs) {
1379       Path inputPath = fs.getStoreFilePath(familyName, compactionInput);
1380       inputFiles.add(inputPath.getName());
1381     }
1382 
1383     //some of the input files might already be deleted
1384     List<StoreFile> inputStoreFiles = new ArrayList<StoreFile>(compactionInputs.size());
1385     for (StoreFile sf : this.getStorefiles()) {
1386       if (inputFiles.contains(sf.getPath().getName())) {
1387         inputStoreFiles.add(sf);
1388       }
1389     }
1390 
1391     // check whether we need to pick up the new files
1392     List<StoreFile> outputStoreFiles = new ArrayList<StoreFile>(compactionOutputs.size());
1393 
1394     if (pickCompactionFiles) {
1395       for (StoreFile sf : this.getStorefiles()) {
1396         compactionOutputs.remove(sf.getPath().getName());
1397       }
1398       for (String compactionOutput : compactionOutputs) {
1399         StoreFileInfo storeFileInfo = fs.getStoreFileInfo(getColumnFamilyName(), compactionOutput);
1400         StoreFile storeFile = createStoreFileAndReader(storeFileInfo);
1401         outputStoreFiles.add(storeFile);
1402       }
1403     }
1404 
1405     if (!inputStoreFiles.isEmpty() || !outputStoreFiles.isEmpty()) {
1406       LOG.info("Replaying compaction marker, replacing input files: " +
1407           inputStoreFiles + " with output files : " + outputStoreFiles);
1408       this.replaceStoreFiles(inputStoreFiles, outputStoreFiles);
1409       this.completeCompaction(inputStoreFiles, removeFiles);
1410     }
1411   }
1412 
1413   /**
1414    * This method tries to compact N recent files for testing.
1415    * Note that because compacting "recent" files only makes sense for some policies,
1416    * e.g. the default one, it assumes default policy is used. It doesn't use policy,
1417    * but instead makes a compaction candidate list by itself.
1418    * @param N Number of files.
1419    */
1420   public void compactRecentForTestingAssumingDefaultPolicy(int N) throws IOException {
1421     List<StoreFile> filesToCompact;
1422     boolean isMajor;
1423 
1424     this.lock.readLock().lock();
1425     try {
1426       synchronized (filesCompacting) {
1427         filesToCompact = Lists.newArrayList(storeEngine.getStoreFileManager().getStorefiles());
1428         if (!filesCompacting.isEmpty()) {
1429           // exclude all files older than the newest file we're currently
1430           // compacting. this allows us to preserve contiguity (HBASE-2856)
1431           StoreFile last = filesCompacting.get(filesCompacting.size() - 1);
1432           int idx = filesToCompact.indexOf(last);
1433           Preconditions.checkArgument(idx != -1);
1434           filesToCompact.subList(0, idx + 1).clear();
1435         }
1436         int count = filesToCompact.size();
1437         if (N > count) {
1438           throw new RuntimeException("Not enough files");
1439         }
1440 
1441         filesToCompact = filesToCompact.subList(count - N, count);
1442         isMajor = (filesToCompact.size() == storeEngine.getStoreFileManager().getStorefileCount());
1443         filesCompacting.addAll(filesToCompact);
1444         Collections.sort(filesCompacting, StoreFile.Comparators.SEQ_ID);
1445       }
1446     } finally {
1447       this.lock.readLock().unlock();
1448     }
1449 
1450     try {
1451       // Ready to go. Have list of files to compact.
1452       List<Path> newFiles = ((DefaultCompactor)this.storeEngine.getCompactor())
1453           .compactForTesting(filesToCompact, isMajor);
1454       for (Path newFile: newFiles) {
1455         // Move the compaction into place.
1456         StoreFile sf = moveFileIntoPlace(newFile);
1457         if (this.getCoprocessorHost() != null) {
1458           this.getCoprocessorHost().postCompact(this, sf, null);
1459         }
1460         replaceStoreFiles(filesToCompact, Lists.newArrayList(sf));
1461         completeCompaction(filesToCompact, true);
1462       }
1463     } finally {
1464       synchronized (filesCompacting) {
1465         filesCompacting.removeAll(filesToCompact);
1466       }
1467     }
1468   }
1469 
1470   @Override
1471   public boolean hasReferences() {
1472     return StoreUtils.hasReferences(this.storeEngine.getStoreFileManager().getStorefiles());
1473   }
1474 
1475   @Override
1476   public CompactionProgress getCompactionProgress() {
1477     return this.storeEngine.getCompactor().getProgress();
1478   }
1479 
1480   @Override
1481   public boolean isMajorCompaction() throws IOException {
1482     for (StoreFile sf : this.storeEngine.getStoreFileManager().getStorefiles()) {
1483       // TODO: what are these reader checks all over the place?
1484       if (sf.getReader() == null) {
1485         LOG.debug("StoreFile " + sf + " has null Reader");
1486         return false;
1487       }
1488     }
1489     return storeEngine.getCompactionPolicy().isMajorCompaction(
1490         this.storeEngine.getStoreFileManager().getStorefiles());
1491   }
1492 
1493   @Override
1494   public CompactionContext requestCompaction() throws IOException {
1495     return requestCompaction(Store.NO_PRIORITY, null);
1496   }
1497 
1498   @Override
1499   public CompactionContext requestCompaction(int priority, CompactionRequest baseRequest)
1500       throws IOException {
1501     // don't even select for compaction if writes are disabled
1502     if (!this.areWritesEnabled()) {
1503       return null;
1504     }
1505 
1506     // Before we do compaction, try to get rid of unneeded files to simplify things.
1507     removeUnneededFiles();
1508 
1509     CompactionContext compaction = storeEngine.createCompaction();
1510     CompactionRequest request = null;
1511     this.lock.readLock().lock();
1512     try {
1513       synchronized (filesCompacting) {
1514         // First, see if coprocessor would want to override selection.
1515         if (this.getCoprocessorHost() != null) {
1516           List<StoreFile> candidatesForCoproc = compaction.preSelect(this.filesCompacting);
1517           boolean override = this.getCoprocessorHost().preCompactSelection(
1518               this, candidatesForCoproc, baseRequest);
1519           if (override) {
1520             // Coprocessor is overriding normal file selection.
1521             compaction.forceSelect(new CompactionRequest(candidatesForCoproc));
1522           }
1523         }
1524 
1525         // Normal case - coprocessor is not overriding file selection.
1526         if (!compaction.hasSelection()) {
1527           boolean isUserCompaction = priority == Store.PRIORITY_USER;
1528           boolean mayUseOffPeak = offPeakHours.isOffPeakHour() &&
1529               offPeakCompactionTracker.compareAndSet(false, true);
1530           try {
1531             compaction.select(this.filesCompacting, isUserCompaction,
1532               mayUseOffPeak, forceMajor && filesCompacting.isEmpty());
1533           } catch (IOException e) {
1534             if (mayUseOffPeak) {
1535               offPeakCompactionTracker.set(false);
1536             }
1537             throw e;
1538           }
1539           assert compaction.hasSelection();
1540           if (mayUseOffPeak && !compaction.getRequest().isOffPeak()) {
1541             // Compaction policy doesn't want to take advantage of off-peak.
1542             offPeakCompactionTracker.set(false);
1543           }
1544         }
1545         if (this.getCoprocessorHost() != null) {
1546           this.getCoprocessorHost().postCompactSelection(
1547               this, ImmutableList.copyOf(compaction.getRequest().getFiles()), baseRequest);
1548         }
1549 
1550         // Selected files; see if we have a compaction with some custom base request.
1551         if (baseRequest != null) {
1552           // Update the request with what the system thinks the request should be;
1553           // its up to the request if it wants to listen.
1554           compaction.forceSelect(
1555               baseRequest.combineWith(compaction.getRequest()));
1556         }
1557         // Finally, we have the resulting files list. Check if we have any files at all.
1558         request = compaction.getRequest();
1559         final Collection<StoreFile> selectedFiles = request.getFiles();
1560         if (selectedFiles.isEmpty()) {
1561           return null;
1562         }
1563 
1564         addToCompactingFiles(selectedFiles);
1565 
1566         // If we're enqueuing a major, clear the force flag.
1567         this.forceMajor = this.forceMajor && !request.isMajor();
1568 
1569         // Set common request properties.
1570         // Set priority, either override value supplied by caller or from store.
1571         request.setPriority((priority != Store.NO_PRIORITY) ? priority : getCompactPriority());
1572         request.setDescription(getRegionInfo().getRegionNameAsString(), getColumnFamilyName());
1573       }
1574     } finally {
1575       this.lock.readLock().unlock();
1576     }
1577 
1578     LOG.debug(getRegionInfo().getEncodedName() + " - "  + getColumnFamilyName()
1579         + ": Initiating " + (request.isMajor() ? "major" : "minor") + " compaction"
1580         + (request.isAllFiles() ? " (all files)" : ""));
1581     this.region.reportCompactionRequestStart(request.isMajor());
1582     return compaction;
1583   }
1584 
1585   /** Adds the files to compacting files. filesCompacting must be locked. */
1586   private void addToCompactingFiles(final Collection<StoreFile> filesToAdd) {
1587     if (filesToAdd == null) return;
1588     // Check that we do not try to compact the same StoreFile twice.
1589     if (!Collections.disjoint(filesCompacting, filesToAdd)) {
1590       Preconditions.checkArgument(false, "%s overlaps with %s", filesToAdd, filesCompacting);
1591     }
1592     filesCompacting.addAll(filesToAdd);
1593     Collections.sort(filesCompacting, StoreFile.Comparators.SEQ_ID);
1594   }
1595 
1596   private void removeUnneededFiles() throws IOException {
1597     if (!conf.getBoolean("hbase.store.delete.expired.storefile", true)) return;
1598     if (getFamily().getMinVersions() > 0) {
1599       LOG.debug("Skipping expired store file removal due to min version being " +
1600           getFamily().getMinVersions());
1601       return;
1602     }
1603     this.lock.readLock().lock();
1604     Collection<StoreFile> delSfs = null;
1605     try {
1606       synchronized (filesCompacting) {
1607         long cfTtl = getStoreFileTtl();
1608         if (cfTtl != Long.MAX_VALUE) {
1609           delSfs = storeEngine.getStoreFileManager().getUnneededFiles(
1610               EnvironmentEdgeManager.currentTime() - cfTtl, filesCompacting);
1611           addToCompactingFiles(delSfs);
1612         }
1613       }
1614     } finally {
1615       this.lock.readLock().unlock();
1616     }
1617     if (delSfs == null || delSfs.isEmpty()) return;
1618 
1619     Collection<StoreFile> newFiles = new ArrayList<StoreFile>(); // No new files.
1620     writeCompactionWalRecord(delSfs, newFiles);
1621     replaceStoreFiles(delSfs, newFiles);
1622     completeCompaction(delSfs);
1623     LOG.info("Completed removal of " + delSfs.size() + " unnecessary (expired) file(s) in "
1624         + this + " of " + this.getRegionInfo().getRegionNameAsString()
1625         + "; total size for store is " + TraditionalBinaryPrefix.long2String(storeSize, "", 1));
1626   }
1627 
1628   @Override
1629   public void cancelRequestedCompaction(CompactionContext compaction) {
1630     finishCompactionRequest(compaction.getRequest());
1631   }
1632 
1633   private void finishCompactionRequest(CompactionRequest cr) {
1634     this.region.reportCompactionRequestEnd(cr.isMajor(), cr.getFiles().size(), cr.getSize());
1635     if (cr.isOffPeak()) {
1636       offPeakCompactionTracker.set(false);
1637       cr.setOffPeak(false);
1638     }
1639     synchronized (filesCompacting) {
1640       filesCompacting.removeAll(cr.getFiles());
1641     }
1642   }
1643 
1644   /**
1645    * Validates a store file by opening and closing it. In HFileV2 this should
1646    * not be an expensive operation.
1647    *
1648    * @param path the path to the store file
1649    */
1650   private void validateStoreFile(Path path)
1651       throws IOException {
1652     StoreFile storeFile = null;
1653     try {
1654       storeFile = createStoreFileAndReader(path);
1655     } catch (IOException e) {
1656       LOG.error("Failed to open store file : " + path
1657           + ", keeping it in tmp location", e);
1658       throw e;
1659     } finally {
1660       if (storeFile != null) {
1661         storeFile.closeReader(false);
1662       }
1663     }
1664   }
1665 
1666   /**
1667    * <p>It works by processing a compaction that's been written to disk.
1668    *
1669    * <p>It is usually invoked at the end of a compaction, but might also be
1670    * invoked at HStore startup, if the prior execution died midway through.
1671    *
1672    * <p>Moving the compacted TreeMap into place means:
1673    * <pre>
1674    * 1) Unload all replaced StoreFile, close and collect list to delete.
1675    * 2) Compute new store size
1676    * </pre>
1677    *
1678    * @param compactedFiles list of files that were compacted
1679    */
1680   @VisibleForTesting
1681   protected void completeCompaction(final Collection<StoreFile> compactedFiles)
1682     throws IOException {
1683     completeCompaction(compactedFiles, true);
1684   }
1685 
1686 
1687   /**
1688    * <p>It works by processing a compaction that's been written to disk.
1689    *
1690    * <p>It is usually invoked at the end of a compaction, but might also be
1691    * invoked at HStore startup, if the prior execution died midway through.
1692    *
1693    * <p>Moving the compacted TreeMap into place means:
1694    * <pre>
1695    * 1) Unload all replaced StoreFile, close and collect list to delete.
1696    * 2) Compute new store size
1697    * </pre>
1698    *
1699    * @param compactedFiles list of files that were compacted
1700    */
1701   @VisibleForTesting
1702   protected void completeCompaction(final Collection<StoreFile> compactedFiles, boolean removeFiles)
1703       throws IOException {
1704     try {
1705       // Do not delete old store files until we have sent out notification of
1706       // change in case old files are still being accessed by outstanding scanners.
1707       // Don't do this under writeLock; see HBASE-4485 for a possible deadlock
1708       // scenario that could have happened if continue to hold the lock.
1709       notifyChangedReadersObservers();
1710       // At this point the store will use new files for all scanners.
1711 
1712       // let the archive util decide if we should archive or delete the files
1713       LOG.debug("Removing store files after compaction...");
1714       for (StoreFile compactedFile : compactedFiles) {
1715         compactedFile.closeReader(true);
1716       }
1717       if (removeFiles) {
1718         this.fs.removeStoreFiles(this.getColumnFamilyName(), compactedFiles);
1719       }
1720     } catch (IOException e) {
1721       e = e instanceof RemoteException ?
1722                 ((RemoteException)e).unwrapRemoteException() : e;
1723       LOG.error("Failed removing compacted files in " + this +
1724         ". Files we were trying to remove are " + compactedFiles.toString() +
1725         "; some of them may have been already removed", e);
1726     }
1727 
1728     // 4. Compute new store size
1729     this.storeSize = 0L;
1730     this.totalUncompressedBytes = 0L;
1731     for (StoreFile hsf : this.storeEngine.getStoreFileManager().getStorefiles()) {
1732       StoreFile.Reader r = hsf.getReader();
1733       if (r == null) {
1734         LOG.warn("StoreFile " + hsf + " has a null Reader");
1735         continue;
1736       }
1737       this.storeSize += r.length();
1738       this.totalUncompressedBytes += r.getTotalUncompressedBytes();
1739     }
1740   }
1741 
1742   /*
1743    * @param wantedVersions How many versions were asked for.
1744    * @return wantedVersions or this families' {@link HConstants#VERSIONS}.
1745    */
1746   int versionsToReturn(final int wantedVersions) {
1747     if (wantedVersions <= 0) {
1748       throw new IllegalArgumentException("Number of versions must be > 0");
1749     }
1750     // Make sure we do not return more than maximum versions for this store.
1751     int maxVersions = this.family.getMaxVersions();
1752     return wantedVersions > maxVersions ? maxVersions: wantedVersions;
1753   }
1754 
1755   /**
1756    * @param cell
1757    * @param oldestTimestamp
1758    * @return true if the cell is expired
1759    */
1760   static boolean isCellTTLExpired(final Cell cell, final long oldestTimestamp, final long now) {
1761     // Do not create an Iterator or Tag objects unless the cell actually has tags.
1762     // TODO: This check for tags is really expensive. We decode an int for key and value. Costs.
1763     if (cell.getTagsLength() > 0) {
1764       // Look for a TTL tag first. Use it instead of the family setting if
1765       // found. If a cell has multiple TTLs, resolve the conflict by using the
1766       // first tag encountered.
1767       Iterator<Tag> i = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(),
1768         cell.getTagsLength());
1769       while (i.hasNext()) {
1770         Tag t = i.next();
1771         if (TagType.TTL_TAG_TYPE == t.getType()) {
1772           // Unlike in schema cell TTLs are stored in milliseconds, no need
1773           // to convert
1774           long ts = cell.getTimestamp();
1775           assert t.getTagLength() == Bytes.SIZEOF_LONG;
1776           long ttl = Bytes.toLong(t.getBuffer(), t.getTagOffset(), t.getTagLength());
1777           if (ts + ttl < now) {
1778             return true;
1779           }
1780           // Per cell TTLs cannot extend lifetime beyond family settings, so
1781           // fall through to check that
1782           break;
1783         }
1784       }
1785     }
1786     return false;
1787   }
1788 
1789   @Override
1790   public Cell getRowKeyAtOrBefore(final byte[] row) throws IOException {
1791     // If minVersions is set, we will not ignore expired KVs.
1792     // As we're only looking for the latest matches, that should be OK.
1793     // With minVersions > 0 we guarantee that any KV that has any version
1794     // at all (expired or not) has at least one version that will not expire.
1795     // Note that this method used to take a KeyValue as arguments. KeyValue
1796     // can be back-dated, a row key cannot.
1797     long ttlToUse = scanInfo.getMinVersions() > 0 ? Long.MAX_VALUE : this.scanInfo.getTtl();
1798 
1799     KeyValue kv = new KeyValue(row, HConstants.LATEST_TIMESTAMP);
1800 
1801     GetClosestRowBeforeTracker state = new GetClosestRowBeforeTracker(
1802       this.comparator, kv, ttlToUse, this.getRegionInfo().isMetaRegion());
1803     this.lock.readLock().lock();
1804     try {
1805       // First go to the memstore.  Pick up deletes and candidates.
1806       this.memstore.getRowKeyAtOrBefore(state);
1807       // Check if match, if we got a candidate on the asked for 'kv' row.
1808       // Process each relevant store file. Run through from newest to oldest.
1809       Iterator<StoreFile> sfIterator = this.storeEngine.getStoreFileManager()
1810           .getCandidateFilesForRowKeyBefore(state.getTargetKey());
1811       while (sfIterator.hasNext()) {
1812         StoreFile sf = sfIterator.next();
1813         sfIterator.remove(); // Remove sf from iterator.
1814         boolean haveNewCandidate = rowAtOrBeforeFromStoreFile(sf, state);
1815         Cell candidate = state.getCandidate();
1816         // we have an optimization here which stops the search if we find exact match.
1817         if (candidate != null && CellUtil.matchingRow(candidate, row)) {
1818           return candidate;
1819         }
1820         if (haveNewCandidate) {
1821           sfIterator = this.storeEngine.getStoreFileManager().updateCandidateFilesForRowKeyBefore(
1822               sfIterator, state.getTargetKey(), candidate);
1823         }
1824       }
1825       return state.getCandidate();
1826     } finally {
1827       this.lock.readLock().unlock();
1828     }
1829   }
1830 
1831   /*
1832    * Check an individual MapFile for the row at or before a given row.
1833    * @param f
1834    * @param state
1835    * @throws IOException
1836    * @return True iff the candidate has been updated in the state.
1837    */
1838   private boolean rowAtOrBeforeFromStoreFile(final StoreFile f,
1839                                           final GetClosestRowBeforeTracker state)
1840       throws IOException {
1841     StoreFile.Reader r = f.getReader();
1842     if (r == null) {
1843       LOG.warn("StoreFile " + f + " has a null Reader");
1844       return false;
1845     }
1846     if (r.getEntries() == 0) {
1847       LOG.warn("StoreFile " + f + " is a empty store file");
1848       return false;
1849     }
1850     // TODO: Cache these keys rather than make each time?
1851     byte [] fk = r.getFirstKey();
1852     if (fk == null) return false;
1853     KeyValue firstKV = KeyValue.createKeyValueFromKey(fk, 0, fk.length);
1854     byte [] lk = r.getLastKey();
1855     KeyValue lastKV = KeyValue.createKeyValueFromKey(lk, 0, lk.length);
1856     KeyValue firstOnRow = state.getTargetKey();
1857     if (this.comparator.compareRows(lastKV, firstOnRow) < 0) {
1858       // If last key in file is not of the target table, no candidates in this
1859       // file.  Return.
1860       if (!state.isTargetTable(lastKV)) return false;
1861       // If the row we're looking for is past the end of file, set search key to
1862       // last key. TODO: Cache last and first key rather than make each time.
1863       firstOnRow = new KeyValue(lastKV.getRow(), HConstants.LATEST_TIMESTAMP);
1864     }
1865     // Get a scanner that caches blocks and that uses pread.
1866     HFileScanner scanner = r.getScanner(true, true, false);
1867     // Seek scanner.  If can't seek it, return.
1868     if (!seekToScanner(scanner, firstOnRow, firstKV)) return false;
1869     // If we found candidate on firstOnRow, just return. THIS WILL NEVER HAPPEN!
1870     // Unlikely that there'll be an instance of actual first row in table.
1871     if (walkForwardInSingleRow(scanner, firstOnRow, state)) return true;
1872     // If here, need to start backing up.
1873     while (scanner.seekBefore(firstOnRow)) {
1874       Cell kv = scanner.getKeyValue();
1875       if (!state.isTargetTable(kv)) break;
1876       if (!state.isBetterCandidate(kv)) break;
1877       // Make new first on row.
1878       firstOnRow = new KeyValue(kv.getRow(), HConstants.LATEST_TIMESTAMP);
1879       // Seek scanner.  If can't seek it, break.
1880       if (!seekToScanner(scanner, firstOnRow, firstKV)) return false;
1881       // If we find something, break;
1882       if (walkForwardInSingleRow(scanner, firstOnRow, state)) return true;
1883     }
1884     return false;
1885   }
1886 
1887   /*
1888    * Seek the file scanner to firstOnRow or first entry in file.
1889    * @param scanner
1890    * @param firstOnRow
1891    * @param firstKV
1892    * @return True if we successfully seeked scanner.
1893    * @throws IOException
1894    */
1895   private boolean seekToScanner(final HFileScanner scanner,
1896                                 final KeyValue firstOnRow,
1897                                 final KeyValue firstKV)
1898       throws IOException {
1899     KeyValue kv = firstOnRow;
1900     // If firstOnRow < firstKV, set to firstKV
1901     if (this.comparator.compareRows(firstKV, firstOnRow) == 0) kv = firstKV;
1902     int result = scanner.seekTo(kv);
1903     return result != -1;
1904   }
1905 
1906   /*
1907    * When we come in here, we are probably at the kv just before we break into
1908    * the row that firstOnRow is on.  Usually need to increment one time to get
1909    * on to the row we are interested in.
1910    * @param scanner
1911    * @param firstOnRow
1912    * @param state
1913    * @return True we found a candidate.
1914    * @throws IOException
1915    */
1916   private boolean walkForwardInSingleRow(final HFileScanner scanner,
1917                                          final KeyValue firstOnRow,
1918                                          final GetClosestRowBeforeTracker state)
1919       throws IOException {
1920     boolean foundCandidate = false;
1921     do {
1922       Cell kv = scanner.getKeyValue();
1923       // If we are not in the row, skip.
1924       if (this.comparator.compareRows(kv, firstOnRow) < 0) continue;
1925       // Did we go beyond the target row? If so break.
1926       if (state.isTooFar(kv, firstOnRow)) break;
1927       if (state.isExpired(kv)) {
1928         continue;
1929       }
1930       // If we added something, this row is a contender. break.
1931       if (state.handle(kv)) {
1932         foundCandidate = true;
1933         break;
1934       }
1935     } while(scanner.next());
1936     return foundCandidate;
1937   }
1938 
1939   @Override
1940   public boolean canSplit() {
1941     this.lock.readLock().lock();
1942     try {
1943       // Not split-able if we find a reference store file present in the store.
1944       boolean result = !hasReferences();
1945       if (!result && LOG.isDebugEnabled()) {
1946         LOG.debug("Cannot split region due to reference files being there");
1947       }
1948       return result;
1949     } finally {
1950       this.lock.readLock().unlock();
1951     }
1952   }
1953 
1954   @Override
1955   public byte[] getSplitPoint() {
1956     this.lock.readLock().lock();
1957     try {
1958       // Should already be enforced by the split policy!
1959       assert !this.getRegionInfo().isMetaRegion();
1960       // Not split-able if we find a reference store file present in the store.
1961       if (hasReferences()) {
1962         return null;
1963       }
1964       return this.storeEngine.getStoreFileManager().getSplitPoint();
1965     } catch(IOException e) {
1966       LOG.warn("Failed getting store size for " + this, e);
1967     } finally {
1968       this.lock.readLock().unlock();
1969     }
1970     return null;
1971   }
1972 
1973   @Override
1974   public long getLastCompactSize() {
1975     return this.lastCompactSize;
1976   }
1977 
1978   @Override
1979   public long getSize() {
1980     return storeSize;
1981   }
1982 
1983   @Override
1984   public void triggerMajorCompaction() {
1985     this.forceMajor = true;
1986   }
1987 
1988 
1989   //////////////////////////////////////////////////////////////////////////////
1990   // File administration
1991   //////////////////////////////////////////////////////////////////////////////
1992 
1993   @Override
1994   public KeyValueScanner getScanner(Scan scan,
1995       final NavigableSet<byte []> targetCols, long readPt) throws IOException {
1996     lock.readLock().lock();
1997     try {
1998       KeyValueScanner scanner = null;
1999       if (this.getCoprocessorHost() != null) {
2000         scanner = this.getCoprocessorHost().preStoreScannerOpen(this, scan, targetCols);
2001       }
2002       if (scanner == null) {
2003         scanner = scan.isReversed() ? new ReversedStoreScanner(this,
2004             getScanInfo(), scan, targetCols, readPt) : new StoreScanner(this,
2005             getScanInfo(), scan, targetCols, readPt);
2006       }
2007       return scanner;
2008     } finally {
2009       lock.readLock().unlock();
2010     }
2011   }
2012 
2013   @Override
2014   public String toString() {
2015     return this.getColumnFamilyName();
2016   }
2017 
2018   @Override
2019   public int getStorefilesCount() {
2020     return this.storeEngine.getStoreFileManager().getStorefileCount();
2021   }
2022 
2023   @Override
2024   public long getStoreSizeUncompressed() {
2025     return this.totalUncompressedBytes;
2026   }
2027 
2028   @Override
2029   public long getStorefilesSize() {
2030     long size = 0;
2031     for (StoreFile s: this.storeEngine.getStoreFileManager().getStorefiles()) {
2032       StoreFile.Reader r = s.getReader();
2033       if (r == null) {
2034         LOG.warn("StoreFile " + s + " has a null Reader");
2035         continue;
2036       }
2037       size += r.length();
2038     }
2039     return size;
2040   }
2041 
2042   @Override
2043   public long getStorefilesIndexSize() {
2044     long size = 0;
2045     for (StoreFile s: this.storeEngine.getStoreFileManager().getStorefiles()) {
2046       StoreFile.Reader r = s.getReader();
2047       if (r == null) {
2048         LOG.warn("StoreFile " + s + " has a null Reader");
2049         continue;
2050       }
2051       size += r.indexSize();
2052     }
2053     return size;
2054   }
2055 
2056   @Override
2057   public long getTotalStaticIndexSize() {
2058     long size = 0;
2059     for (StoreFile s : this.storeEngine.getStoreFileManager().getStorefiles()) {
2060       size += s.getReader().getUncompressedDataIndexSize();
2061     }
2062     return size;
2063   }
2064 
2065   @Override
2066   public long getTotalStaticBloomSize() {
2067     long size = 0;
2068     for (StoreFile s : this.storeEngine.getStoreFileManager().getStorefiles()) {
2069       StoreFile.Reader r = s.getReader();
2070       size += r.getTotalBloomSize();
2071     }
2072     return size;
2073   }
2074 
2075   @Override
2076   public long getMemStoreSize() {
2077     return this.memstore.size();
2078   }
2079 
2080   @Override
2081   public int getCompactPriority() {
2082     int priority = this.storeEngine.getStoreFileManager().getStoreCompactionPriority();
2083     if (priority == PRIORITY_USER) {
2084       LOG.warn("Compaction priority is USER despite there being no user compaction");
2085     }
2086     return priority;
2087   }
2088 
2089   @Override
2090   public boolean throttleCompaction(long compactionSize) {
2091     return storeEngine.getCompactionPolicy().throttleCompaction(compactionSize);
2092   }
2093 
2094   public HRegion getHRegion() {
2095     return this.region;
2096   }
2097 
2098   @Override
2099   public RegionCoprocessorHost getCoprocessorHost() {
2100     return this.region.getCoprocessorHost();
2101   }
2102 
2103   @Override
2104   public HRegionInfo getRegionInfo() {
2105     return this.fs.getRegionInfo();
2106   }
2107 
2108   @Override
2109   public boolean areWritesEnabled() {
2110     return this.region.areWritesEnabled();
2111   }
2112 
2113   @Override
2114   public long getSmallestReadPoint() {
2115     return this.region.getSmallestReadPoint();
2116   }
2117 
2118   /**
2119    * Used in tests. TODO: Remove
2120    *
2121    * Updates the value for the given row/family/qualifier. This function will always be seen as
2122    * atomic by other readers because it only puts a single KV to memstore. Thus no read/write
2123    * control necessary.
2124    * @param row row to update
2125    * @param f family to update
2126    * @param qualifier qualifier to update
2127    * @param newValue the new value to set into memstore
2128    * @return memstore size delta
2129    * @throws IOException
2130    */
2131   public long updateColumnValue(byte [] row, byte [] f,
2132                                 byte [] qualifier, long newValue)
2133       throws IOException {
2134 
2135     this.lock.readLock().lock();
2136     try {
2137       long now = EnvironmentEdgeManager.currentTime();
2138 
2139       return this.memstore.updateColumnValue(row,
2140           f,
2141           qualifier,
2142           newValue,
2143           now);
2144 
2145     } finally {
2146       this.lock.readLock().unlock();
2147     }
2148   }
2149 
2150   @Override
2151   public long upsert(Iterable<Cell> cells, long readpoint) throws IOException {
2152     this.lock.readLock().lock();
2153     try {
2154       return this.memstore.upsert(cells, readpoint);
2155     } finally {
2156       this.lock.readLock().unlock();
2157     }
2158   }
2159 
2160   @Override
2161   public StoreFlushContext createFlushContext(long cacheFlushId) {
2162     return new StoreFlusherImpl(cacheFlushId);
2163   }
2164 
2165   private class StoreFlusherImpl implements StoreFlushContext {
2166 
2167     private long cacheFlushSeqNum;
2168     private MemStoreSnapshot snapshot;
2169     private List<Path> tempFiles;
2170     private List<Path> committedFiles;
2171     private long cacheFlushCount;
2172     private long cacheFlushSize;
2173 
2174     private StoreFlusherImpl(long cacheFlushSeqNum) {
2175       this.cacheFlushSeqNum = cacheFlushSeqNum;
2176     }
2177 
2178     /**
2179      * This is not thread safe. The caller should have a lock on the region or the store.
2180      * If necessary, the lock can be added with the patch provided in HBASE-10087
2181      */
2182     @Override
2183     public void prepare() {
2184       this.snapshot = memstore.snapshot();
2185       this.cacheFlushCount = snapshot.getCellsCount();
2186       this.cacheFlushSize = snapshot.getSize();
2187       committedFiles = new ArrayList<Path>(1);
2188     }
2189 
2190     @Override
2191     public void flushCache(MonitoredTask status) throws IOException {
2192       tempFiles = HStore.this.flushCache(cacheFlushSeqNum, snapshot, status);
2193     }
2194 
2195     @Override
2196     public boolean commit(MonitoredTask status) throws IOException {
2197       if (this.tempFiles == null || this.tempFiles.isEmpty()) {
2198         return false;
2199       }
2200       List<StoreFile> storeFiles = new ArrayList<StoreFile>(this.tempFiles.size());
2201       for (Path storeFilePath : tempFiles) {
2202         try {
2203           storeFiles.add(HStore.this.commitFile(storeFilePath, cacheFlushSeqNum, status));
2204         } catch (IOException ex) {
2205           LOG.error("Failed to commit store file " + storeFilePath, ex);
2206           // Try to delete the files we have committed before.
2207           for (StoreFile sf : storeFiles) {
2208             Path pathToDelete = sf.getPath();
2209             try {
2210               sf.deleteReader();
2211             } catch (IOException deleteEx) {
2212               LOG.fatal("Failed to delete store file we committed, halting " + pathToDelete, ex);
2213               Runtime.getRuntime().halt(1);
2214             }
2215           }
2216           throw new IOException("Failed to commit the flush", ex);
2217         }
2218       }
2219 
2220       for (StoreFile sf : storeFiles) {
2221         if (HStore.this.getCoprocessorHost() != null) {
2222           HStore.this.getCoprocessorHost().postFlush(HStore.this, sf);
2223         }
2224         committedFiles.add(sf.getPath());
2225       }
2226 
2227       HStore.this.flushedCellsCount += cacheFlushCount;
2228       HStore.this.flushedCellsSize += cacheFlushSize;
2229 
2230       // Add new file to store files.  Clear snapshot too while we have the Store write lock.
2231       return HStore.this.updateStorefiles(storeFiles, snapshot.getId());
2232     }
2233 
2234     @Override
2235     public List<Path> getCommittedFiles() {
2236       return committedFiles;
2237     }
2238 
2239     /**
2240      * Similar to commit, but called in secondary region replicas for replaying the
2241      * flush cache from primary region. Adds the new files to the store, and drops the
2242      * snapshot depending on dropMemstoreSnapshot argument.
2243      * @param fileNames names of the flushed files
2244      * @param dropMemstoreSnapshot whether to drop the prepared memstore snapshot
2245      * @throws IOException
2246      */
2247     @Override
2248     public void replayFlush(List<String> fileNames, boolean dropMemstoreSnapshot)
2249         throws IOException {
2250       List<StoreFile> storeFiles = new ArrayList<StoreFile>(fileNames.size());
2251       for (String file : fileNames) {
2252         // open the file as a store file (hfile link, etc)
2253         StoreFileInfo storeFileInfo = fs.getStoreFileInfo(getColumnFamilyName(), file);
2254         StoreFile storeFile = createStoreFileAndReader(storeFileInfo);
2255         storeFiles.add(storeFile);
2256         HStore.this.storeSize += storeFile.getReader().length();
2257         HStore.this.totalUncompressedBytes += storeFile.getReader().getTotalUncompressedBytes();
2258         if (LOG.isInfoEnabled()) {
2259           LOG.info("Region: " + HStore.this.getRegionInfo().getEncodedName() +
2260             " added " + storeFile + ", entries=" + storeFile.getReader().getEntries() +
2261             ", sequenceid=" +  + storeFile.getReader().getSequenceID() +
2262             ", filesize=" + StringUtils.humanReadableInt(storeFile.getReader().length()));
2263         }
2264       }
2265 
2266       long snapshotId = -1; // -1 means do not drop
2267       if (dropMemstoreSnapshot && snapshot != null) {
2268         snapshotId = snapshot.getId();
2269       }
2270       HStore.this.updateStorefiles(storeFiles, snapshotId);
2271     }
2272 
2273     /**
2274      * Abort the snapshot preparation. Drops the snapshot if any.
2275      * @throws IOException
2276      */
2277     @Override
2278     public void abort() throws IOException {
2279       if (snapshot == null) {
2280         return;
2281       }
2282       HStore.this.updateStorefiles(new ArrayList<StoreFile>(0), snapshot.getId());
2283     }
2284   }
2285 
2286   @Override
2287   public boolean needsCompaction() {
2288     return this.storeEngine.needsCompaction(this.filesCompacting);
2289   }
2290 
2291   @Override
2292   public CacheConfig getCacheConfig() {
2293     return this.cacheConf;
2294   }
2295 
2296   public static final long FIXED_OVERHEAD =
2297       ClassSize.align(ClassSize.OBJECT + (16 * ClassSize.REFERENCE) + (10 * Bytes.SIZEOF_LONG)
2298               + (5 * Bytes.SIZEOF_INT) + (2 * Bytes.SIZEOF_BOOLEAN));
2299 
2300   public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD
2301       + ClassSize.OBJECT + ClassSize.REENTRANT_LOCK
2302       + ClassSize.CONCURRENT_SKIPLISTMAP
2303       + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY + ClassSize.OBJECT
2304       + ScanInfo.FIXED_OVERHEAD);
2305 
2306   @Override
2307   public long heapSize() {
2308     return DEEP_OVERHEAD + this.memstore.heapSize();
2309   }
2310 
2311   @Override
2312   public KeyValue.KVComparator getComparator() {
2313     return comparator;
2314   }
2315 
2316   @Override
2317   public ScanInfo getScanInfo() {
2318     return scanInfo;
2319   }
2320 
2321   /**
2322    * Set scan info, used by test
2323    * @param scanInfo new scan info to use for test
2324    */
2325   void setScanInfo(ScanInfo scanInfo) {
2326     this.scanInfo = scanInfo;
2327   }
2328 
2329   @Override
2330   public boolean hasTooManyStoreFiles() {
2331     return getStorefilesCount() > this.blockingFileCount;
2332   }
2333 
2334   @Override
2335   public long getFlushedCellsCount() {
2336     return flushedCellsCount;
2337   }
2338 
2339   @Override
2340   public long getFlushedCellsSize() {
2341     return flushedCellsSize;
2342   }
2343 
2344   @Override
2345   public long getCompactedCellsCount() {
2346     return compactedCellsCount;
2347   }
2348 
2349   @Override
2350   public long getCompactedCellsSize() {
2351     return compactedCellsSize;
2352   }
2353 
2354   @Override
2355   public long getMajorCompactedCellsCount() {
2356     return majorCompactedCellsCount;
2357   }
2358 
2359   @Override
2360   public long getMajorCompactedCellsSize() {
2361     return majorCompactedCellsSize;
2362   }
2363 
2364   /**
2365    * Returns the StoreEngine that is backing this concrete implementation of Store.
2366    * @return Returns the {@link StoreEngine} object used internally inside this HStore object.
2367    */
2368   @VisibleForTesting
2369   public StoreEngine<?, ?, ?, ?> getStoreEngine() {
2370     return this.storeEngine;
2371   }
2372 
2373   protected OffPeakHours getOffPeakHours() {
2374     return this.offPeakHours;
2375   }
2376 
2377   /**
2378    * {@inheritDoc}
2379    */
2380   @Override
2381   public void onConfigurationChange(Configuration conf) {
2382     this.conf = new CompoundConfiguration()
2383             .add(conf)
2384             .addBytesMap(family.getValues());
2385     this.storeEngine.compactionPolicy.setConf(conf);
2386     this.offPeakHours = OffPeakHours.getInstance(conf);
2387   }
2388 
2389   /**
2390    * {@inheritDoc}
2391    */
2392   @Override
2393   public void registerChildren(ConfigurationManager manager) {
2394     // No children to register
2395   }
2396 
2397   /**
2398    * {@inheritDoc}
2399    */
2400   @Override
2401   public void deregisterChildren(ConfigurationManager manager) {
2402     // No children to deregister
2403   }
2404 
2405   @Override
2406   public double getCompactionPressure() {
2407     return storeEngine.getStoreFileManager().getCompactionPressure();
2408   }
2409 }