View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import java.io.IOException;
22  import java.io.InterruptedIOException;
23  import java.net.InetSocketAddress;
24  import java.security.PrivilegedExceptionAction;
25  import java.util.ArrayList;
26  import java.util.Collection;
27  import java.util.Collections;
28  import java.util.HashMap;
29  import java.util.HashSet;
30  import java.util.Iterator;
31  import java.util.List;
32  import java.util.NavigableSet;
33  import java.util.Set;
34  import java.util.concurrent.Callable;
35  import java.util.concurrent.CompletionService;
36  import java.util.concurrent.ConcurrentHashMap;
37  import java.util.concurrent.ExecutionException;
38  import java.util.concurrent.ExecutorCompletionService;
39  import java.util.concurrent.Future;
40  import java.util.concurrent.ThreadPoolExecutor;
41  import java.util.concurrent.atomic.AtomicBoolean;
42  import java.util.concurrent.locks.ReentrantReadWriteLock;
43  
44  import org.apache.commons.logging.Log;
45  import org.apache.commons.logging.LogFactory;
46  import org.apache.hadoop.conf.Configuration;
47  import org.apache.hadoop.fs.FileSystem;
48  import org.apache.hadoop.fs.Path;
49  import org.apache.hadoop.hbase.Cell;
50  import org.apache.hadoop.hbase.CellComparator;
51  import org.apache.hadoop.hbase.CellUtil;
52  import org.apache.hadoop.hbase.CompoundConfiguration;
53  import org.apache.hadoop.hbase.HColumnDescriptor;
54  import org.apache.hadoop.hbase.HConstants;
55  import org.apache.hadoop.hbase.HRegionInfo;
56  import org.apache.hadoop.hbase.KeyValue;
57  import org.apache.hadoop.hbase.TableName;
58  import org.apache.hadoop.hbase.Tag;
59  import org.apache.hadoop.hbase.TagType;
60  import org.apache.hadoop.hbase.TagUtil;
61  import org.apache.hadoop.hbase.classification.InterfaceAudience;
62  import org.apache.hadoop.hbase.client.Scan;
63  import org.apache.hadoop.hbase.conf.ConfigurationManager;
64  import org.apache.hadoop.hbase.io.compress.Compression;
65  import org.apache.hadoop.hbase.io.crypto.Encryption;
66  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
67  import org.apache.hadoop.hbase.io.hfile.HFile;
68  import org.apache.hadoop.hbase.io.hfile.HFileContext;
69  import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
70  import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
71  import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
72  import org.apache.hadoop.hbase.io.hfile.HFileScanner;
73  import org.apache.hadoop.hbase.io.hfile.InvalidHFileException;
74  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
75  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
76  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
77  import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
78  import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
79  import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
80  import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
81  import org.apache.hadoop.hbase.regionserver.compactions.OffPeakHours;
82  import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;
83  import org.apache.hadoop.hbase.regionserver.wal.WALUtil;
84  import org.apache.hadoop.hbase.security.EncryptionUtil;
85  import org.apache.hadoop.hbase.security.User;
86  import org.apache.hadoop.hbase.util.Bytes;
87  import org.apache.hadoop.hbase.util.ChecksumType;
88  import org.apache.hadoop.hbase.util.ClassSize;
89  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
90  import org.apache.hadoop.hbase.util.ReflectionUtils;
91  import org.apache.hadoop.util.StringUtils;
92  import org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix;
93  
94  import com.google.common.annotations.VisibleForTesting;
95  import com.google.common.base.Preconditions;
96  import com.google.common.collect.ImmutableCollection;
97  import com.google.common.collect.ImmutableList;
98  import com.google.common.collect.Lists;
99  import com.google.common.collect.Sets;
100 
101 /**
102  * A Store holds a column family in a Region.  Its a memstore and a set of zero
103  * or more StoreFiles, which stretch backwards over time.
104  *
105  * <p>There's no reason to consider append-logging at this level; all logging
106  * and locking is handled at the HRegion level.  Store just provides
107  * services to manage sets of StoreFiles.  One of the most important of those
108  * services is compaction services where files are aggregated once they pass
109  * a configurable threshold.
110  *
111  * <p>Locking and transactions are handled at a higher level.  This API should
112  * not be called directly but by an HRegion manager.
113  */
114 @InterfaceAudience.Private
115 public class HStore implements Store {
116   private static final String MEMSTORE_CLASS_NAME = "hbase.regionserver.memstore.class";
117   public static final String COMPACTCHECKER_INTERVAL_MULTIPLIER_KEY =
118       "hbase.server.compactchecker.interval.multiplier";
119   public static final String BLOCKING_STOREFILES_KEY = "hbase.hstore.blockingStoreFiles";
120   public static final int DEFAULT_COMPACTCHECKER_INTERVAL_MULTIPLIER = 1000;
121   public static final int DEFAULT_BLOCKING_STOREFILE_COUNT = 7;
122 
123   private static final Log LOG = LogFactory.getLog(HStore.class);
124 
125   protected final MemStore memstore;
126   // This stores directory in the filesystem.
127   protected final HRegion region;
128   private final HColumnDescriptor family;
129   private final HRegionFileSystem fs;
130   protected Configuration conf;
131   protected CacheConfig cacheConf;
132   private long lastCompactSize = 0;
133   volatile boolean forceMajor = false;
134   /* how many bytes to write between status checks */
135   static int closeCheckInterval = 0;
136   private volatile long storeSize = 0L;
137   private volatile long totalUncompressedBytes = 0L;
138 
139   /**
140    * RWLock for store operations.
141    * Locked in shared mode when the list of component stores is looked at:
142    *   - all reads/writes to table data
143    *   - checking for split
144    * Locked in exclusive mode when the list of component stores is modified:
145    *   - closing
146    *   - completing a compaction
147    */
148   final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
149   private final boolean verifyBulkLoads;
150 
151   private ScanInfo scanInfo;
152 
153   // TODO: ideally, this should be part of storeFileManager, as we keep passing this to it.
154   final List<StoreFile> filesCompacting = Lists.newArrayList();
155 
156   // All access must be synchronized.
157   private final Set<ChangedReadersObserver> changedReaderObservers =
158     Collections.newSetFromMap(new ConcurrentHashMap<ChangedReadersObserver, Boolean>());
159 
160   protected final int blocksize;
161   private HFileDataBlockEncoder dataBlockEncoder;
162 
163   /** Checksum configuration */
164   protected ChecksumType checksumType;
165   protected int bytesPerChecksum;
166 
167   // Comparing KeyValues
168   private final CellComparator comparator;
169 
170   final StoreEngine<?, ?, ?, ?> storeEngine;
171 
172   private static final AtomicBoolean offPeakCompactionTracker = new AtomicBoolean();
173   private volatile OffPeakHours offPeakHours;
174 
175   private static final int DEFAULT_FLUSH_RETRIES_NUMBER = 10;
176   private int flushRetriesNumber;
177   private int pauseTime;
178 
179   private long blockingFileCount;
180   private int compactionCheckMultiplier;
181   protected Encryption.Context cryptoContext = Encryption.Context.NONE;
182 
183   private volatile long flushedCellsCount = 0;
184   private volatile long compactedCellsCount = 0;
185   private volatile long majorCompactedCellsCount = 0;
186   private volatile long flushedCellsSize = 0;
187   private volatile long compactedCellsSize = 0;
188   private volatile long majorCompactedCellsSize = 0;
189 
190   /**
191    * Constructor
192    * @param region
193    * @param family HColumnDescriptor for this column
194    * @param confParam configuration object
195    * failed.  Can be null.
196    * @throws IOException
197    */
198   protected HStore(final HRegion region, final HColumnDescriptor family,
199       final Configuration confParam) throws IOException {
200 
201     this.fs = region.getRegionFileSystem();
202 
203     // Assemble the store's home directory and Ensure it exists.
204     fs.createStoreDir(family.getNameAsString());
205     this.region = region;
206     this.family = family;
207     // 'conf' renamed to 'confParam' b/c we use this.conf in the constructor
208     // CompoundConfiguration will look for keys in reverse order of addition, so we'd
209     // add global config first, then table and cf overrides, then cf metadata.
210     this.conf = new CompoundConfiguration()
211       .add(confParam)
212       .addStringMap(region.getTableDesc().getConfiguration())
213       .addStringMap(family.getConfiguration())
214       .addBytesMap(family.getValues());
215     this.blocksize = family.getBlocksize();
216 
217     this.dataBlockEncoder =
218         new HFileDataBlockEncoderImpl(family.getDataBlockEncoding());
219 
220     this.comparator = region.getCellCompartor();
221     // used by ScanQueryMatcher
222     long timeToPurgeDeletes =
223         Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0);
224     LOG.trace("Time to purge deletes set to " + timeToPurgeDeletes +
225         "ms in store " + this);
226     // Get TTL
227     long ttl = determineTTLFromFamily(family);
228     // Why not just pass a HColumnDescriptor in here altogether?  Even if have
229     // to clone it?
230     scanInfo = new ScanInfo(conf, family, ttl, timeToPurgeDeletes, this.comparator);
231     String className = conf.get(MEMSTORE_CLASS_NAME, DefaultMemStore.class.getName());
232     this.memstore = ReflectionUtils.instantiateWithCustomCtor(className, new Class[] {
233         Configuration.class, CellComparator.class }, new Object[] { conf, this.comparator });
234     this.offPeakHours = OffPeakHours.getInstance(conf);
235 
236     // Setting up cache configuration for this family
237     createCacheConf(family);
238 
239     this.verifyBulkLoads = conf.getBoolean("hbase.hstore.bulkload.verify", false);
240 
241     this.blockingFileCount =
242         conf.getInt(BLOCKING_STOREFILES_KEY, DEFAULT_BLOCKING_STOREFILE_COUNT);
243     this.compactionCheckMultiplier = conf.getInt(
244         COMPACTCHECKER_INTERVAL_MULTIPLIER_KEY, DEFAULT_COMPACTCHECKER_INTERVAL_MULTIPLIER);
245     if (this.compactionCheckMultiplier <= 0) {
246       LOG.error("Compaction check period multiplier must be positive, setting default: "
247           + DEFAULT_COMPACTCHECKER_INTERVAL_MULTIPLIER);
248       this.compactionCheckMultiplier = DEFAULT_COMPACTCHECKER_INTERVAL_MULTIPLIER;
249     }
250 
251     if (HStore.closeCheckInterval == 0) {
252       HStore.closeCheckInterval = conf.getInt(
253           "hbase.hstore.close.check.interval", 10*1000*1000 /* 10 MB */);
254     }
255 
256     this.storeEngine = createStoreEngine(this, this.conf, this.comparator);
257     this.storeEngine.getStoreFileManager().loadFiles(loadStoreFiles());
258 
259     // Initialize checksum type from name. The names are CRC32, CRC32C, etc.
260     this.checksumType = getChecksumType(conf);
261     // initilize bytes per checksum
262     this.bytesPerChecksum = getBytesPerChecksum(conf);
263     flushRetriesNumber = conf.getInt(
264         "hbase.hstore.flush.retries.number", DEFAULT_FLUSH_RETRIES_NUMBER);
265     pauseTime = conf.getInt(HConstants.HBASE_SERVER_PAUSE, HConstants.DEFAULT_HBASE_SERVER_PAUSE);
266     if (flushRetriesNumber <= 0) {
267       throw new IllegalArgumentException(
268           "hbase.hstore.flush.retries.number must be > 0, not "
269               + flushRetriesNumber);
270     }
271     cryptoContext = EncryptionUtil.createEncryptionContext(conf, family);
272   }
273 
274   /**
275    * Creates the cache config.
276    * @param family The current column family.
277    */
278   protected void createCacheConf(final HColumnDescriptor family) {
279     this.cacheConf = new CacheConfig(conf, family);
280   }
281 
282   /**
283    * Creates the store engine configured for the given Store.
284    * @param store The store. An unfortunate dependency needed due to it
285    *              being passed to coprocessors via the compactor.
286    * @param conf Store configuration.
287    * @param kvComparator KVComparator for storeFileManager.
288    * @return StoreEngine to use.
289    */
290   protected StoreEngine<?, ?, ?, ?> createStoreEngine(Store store, Configuration conf,
291       CellComparator kvComparator) throws IOException {
292     return StoreEngine.create(store, conf, comparator);
293   }
294 
295   /**
296    * @param family
297    * @return TTL in seconds of the specified family
298    */
299   public static long determineTTLFromFamily(final HColumnDescriptor family) {
300     // HCD.getTimeToLive returns ttl in seconds.  Convert to milliseconds.
301     long ttl = family.getTimeToLive();
302     if (ttl == HConstants.FOREVER) {
303       // Default is unlimited ttl.
304       ttl = Long.MAX_VALUE;
305     } else if (ttl == -1) {
306       ttl = Long.MAX_VALUE;
307     } else {
308       // Second -> ms adjust for user data
309       ttl *= 1000;
310     }
311     return ttl;
312   }
313 
314   @Override
315   public String getColumnFamilyName() {
316     return this.family.getNameAsString();
317   }
318 
319   @Override
320   public TableName getTableName() {
321     return this.getRegionInfo().getTable();
322   }
323 
324   @Override
325   public FileSystem getFileSystem() {
326     return this.fs.getFileSystem();
327   }
328 
329   public HRegionFileSystem getRegionFileSystem() {
330     return this.fs;
331   }
332 
333   /* Implementation of StoreConfigInformation */
334   @Override
335   public long getStoreFileTtl() {
336     // TTL only applies if there's no MIN_VERSIONs setting on the column.
337     return (this.scanInfo.getMinVersions() == 0) ? this.scanInfo.getTtl() : Long.MAX_VALUE;
338   }
339 
340   @Override
341   public long getMemstoreFlushSize() {
342     // TODO: Why is this in here?  The flushsize of the region rather than the store?  St.Ack
343     return this.region.memstoreFlushSize;
344   }
345 
346   @Override
347   public long getFlushableSize() {
348     return this.memstore.getFlushableSize();
349   }
350 
351   @Override
352   public long getSnapshotSize() {
353     return this.memstore.getSnapshotSize();
354   }
355 
356   @Override
357   public long getCompactionCheckMultiplier() {
358     return this.compactionCheckMultiplier;
359   }
360 
361   @Override
362   public long getBlockingFileCount() {
363     return blockingFileCount;
364   }
365   /* End implementation of StoreConfigInformation */
366 
367   /**
368    * Returns the configured bytesPerChecksum value.
369    * @param conf The configuration
370    * @return The bytesPerChecksum that is set in the configuration
371    */
372   public static int getBytesPerChecksum(Configuration conf) {
373     return conf.getInt(HConstants.BYTES_PER_CHECKSUM,
374                        HFile.DEFAULT_BYTES_PER_CHECKSUM);
375   }
376 
377   /**
378    * Returns the configured checksum algorithm.
379    * @param conf The configuration
380    * @return The checksum algorithm that is set in the configuration
381    */
382   public static ChecksumType getChecksumType(Configuration conf) {
383     String checksumName = conf.get(HConstants.CHECKSUM_TYPE_NAME);
384     if (checksumName == null) {
385       return ChecksumType.getDefaultChecksumType();
386     } else {
387       return ChecksumType.nameToType(checksumName);
388     }
389   }
390 
391   /**
392    * @return how many bytes to write between status checks
393    */
394   public static int getCloseCheckInterval() {
395     return closeCheckInterval;
396   }
397 
398   @Override
399   public HColumnDescriptor getFamily() {
400     return this.family;
401   }
402 
403   /**
404    * @return The maximum sequence id in all store files. Used for log replay.
405    */
406   @Override
407   public long getMaxSequenceId() {
408     return StoreFile.getMaxSequenceIdInList(this.getStorefiles());
409   }
410 
411   @Override
412   public long getMaxMemstoreTS() {
413     return StoreFile.getMaxMemstoreTSInList(this.getStorefiles());
414   }
415 
416   /**
417    * @param tabledir {@link Path} to where the table is being stored
418    * @param hri {@link HRegionInfo} for the region.
419    * @param family {@link HColumnDescriptor} describing the column family
420    * @return Path to family/Store home directory.
421    */
422   @Deprecated
423   public static Path getStoreHomedir(final Path tabledir,
424       final HRegionInfo hri, final byte[] family) {
425     return getStoreHomedir(tabledir, hri.getEncodedName(), family);
426   }
427 
428   /**
429    * @param tabledir {@link Path} to where the table is being stored
430    * @param encodedName Encoded region name.
431    * @param family {@link HColumnDescriptor} describing the column family
432    * @return Path to family/Store home directory.
433    */
434   @Deprecated
435   public static Path getStoreHomedir(final Path tabledir,
436       final String encodedName, final byte[] family) {
437     return new Path(tabledir, new Path(encodedName, Bytes.toString(family)));
438   }
439 
440   @Override
441   public HFileDataBlockEncoder getDataBlockEncoder() {
442     return dataBlockEncoder;
443   }
444 
445   /**
446    * Should be used only in tests.
447    * @param blockEncoder the block delta encoder to use
448    */
449   void setDataBlockEncoderInTest(HFileDataBlockEncoder blockEncoder) {
450     this.dataBlockEncoder = blockEncoder;
451   }
452 
453   /**
454    * Creates an unsorted list of StoreFile loaded in parallel
455    * from the given directory.
456    * @throws IOException
457    */
458   private List<StoreFile> loadStoreFiles() throws IOException {
459     Collection<StoreFileInfo> files = fs.getStoreFiles(getColumnFamilyName());
460     return openStoreFiles(files);
461   }
462 
463   private List<StoreFile> openStoreFiles(Collection<StoreFileInfo> files) throws IOException {
464     if (files == null || files.size() == 0) {
465       return new ArrayList<StoreFile>();
466     }
467     // initialize the thread pool for opening store files in parallel..
468     ThreadPoolExecutor storeFileOpenerThreadPool =
469       this.region.getStoreFileOpenAndCloseThreadPool("StoreFileOpenerThread-" +
470           this.getColumnFamilyName());
471     CompletionService<StoreFile> completionService =
472       new ExecutorCompletionService<StoreFile>(storeFileOpenerThreadPool);
473 
474     int totalValidStoreFile = 0;
475     for (final StoreFileInfo storeFileInfo: files) {
476       // open each store file in parallel
477       completionService.submit(new Callable<StoreFile>() {
478         @Override
479         public StoreFile call() throws IOException {
480           StoreFile storeFile = createStoreFileAndReader(storeFileInfo);
481           return storeFile;
482         }
483       });
484       totalValidStoreFile++;
485     }
486 
487     ArrayList<StoreFile> results = new ArrayList<StoreFile>(files.size());
488     IOException ioe = null;
489     try {
490       for (int i = 0; i < totalValidStoreFile; i++) {
491         try {
492           Future<StoreFile> future = completionService.take();
493           StoreFile storeFile = future.get();
494           if (storeFile != null) {
495             long length = storeFile.getReader().length();
496             this.storeSize += length;
497             this.totalUncompressedBytes += storeFile.getReader().getTotalUncompressedBytes();
498             if (LOG.isDebugEnabled()) {
499               LOG.debug("loaded " + storeFile.toStringDetailed());
500             }
501             results.add(storeFile);
502           }
503         } catch (InterruptedException e) {
504           if (ioe == null) ioe = new InterruptedIOException(e.getMessage());
505         } catch (ExecutionException e) {
506           if (ioe == null) ioe = new IOException(e.getCause());
507         }
508       }
509     } finally {
510       storeFileOpenerThreadPool.shutdownNow();
511     }
512     if (ioe != null) {
513       // close StoreFile readers
514       boolean evictOnClose = 
515           cacheConf != null? cacheConf.shouldEvictOnClose(): true; 
516       for (StoreFile file : results) {
517         try {
518           if (file != null) file.closeReader(evictOnClose);
519         } catch (IOException e) {
520           LOG.warn(e.getMessage());
521         }
522       }
523       throw ioe;
524     }
525 
526     return results;
527   }
528 
529   /**
530    * Checks the underlying store files, and opens the files that  have not
531    * been opened, and removes the store file readers for store files no longer
532    * available. Mainly used by secondary region replicas to keep up to date with
533    * the primary region files.
534    * @throws IOException
535    */
536   @Override
537   public void refreshStoreFiles() throws IOException {
538     Collection<StoreFileInfo> newFiles = fs.getStoreFiles(getColumnFamilyName());
539     refreshStoreFilesInternal(newFiles);
540   }
541 
542   @Override
543   public void refreshStoreFiles(Collection<String> newFiles) throws IOException {
544     List<StoreFileInfo> storeFiles = new ArrayList<StoreFileInfo>(newFiles.size());
545     for (String file : newFiles) {
546       storeFiles.add(fs.getStoreFileInfo(getColumnFamilyName(), file));
547     }
548     refreshStoreFilesInternal(storeFiles);
549   }
550 
551   /**
552    * Checks the underlying store files, and opens the files that  have not
553    * been opened, and removes the store file readers for store files no longer
554    * available. Mainly used by secondary region replicas to keep up to date with
555    * the primary region files.
556    * @throws IOException
557    */
558   private void refreshStoreFilesInternal(Collection<StoreFileInfo> newFiles) throws IOException {
559     StoreFileManager sfm = storeEngine.getStoreFileManager();
560     Collection<StoreFile> currentFiles = sfm.getStorefiles();
561     if (currentFiles == null) currentFiles = new ArrayList<StoreFile>(0);
562 
563     if (newFiles == null) newFiles = new ArrayList<StoreFileInfo>(0);
564 
565     HashMap<StoreFileInfo, StoreFile> currentFilesSet =
566         new HashMap<StoreFileInfo, StoreFile>(currentFiles.size());
567     for (StoreFile sf : currentFiles) {
568       currentFilesSet.put(sf.getFileInfo(), sf);
569     }
570     HashSet<StoreFileInfo> newFilesSet = new HashSet<StoreFileInfo>(newFiles);
571 
572     Set<StoreFileInfo> toBeAddedFiles = Sets.difference(newFilesSet, currentFilesSet.keySet());
573     Set<StoreFileInfo> toBeRemovedFiles = Sets.difference(currentFilesSet.keySet(), newFilesSet);
574 
575     if (toBeAddedFiles.isEmpty() && toBeRemovedFiles.isEmpty()) {
576       return;
577     }
578 
579     LOG.info("Refreshing store files for region " + this.getRegionInfo().getRegionNameAsString()
580       + " files to add: " + toBeAddedFiles + " files to remove: " + toBeRemovedFiles);
581 
582     Set<StoreFile> toBeRemovedStoreFiles = new HashSet<StoreFile>(toBeRemovedFiles.size());
583     for (StoreFileInfo sfi : toBeRemovedFiles) {
584       toBeRemovedStoreFiles.add(currentFilesSet.get(sfi));
585     }
586 
587     // try to open the files
588     List<StoreFile> openedFiles = openStoreFiles(toBeAddedFiles);
589 
590     // propogate the file changes to the underlying store file manager
591     replaceStoreFiles(toBeRemovedStoreFiles, openedFiles); //won't throw an exception
592 
593     // Advance the memstore read point to be at least the new store files seqIds so that
594     // readers might pick it up. This assumes that the store is not getting any writes (otherwise
595     // in-flight transactions might be made visible)
596     if (!toBeAddedFiles.isEmpty()) {
597       region.getMVCC().advanceTo(this.getMaxSequenceId());
598     }
599 
600     completeCompaction(toBeRemovedStoreFiles);
601   }
602 
603   private StoreFile createStoreFileAndReader(final Path p) throws IOException {
604     StoreFileInfo info = new StoreFileInfo(conf, this.getFileSystem(), p);
605     return createStoreFileAndReader(info);
606   }
607 
608   private StoreFile createStoreFileAndReader(final StoreFileInfo info)
609       throws IOException {
610     info.setRegionCoprocessorHost(this.region.getCoprocessorHost());
611     StoreFile storeFile = new StoreFile(this.getFileSystem(), info, this.conf, this.cacheConf,
612       this.family.getBloomFilterType());
613     StoreFile.Reader r = storeFile.createReader();
614     r.setReplicaStoreFile(isPrimaryReplicaStore());
615     return storeFile;
616   }
617 
618   @Override
619   public long add(final Cell cell) {
620     lock.readLock().lock();
621     try {
622        return this.memstore.add(cell);
623     } finally {
624       lock.readLock().unlock();
625     }
626   }
627 
628   @Override
629   public long timeOfOldestEdit() {
630     return memstore.timeOfOldestEdit();
631   }
632 
633   /**
634    * Adds a value to the memstore
635    *
636    * @param kv
637    * @return memstore size delta
638    */
639   protected long delete(final KeyValue kv) {
640     lock.readLock().lock();
641     try {
642       return this.memstore.delete(kv);
643     } finally {
644       lock.readLock().unlock();
645     }
646   }
647 
648   @Override
649   public void rollback(final Cell cell) {
650     lock.readLock().lock();
651     try {
652       this.memstore.rollback(cell);
653     } finally {
654       lock.readLock().unlock();
655     }
656   }
657 
658   /**
659    * @return All store files.
660    */
661   @Override
662   public Collection<StoreFile> getStorefiles() {
663     return this.storeEngine.getStoreFileManager().getStorefiles();
664   }
665 
666   @Override
667   public void assertBulkLoadHFileOk(Path srcPath) throws IOException {
668     HFile.Reader reader  = null;
669     try {
670       LOG.info("Validating hfile at " + srcPath + " for inclusion in "
671           + "store " + this + " region " + this.getRegionInfo().getRegionNameAsString());
672       reader = HFile.createReader(srcPath.getFileSystem(conf),
673           srcPath, cacheConf, conf);
674       reader.loadFileInfo();
675 
676       byte[] firstKey = reader.getFirstRowKey();
677       Preconditions.checkState(firstKey != null, "First key can not be null");
678       Cell lk = reader.getLastKey();
679       Preconditions.checkState(lk != null, "Last key can not be null");
680       byte[] lastKey =  CellUtil.cloneRow(lk);
681 
682       LOG.debug("HFile bounds: first=" + Bytes.toStringBinary(firstKey) +
683           " last=" + Bytes.toStringBinary(lastKey));
684       LOG.debug("Region bounds: first=" +
685           Bytes.toStringBinary(getRegionInfo().getStartKey()) +
686           " last=" + Bytes.toStringBinary(getRegionInfo().getEndKey()));
687 
688       if (!this.getRegionInfo().containsRange(firstKey, lastKey)) {
689         throw new WrongRegionException(
690             "Bulk load file " + srcPath.toString() + " does not fit inside region "
691             + this.getRegionInfo().getRegionNameAsString());
692       }
693 
694       if(reader.length() > conf.getLong(HConstants.HREGION_MAX_FILESIZE,
695           HConstants.DEFAULT_MAX_FILE_SIZE)) {
696         LOG.warn("Trying to bulk load hfile " + srcPath.toString() + " with size: " +
697             reader.length() + " bytes can be problematic as it may lead to oversplitting.");
698       }
699 
700       if (verifyBulkLoads) {
701         long verificationStartTime = EnvironmentEdgeManager.currentTime();
702         LOG.info("Full verification started for bulk load hfile: " + srcPath.toString());
703         Cell prevCell = null;
704         HFileScanner scanner = reader.getScanner(false, false, false);
705         scanner.seekTo();
706         do {
707           Cell cell = scanner.getCell();
708           if (prevCell != null) {
709             if (comparator.compareRows(prevCell, cell) > 0) {
710               throw new InvalidHFileException("Previous row is greater than"
711                   + " current row: path=" + srcPath + " previous="
712                   + CellUtil.getCellKeyAsString(prevCell) + " current="
713                   + CellUtil.getCellKeyAsString(cell));
714             }
715             if (CellComparator.compareFamilies(prevCell, cell) != 0) {
716               throw new InvalidHFileException("Previous key had different"
717                   + " family compared to current key: path=" + srcPath
718                   + " previous="
719                   + Bytes.toStringBinary(prevCell.getFamilyArray(), prevCell.getFamilyOffset(),
720                       prevCell.getFamilyLength())
721                   + " current="
722                   + Bytes.toStringBinary(cell.getFamilyArray(), cell.getFamilyOffset(),
723                       cell.getFamilyLength()));
724             }
725           }
726           prevCell = cell;
727         } while (scanner.next());
728       LOG.info("Full verification complete for bulk load hfile: " + srcPath.toString()
729          + " took " + (EnvironmentEdgeManager.currentTime() - verificationStartTime)
730          + " ms");
731       }
732     } finally {
733       if (reader != null) reader.close();
734     }
735   }
736 
737   @Override
738   public Path bulkLoadHFile(String srcPathStr, long seqNum) throws IOException {
739     Path srcPath = new Path(srcPathStr);
740     Path dstPath = fs.bulkLoadStoreFile(getColumnFamilyName(), srcPath, seqNum);
741 
742     LOG.info("Loaded HFile " + srcPath + " into store '" + getColumnFamilyName() + "' as "
743         + dstPath + " - updating store file list.");
744 
745     StoreFile sf = createStoreFileAndReader(dstPath);
746     bulkLoadHFile(sf);
747 
748     LOG.info("Successfully loaded store file " + srcPath + " into store " + this
749         + " (new location: " + dstPath + ")");
750 
751     return dstPath;
752   }
753 
754   @Override
755   public void bulkLoadHFile(StoreFileInfo fileInfo) throws IOException {
756     StoreFile sf = createStoreFileAndReader(fileInfo);
757     bulkLoadHFile(sf);
758   }
759 
760   private void bulkLoadHFile(StoreFile sf) throws IOException {
761     StoreFile.Reader r = sf.getReader();
762     this.storeSize += r.length();
763     this.totalUncompressedBytes += r.getTotalUncompressedBytes();
764 
765     // Append the new storefile into the list
766     this.lock.writeLock().lock();
767     try {
768       this.storeEngine.getStoreFileManager().insertNewFiles(Lists.newArrayList(sf));
769     } finally {
770       // We need the lock, as long as we are updating the storeFiles
771       // or changing the memstore. Let us release it before calling
772       // notifyChangeReadersObservers. See HBASE-4485 for a possible
773       // deadlock scenario that could have happened if continue to hold
774       // the lock.
775       this.lock.writeLock().unlock();
776     }
777     LOG.info("Loaded HFile " + sf.getFileInfo() + " into store '" + getColumnFamilyName());
778     if (LOG.isTraceEnabled()) {
779       String traceMessage = "BULK LOAD time,size,store size,store files ["
780           + EnvironmentEdgeManager.currentTime() + "," + r.length() + "," + storeSize
781           + "," + storeEngine.getStoreFileManager().getStorefileCount() + "]";
782       LOG.trace(traceMessage);
783     }
784   }
785 
786   @Override
787   public ImmutableCollection<StoreFile> close() throws IOException {
788     this.lock.writeLock().lock();
789     try {
790       // Clear so metrics doesn't find them.
791       ImmutableCollection<StoreFile> result = storeEngine.getStoreFileManager().clearFiles();
792       Collection<StoreFile> compactedfiles =
793           storeEngine.getStoreFileManager().clearCompactedFiles();
794       // clear the compacted files
795       if (compactedfiles != null && !compactedfiles.isEmpty()) {
796         removeCompactedfiles(compactedfiles);
797       }
798       if (!result.isEmpty()) {
799         // initialize the thread pool for closing store files in parallel.
800         ThreadPoolExecutor storeFileCloserThreadPool = this.region
801             .getStoreFileOpenAndCloseThreadPool("StoreFileCloserThread-"
802                 + this.getColumnFamilyName());
803 
804         // close each store file in parallel
805         CompletionService<Void> completionService =
806           new ExecutorCompletionService<Void>(storeFileCloserThreadPool);
807         for (final StoreFile f : result) {
808           completionService.submit(new Callable<Void>() {
809             @Override
810             public Void call() throws IOException {
811               boolean evictOnClose = 
812                   cacheConf != null? cacheConf.shouldEvictOnClose(): true;
813               f.closeReader(evictOnClose);
814               return null;
815             }
816           });
817         }
818 
819         IOException ioe = null;
820         try {
821           for (int i = 0; i < result.size(); i++) {
822             try {
823               Future<Void> future = completionService.take();
824               future.get();
825             } catch (InterruptedException e) {
826               if (ioe == null) {
827                 ioe = new InterruptedIOException();
828                 ioe.initCause(e);
829               }
830             } catch (ExecutionException e) {
831               if (ioe == null) ioe = new IOException(e.getCause());
832             }
833           }
834         } finally {
835           storeFileCloserThreadPool.shutdownNow();
836         }
837         if (ioe != null) throw ioe;
838       }
839       LOG.info("Closed " + this);
840       return result;
841     } finally {
842       this.lock.writeLock().unlock();
843     }
844   }
845 
846   /**
847    * Snapshot this stores memstore. Call before running
848    * {@link #flushCache(long, MemStoreSnapshot, MonitoredTask, ThroughputController)}
849    *  so it has some work to do.
850    */
851   void snapshot() {
852     this.lock.writeLock().lock();
853     try {
854       this.memstore.snapshot();
855     } finally {
856       this.lock.writeLock().unlock();
857     }
858   }
859 
860   /**
861    * Write out current snapshot. Presumes {@link #snapshot()} has been called previously.
862    * @param logCacheFlushId flush sequence number
863    * @param snapshot
864    * @param status
865    * @param throughputController
866    * @return The path name of the tmp file to which the store was flushed
867    * @throws IOException if exception occurs during process
868    */
869   protected List<Path> flushCache(final long logCacheFlushId, MemStoreSnapshot snapshot,
870       MonitoredTask status, ThroughputController throughputController) throws IOException {
871     // If an exception happens flushing, we let it out without clearing
872     // the memstore snapshot.  The old snapshot will be returned when we say
873     // 'snapshot', the next time flush comes around.
874     // Retry after catching exception when flushing, otherwise server will abort
875     // itself
876     StoreFlusher flusher = storeEngine.getStoreFlusher();
877     IOException lastException = null;
878     for (int i = 0; i < flushRetriesNumber; i++) {
879       try {
880         List<Path> pathNames =
881             flusher.flushSnapshot(snapshot, logCacheFlushId, status, throughputController);
882         Path lastPathName = null;
883         try {
884           for (Path pathName : pathNames) {
885             lastPathName = pathName;
886             validateStoreFile(pathName);
887           }
888           return pathNames;
889         } catch (Exception e) {
890           LOG.warn("Failed validating store file " + lastPathName + ", retrying num=" + i, e);
891           if (e instanceof IOException) {
892             lastException = (IOException) e;
893           } else {
894             lastException = new IOException(e);
895           }
896         }
897       } catch (IOException e) {
898         LOG.warn("Failed flushing store file, retrying num=" + i, e);
899         lastException = e;
900       }
901       if (lastException != null && i < (flushRetriesNumber - 1)) {
902         try {
903           Thread.sleep(pauseTime);
904         } catch (InterruptedException e) {
905           IOException iie = new InterruptedIOException();
906           iie.initCause(e);
907           throw iie;
908         }
909       }
910     }
911     throw lastException;
912   }
913 
914   /*
915    * @param path The pathname of the tmp file into which the store was flushed
916    * @param logCacheFlushId
917    * @param status
918    * @return StoreFile created.
919    * @throws IOException
920    */
921   private StoreFile commitFile(final Path path, final long logCacheFlushId, MonitoredTask status)
922       throws IOException {
923     // Write-out finished successfully, move into the right spot
924     Path dstPath = fs.commitStoreFile(getColumnFamilyName(), path);
925 
926     status.setStatus("Flushing " + this + ": reopening flushed file");
927     StoreFile sf = createStoreFileAndReader(dstPath);
928 
929     StoreFile.Reader r = sf.getReader();
930     this.storeSize += r.length();
931     this.totalUncompressedBytes += r.getTotalUncompressedBytes();
932 
933     if (LOG.isInfoEnabled()) {
934       LOG.info("Added " + sf + ", entries=" + r.getEntries() +
935         ", sequenceid=" + logCacheFlushId +
936         ", filesize=" + TraditionalBinaryPrefix.long2String(r.length(), "", 1));
937     }
938     return sf;
939   }
940 
941   @Override
942   public StoreFile.Writer createWriterInTmp(long maxKeyCount, Compression.Algorithm compression,
943                                             boolean isCompaction, boolean includeMVCCReadpoint,
944                                             boolean includesTag)
945       throws IOException {
946     return createWriterInTmp(maxKeyCount, compression, isCompaction, includeMVCCReadpoint,
947         includesTag, false);
948   }
949 
950   /*
951    * @param maxKeyCount
952    * @param compression Compression algorithm to use
953    * @param isCompaction whether we are creating a new file in a compaction
954    * @param includesMVCCReadPoint - whether to include MVCC or not
955    * @param includesTag - includesTag or not
956    * @return Writer for a new StoreFile in the tmp dir.
957    */
958   @Override
959   public StoreFile.Writer createWriterInTmp(long maxKeyCount, Compression.Algorithm compression,
960       boolean isCompaction, boolean includeMVCCReadpoint, boolean includesTag,
961       boolean shouldDropBehind)
962   throws IOException {
963     final CacheConfig writerCacheConf;
964     if (isCompaction) {
965       // Don't cache data on write on compactions.
966       writerCacheConf = new CacheConfig(cacheConf);
967       writerCacheConf.setCacheDataOnWrite(false);
968     } else {
969       writerCacheConf = cacheConf;
970     }
971     InetSocketAddress[] favoredNodes = null;
972     if (region.getRegionServerServices() != null) {
973       favoredNodes = region.getRegionServerServices().getFavoredNodesForRegion(
974           region.getRegionInfo().getEncodedName());
975     }
976     HFileContext hFileContext = createFileContext(compression, includeMVCCReadpoint, includesTag,
977       cryptoContext);
978     StoreFile.Writer w = new StoreFile.WriterBuilder(conf, writerCacheConf,
979         this.getFileSystem())
980             .withFilePath(fs.createTempName())
981             .withComparator(comparator)
982             .withBloomType(family.getBloomFilterType())
983             .withMaxKeyCount(maxKeyCount)
984             .withFavoredNodes(favoredNodes)
985             .withFileContext(hFileContext)
986             .withShouldDropCacheBehind(shouldDropBehind)
987             .build();
988     return w;
989   }
990 
991   private HFileContext createFileContext(Compression.Algorithm compression,
992       boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context cryptoContext) {
993     if (compression == null) {
994       compression = HFile.DEFAULT_COMPRESSION_ALGORITHM;
995     }
996     HFileContext hFileContext = new HFileContextBuilder()
997                                 .withIncludesMvcc(includeMVCCReadpoint)
998                                 .withIncludesTags(includesTag)
999                                 .withCompression(compression)
1000                                 .withCompressTags(family.isCompressTags())
1001                                 .withChecksumType(checksumType)
1002                                 .withBytesPerCheckSum(bytesPerChecksum)
1003                                 .withBlockSize(blocksize)
1004                                 .withHBaseCheckSum(true)
1005                                 .withDataBlockEncoding(family.getDataBlockEncoding())
1006                                 .withEncryptionContext(cryptoContext)
1007                                 .withCreateTime(EnvironmentEdgeManager.currentTime())
1008                                 .build();
1009     return hFileContext;
1010   }
1011 
1012 
1013   /*
1014    * Change storeFiles adding into place the Reader produced by this new flush.
1015    * @param sfs Store files
1016    * @param snapshotId
1017    * @throws IOException
1018    * @return Whether compaction is required.
1019    */
1020   private boolean updateStorefiles(final List<StoreFile> sfs, final long snapshotId)
1021       throws IOException {
1022     this.lock.writeLock().lock();
1023     try {
1024       this.storeEngine.getStoreFileManager().insertNewFiles(sfs);
1025       if (snapshotId > 0) {
1026         this.memstore.clearSnapshot(snapshotId);
1027       }
1028     } finally {
1029       // We need the lock, as long as we are updating the storeFiles
1030       // or changing the memstore. Let us release it before calling
1031       // notifyChangeReadersObservers. See HBASE-4485 for a possible
1032       // deadlock scenario that could have happened if continue to hold
1033       // the lock.
1034       this.lock.writeLock().unlock();
1035     }
1036     // notify to be called here - only in case of flushes
1037     notifyChangedReadersObservers(sfs);
1038     if (LOG.isTraceEnabled()) {
1039       long totalSize = 0;
1040       for (StoreFile sf : sfs) {
1041         totalSize += sf.getReader().length();
1042       }
1043       String traceMessage = "FLUSH time,count,size,store size,store files ["
1044           + EnvironmentEdgeManager.currentTime() + "," + sfs.size() + "," + totalSize
1045           + "," + storeSize + "," + storeEngine.getStoreFileManager().getStorefileCount() + "]";
1046       LOG.trace(traceMessage);
1047     }
1048     return needsCompaction();
1049   }
1050 
1051   /*
1052    * Notify all observers that set of Readers has changed.
1053    * @throws IOException
1054    */
1055   private void notifyChangedReadersObservers(List<StoreFile> sfs) throws IOException {
1056     for (ChangedReadersObserver o : this.changedReaderObservers) {
1057       o.updateReaders(sfs);
1058     }
1059   }
1060 
1061   /**
1062    * Get all scanners with no filtering based on TTL (that happens further down
1063    * the line).
1064    * @return all scanners for this store
1065    */
1066   @Override
1067   public List<KeyValueScanner> getScanners(boolean cacheBlocks, boolean isGet,
1068       boolean usePread, boolean isCompaction, ScanQueryMatcher matcher, byte[] startRow,
1069       byte[] stopRow, long readPt) throws IOException {
1070     Collection<StoreFile> storeFilesToScan;
1071     List<KeyValueScanner> memStoreScanners;
1072     this.lock.readLock().lock();
1073     try {
1074       storeFilesToScan =
1075           this.storeEngine.getStoreFileManager().getFilesForScanOrGet(isGet, startRow, stopRow);
1076       memStoreScanners = this.memstore.getScanners(readPt);
1077     } finally {
1078       this.lock.readLock().unlock();
1079     }
1080 
1081     // First the store file scanners
1082 
1083     // TODO this used to get the store files in descending order,
1084     // but now we get them in ascending order, which I think is
1085     // actually more correct, since memstore get put at the end.
1086     List<StoreFileScanner> sfScanners = StoreFileScanner.getScannersForStoreFiles(storeFilesToScan,
1087         cacheBlocks, usePread, isCompaction, false, matcher, readPt, isPrimaryReplicaStore());
1088     List<KeyValueScanner> scanners =
1089       new ArrayList<KeyValueScanner>(sfScanners.size()+1);
1090     scanners.addAll(sfScanners);
1091     // Then the memstore scanners
1092     scanners.addAll(memStoreScanners);
1093     return scanners;
1094   }
1095 
1096   @Override
1097   public List<KeyValueScanner> getScanners(List<StoreFile> files, boolean cacheBlocks,
1098       boolean isGet, boolean usePread, boolean isCompaction, ScanQueryMatcher matcher,
1099       byte[] startRow, byte[] stopRow, long readPt, boolean includeMemstoreScanner) throws IOException {
1100     List<KeyValueScanner> memStoreScanners = null;
1101     if (includeMemstoreScanner) {
1102       this.lock.readLock().lock();
1103       try {
1104         memStoreScanners = this.memstore.getScanners(readPt);
1105       } finally {
1106         this.lock.readLock().unlock();
1107       }
1108     }
1109     List<StoreFileScanner> sfScanners = StoreFileScanner.getScannersForStoreFiles(files,
1110       cacheBlocks, usePread, isCompaction, false, matcher, readPt, isPrimaryReplicaStore());
1111     List<KeyValueScanner> scanners = new ArrayList<KeyValueScanner>(sfScanners.size() + 1);
1112     scanners.addAll(sfScanners);
1113     // Then the memstore scanners
1114     if (memStoreScanners != null) {
1115       scanners.addAll(memStoreScanners);
1116     }
1117     return scanners;
1118   }
1119 
1120   @Override
1121   public void addChangedReaderObserver(ChangedReadersObserver o) {
1122     this.changedReaderObservers.add(o);
1123   }
1124 
1125   @Override
1126   public void deleteChangedReaderObserver(ChangedReadersObserver o) {
1127     // We don't check if observer present; it may not be (legitimately)
1128     this.changedReaderObservers.remove(o);
1129   }
1130 
1131   //////////////////////////////////////////////////////////////////////////////
1132   // Compaction
1133   //////////////////////////////////////////////////////////////////////////////
1134 
1135   /**
1136    * Compact the StoreFiles.  This method may take some time, so the calling
1137    * thread must be able to block for long periods.
1138    *
1139    * <p>During this time, the Store can work as usual, getting values from
1140    * StoreFiles and writing new StoreFiles from the memstore.
1141    *
1142    * Existing StoreFiles are not destroyed until the new compacted StoreFile is
1143    * completely written-out to disk.
1144    *
1145    * <p>The compactLock prevents multiple simultaneous compactions.
1146    * The structureLock prevents us from interfering with other write operations.
1147    *
1148    * <p>We don't want to hold the structureLock for the whole time, as a compact()
1149    * can be lengthy and we want to allow cache-flushes during this period.
1150    *
1151    * <p> Compaction event should be idempotent, since there is no IO Fencing for
1152    * the region directory in hdfs. A region server might still try to complete the
1153    * compaction after it lost the region. That is why the following events are carefully
1154    * ordered for a compaction:
1155    *  1. Compaction writes new files under region/.tmp directory (compaction output)
1156    *  2. Compaction atomically moves the temporary file under region directory
1157    *  3. Compaction appends a WAL edit containing the compaction input and output files.
1158    *  Forces sync on WAL.
1159    *  4. Compaction deletes the input files from the region directory.
1160    *
1161    * Failure conditions are handled like this:
1162    *  - If RS fails before 2, compaction wont complete. Even if RS lives on and finishes
1163    *  the compaction later, it will only write the new data file to the region directory.
1164    *  Since we already have this data, this will be idempotent but we will have a redundant
1165    *  copy of the data.
1166    *  - If RS fails between 2 and 3, the region will have a redundant copy of the data. The
1167    *  RS that failed won't be able to finish snyc() for WAL because of lease recovery in WAL.
1168    *  - If RS fails after 3, the region region server who opens the region will pick up the
1169    *  the compaction marker from the WAL and replay it by removing the compaction input files.
1170    *  Failed RS can also attempt to delete those files, but the operation will be idempotent
1171    *
1172    * See HBASE-2231 for details.
1173    *
1174    * @param compaction compaction details obtained from requestCompaction()
1175    * @throws IOException
1176    * @return Storefile we compacted into or null if we failed or opted out early.
1177    */
1178   @Override
1179   public List<StoreFile> compact(CompactionContext compaction,
1180       ThroughputController throughputController) throws IOException {
1181     return compact(compaction, throughputController, null);
1182   }
1183 
1184   @Override
1185   public List<StoreFile> compact(CompactionContext compaction,
1186     ThroughputController throughputController, User user) throws IOException {
1187     assert compaction != null;
1188     List<StoreFile> sfs = null;
1189     CompactionRequest cr = compaction.getRequest();
1190     try {
1191       // Do all sanity checking in here if we have a valid CompactionRequest
1192       // because we need to clean up after it on the way out in a finally
1193       // block below
1194       long compactionStartTime = EnvironmentEdgeManager.currentTime();
1195       assert compaction.hasSelection();
1196       Collection<StoreFile> filesToCompact = cr.getFiles();
1197       assert !filesToCompact.isEmpty();
1198       synchronized (filesCompacting) {
1199         // sanity check: we're compacting files that this store knows about
1200         // TODO: change this to LOG.error() after more debugging
1201         Preconditions.checkArgument(filesCompacting.containsAll(filesToCompact));
1202       }
1203 
1204       // Ready to go. Have list of files to compact.
1205       LOG.info("Starting compaction of " + filesToCompact.size() + " file(s) in "
1206           + this + " of " + this.getRegionInfo().getRegionNameAsString()
1207           + " into tmpdir=" + fs.getTempDir() + ", totalSize="
1208           + TraditionalBinaryPrefix.long2String(cr.getSize(), "", 1));
1209 
1210       // Commence the compaction.
1211       List<Path> newFiles = compaction.compact(throughputController, user);
1212 
1213       // TODO: get rid of this!
1214       if (!this.conf.getBoolean("hbase.hstore.compaction.complete", true)) {
1215         LOG.warn("hbase.hstore.compaction.complete is set to false");
1216         sfs = new ArrayList<StoreFile>(newFiles.size());
1217         final boolean evictOnClose =
1218             cacheConf != null? cacheConf.shouldEvictOnClose(): true;
1219         for (Path newFile : newFiles) {
1220           // Create storefile around what we wrote with a reader on it.
1221           StoreFile sf = createStoreFileAndReader(newFile);
1222           sf.closeReader(evictOnClose);
1223           sfs.add(sf);
1224         }
1225         return sfs;
1226       }
1227       // Do the steps necessary to complete the compaction.
1228       sfs = moveCompatedFilesIntoPlace(cr, newFiles, user);
1229       writeCompactionWalRecord(filesToCompact, sfs);
1230       replaceStoreFiles(filesToCompact, sfs);
1231       if (cr.isMajor()) {
1232         majorCompactedCellsCount += getCompactionProgress().totalCompactingKVs;
1233         majorCompactedCellsSize += getCompactionProgress().totalCompactedSize;
1234       } else {
1235         compactedCellsCount += getCompactionProgress().totalCompactingKVs;
1236         compactedCellsSize += getCompactionProgress().totalCompactedSize;
1237       }
1238       // At this point the store will use new files for all new scanners.
1239       completeCompaction(filesToCompact); // update store size.
1240 
1241       logCompactionEndMessage(cr, sfs, compactionStartTime);
1242       return sfs;
1243     } finally {
1244       finishCompactionRequest(cr);
1245     }
1246   }
1247 
1248   private List<StoreFile> moveCompatedFilesIntoPlace(
1249       final CompactionRequest cr, List<Path> newFiles, User user) throws IOException {
1250     List<StoreFile> sfs = new ArrayList<StoreFile>(newFiles.size());
1251     for (Path newFile : newFiles) {
1252       assert newFile != null;
1253       final StoreFile sf = moveFileIntoPlace(newFile);
1254       if (this.getCoprocessorHost() != null) {
1255         final Store thisStore = this;
1256         if (user == null) {
1257           getCoprocessorHost().postCompact(thisStore, sf, cr);
1258         } else {
1259           try {
1260             user.getUGI().doAs(new PrivilegedExceptionAction<Void>() {
1261               @Override
1262               public Void run() throws Exception {
1263                 getCoprocessorHost().postCompact(thisStore, sf, cr);
1264                 return null;
1265               }
1266             });
1267           } catch (InterruptedException ie) {
1268             InterruptedIOException iioe = new InterruptedIOException();
1269             iioe.initCause(ie);
1270             throw iioe;
1271           }
1272         }
1273       }
1274       assert sf != null;
1275       sfs.add(sf);
1276     }
1277     return sfs;
1278   }
1279 
1280   // Package-visible for tests
1281   StoreFile moveFileIntoPlace(final Path newFile) throws IOException {
1282     validateStoreFile(newFile);
1283     // Move the file into the right spot
1284     Path destPath = fs.commitStoreFile(getColumnFamilyName(), newFile);
1285     return createStoreFileAndReader(destPath);
1286   }
1287 
1288   /**
1289    * Writes the compaction WAL record.
1290    * @param filesCompacted Files compacted (input).
1291    * @param newFiles Files from compaction.
1292    */
1293   private void writeCompactionWalRecord(Collection<StoreFile> filesCompacted,
1294       Collection<StoreFile> newFiles) throws IOException {
1295     if (region.getWAL() == null) return;
1296     List<Path> inputPaths = new ArrayList<Path>(filesCompacted.size());
1297     for (StoreFile f : filesCompacted) {
1298       inputPaths.add(f.getPath());
1299     }
1300     List<Path> outputPaths = new ArrayList<Path>(newFiles.size());
1301     for (StoreFile f : newFiles) {
1302       outputPaths.add(f.getPath());
1303     }
1304     HRegionInfo info = this.region.getRegionInfo();
1305     CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(info,
1306         family.getName(), inputPaths, outputPaths, fs.getStoreDir(getFamily().getNameAsString()));
1307     // Fix reaching into Region to get the maxWaitForSeqId.
1308     // Does this method belong in Region altogether given it is making so many references up there?
1309     // Could be Region#writeCompactionMarker(compactionDescriptor);
1310     WALUtil.writeCompactionMarker(this.region.getWAL(), this.region.getTableDesc(),
1311         this.region.getRegionInfo(), compactionDescriptor, this.region.getMVCC());
1312   }
1313 
1314   @VisibleForTesting
1315   void replaceStoreFiles(final Collection<StoreFile> compactedFiles,
1316       final Collection<StoreFile> result) throws IOException {
1317     this.lock.writeLock().lock();
1318     try {
1319       this.storeEngine.getStoreFileManager().addCompactionResults(compactedFiles, result);
1320       filesCompacting.removeAll(compactedFiles); // safe bc: lock.writeLock();
1321     } finally {
1322       this.lock.writeLock().unlock();
1323     }
1324   }
1325 
1326   /**
1327    * Log a very elaborate compaction completion message.
1328    * @param cr Request.
1329    * @param sfs Resulting files.
1330    * @param compactionStartTime Start time.
1331    */
1332   private void logCompactionEndMessage(
1333       CompactionRequest cr, List<StoreFile> sfs, long compactionStartTime) {
1334     long now = EnvironmentEdgeManager.currentTime();
1335     StringBuilder message = new StringBuilder(
1336       "Completed" + (cr.isMajor() ? " major" : "") + " compaction of "
1337       + cr.getFiles().size() + (cr.isAllFiles() ? " (all)" : "") + " file(s) in "
1338       + this + " of " + this.getRegionInfo().getRegionNameAsString() + " into ");
1339     if (sfs.isEmpty()) {
1340       message.append("none, ");
1341     } else {
1342       for (StoreFile sf: sfs) {
1343         message.append(sf.getPath().getName());
1344         message.append("(size=");
1345         message.append(TraditionalBinaryPrefix.long2String(sf.getReader().length(), "", 1));
1346         message.append("), ");
1347       }
1348     }
1349     message.append("total size for store is ")
1350       .append(StringUtils.TraditionalBinaryPrefix.long2String(storeSize, "", 1))
1351       .append(". This selection was in queue for ")
1352       .append(StringUtils.formatTimeDiff(compactionStartTime, cr.getSelectionTime()))
1353       .append(", and took ").append(StringUtils.formatTimeDiff(now, compactionStartTime))
1354       .append(" to execute.");
1355     LOG.info(message.toString());
1356     if (LOG.isTraceEnabled()) {
1357       int fileCount = storeEngine.getStoreFileManager().getStorefileCount();
1358       long resultSize = 0;
1359       for (StoreFile sf : sfs) {
1360         resultSize += sf.getReader().length();
1361       }
1362       String traceMessage = "COMPACTION start,end,size out,files in,files out,store size,"
1363         + "store files [" + compactionStartTime + "," + now + "," + resultSize + ","
1364           + cr.getFiles().size() + "," + sfs.size() + "," +  storeSize + "," + fileCount + "]";
1365       LOG.trace(traceMessage);
1366     }
1367   }
1368 
1369   /**
1370    * Call to complete a compaction. Its for the case where we find in the WAL a compaction
1371    * that was not finished.  We could find one recovering a WAL after a regionserver crash.
1372    * See HBASE-2231.
1373    * @param compaction
1374    */
1375   @Override
1376   public void replayCompactionMarker(CompactionDescriptor compaction,
1377       boolean pickCompactionFiles, boolean removeFiles)
1378       throws IOException {
1379     LOG.debug("Completing compaction from the WAL marker");
1380     List<String> compactionInputs = compaction.getCompactionInputList();
1381     List<String> compactionOutputs = Lists.newArrayList(compaction.getCompactionOutputList());
1382 
1383     // The Compaction Marker is written after the compaction is completed,
1384     // and the files moved into the region/family folder.
1385     //
1386     // If we crash after the entry is written, we may not have removed the
1387     // input files, but the output file is present.
1388     // (The unremoved input files will be removed by this function)
1389     //
1390     // If we scan the directory and the file is not present, it can mean that:
1391     //   - The file was manually removed by the user
1392     //   - The file was removed as consequence of subsequent compaction
1393     // so, we can't do anything with the "compaction output list" because those
1394     // files have already been loaded when opening the region (by virtue of
1395     // being in the store's folder) or they may be missing due to a compaction.
1396 
1397     String familyName = this.getColumnFamilyName();
1398     List<String> inputFiles = new ArrayList<String>(compactionInputs.size());
1399     for (String compactionInput : compactionInputs) {
1400       Path inputPath = fs.getStoreFilePath(familyName, compactionInput);
1401       inputFiles.add(inputPath.getName());
1402     }
1403 
1404     //some of the input files might already be deleted
1405     List<StoreFile> inputStoreFiles = new ArrayList<StoreFile>(compactionInputs.size());
1406     for (StoreFile sf : this.getStorefiles()) {
1407       if (inputFiles.contains(sf.getPath().getName())) {
1408         inputStoreFiles.add(sf);
1409       }
1410     }
1411 
1412     // check whether we need to pick up the new files
1413     List<StoreFile> outputStoreFiles = new ArrayList<StoreFile>(compactionOutputs.size());
1414 
1415     if (pickCompactionFiles) {
1416       for (StoreFile sf : this.getStorefiles()) {
1417         compactionOutputs.remove(sf.getPath().getName());
1418       }
1419       for (String compactionOutput : compactionOutputs) {
1420         StoreFileInfo storeFileInfo = fs.getStoreFileInfo(getColumnFamilyName(), compactionOutput);
1421         StoreFile storeFile = createStoreFileAndReader(storeFileInfo);
1422         outputStoreFiles.add(storeFile);
1423       }
1424     }
1425 
1426     if (!inputStoreFiles.isEmpty() || !outputStoreFiles.isEmpty()) {
1427       LOG.info("Replaying compaction marker, replacing input files: " +
1428           inputStoreFiles + " with output files : " + outputStoreFiles);
1429       this.replaceStoreFiles(inputStoreFiles, outputStoreFiles);
1430       this.completeCompaction(inputStoreFiles);
1431     }
1432   }
1433 
1434   /**
1435    * This method tries to compact N recent files for testing.
1436    * Note that because compacting "recent" files only makes sense for some policies,
1437    * e.g. the default one, it assumes default policy is used. It doesn't use policy,
1438    * but instead makes a compaction candidate list by itself.
1439    * @param N Number of files.
1440    */
1441   public void compactRecentForTestingAssumingDefaultPolicy(int N) throws IOException {
1442     List<StoreFile> filesToCompact;
1443     boolean isMajor;
1444 
1445     this.lock.readLock().lock();
1446     try {
1447       synchronized (filesCompacting) {
1448         filesToCompact = Lists.newArrayList(storeEngine.getStoreFileManager().getStorefiles());
1449         if (!filesCompacting.isEmpty()) {
1450           // exclude all files older than the newest file we're currently
1451           // compacting. this allows us to preserve contiguity (HBASE-2856)
1452           StoreFile last = filesCompacting.get(filesCompacting.size() - 1);
1453           int idx = filesToCompact.indexOf(last);
1454           Preconditions.checkArgument(idx != -1);
1455           filesToCompact.subList(0, idx + 1).clear();
1456         }
1457         int count = filesToCompact.size();
1458         if (N > count) {
1459           throw new RuntimeException("Not enough files");
1460         }
1461 
1462         filesToCompact = filesToCompact.subList(count - N, count);
1463         isMajor = (filesToCompact.size() == storeEngine.getStoreFileManager().getStorefileCount());
1464         filesCompacting.addAll(filesToCompact);
1465         Collections.sort(filesCompacting, StoreFile.Comparators.SEQ_ID);
1466       }
1467     } finally {
1468       this.lock.readLock().unlock();
1469     }
1470 
1471     try {
1472       // Ready to go. Have list of files to compact.
1473       List<Path> newFiles = ((DefaultCompactor)this.storeEngine.getCompactor())
1474           .compactForTesting(filesToCompact, isMajor);
1475       for (Path newFile: newFiles) {
1476         // Move the compaction into place.
1477         StoreFile sf = moveFileIntoPlace(newFile);
1478         if (this.getCoprocessorHost() != null) {
1479           this.getCoprocessorHost().postCompact(this, sf, null);
1480         }
1481         replaceStoreFiles(filesToCompact, Lists.newArrayList(sf));
1482         completeCompaction(filesToCompact);
1483       }
1484     } finally {
1485       synchronized (filesCompacting) {
1486         filesCompacting.removeAll(filesToCompact);
1487       }
1488     }
1489   }
1490 
1491   @Override
1492   public boolean hasReferences() {
1493     return StoreUtils.hasReferences(this.storeEngine.getStoreFileManager().getStorefiles());
1494   }
1495 
1496   @Override
1497   public CompactionProgress getCompactionProgress() {
1498     return this.storeEngine.getCompactor().getProgress();
1499   }
1500 
1501   @Override
1502   public boolean isMajorCompaction() throws IOException {
1503     for (StoreFile sf : this.storeEngine.getStoreFileManager().getStorefiles()) {
1504       // TODO: what are these reader checks all over the place?
1505       if (sf.getReader() == null) {
1506         LOG.debug("StoreFile " + sf + " has null Reader");
1507         return false;
1508       }
1509     }
1510     return storeEngine.getCompactionPolicy().isMajorCompaction(
1511         this.storeEngine.getStoreFileManager().getStorefiles());
1512   }
1513 
1514   @Override
1515   public CompactionContext requestCompaction() throws IOException {
1516     return requestCompaction(Store.NO_PRIORITY, null);
1517   }
1518 
1519   @Override
1520   public CompactionContext requestCompaction(int priority, CompactionRequest baseRequest)
1521       throws IOException {
1522     return requestCompaction(priority, baseRequest, null);
1523   }
1524   @Override
1525   public CompactionContext requestCompaction(int priority, final CompactionRequest baseRequest,
1526       User user) throws IOException {
1527     // don't even select for compaction if writes are disabled
1528     if (!this.areWritesEnabled()) {
1529       return null;
1530     }
1531 
1532     // Before we do compaction, try to get rid of unneeded files to simplify things.
1533     removeUnneededFiles();
1534 
1535     final CompactionContext compaction = storeEngine.createCompaction();
1536     CompactionRequest request = null;
1537     this.lock.readLock().lock();
1538     try {
1539       synchronized (filesCompacting) {
1540         final Store thisStore = this;
1541         // First, see if coprocessor would want to override selection.
1542         if (this.getCoprocessorHost() != null) {
1543           final List<StoreFile> candidatesForCoproc = compaction.preSelect(this.filesCompacting);
1544           boolean override = false;
1545           if (user == null) {
1546             override = getCoprocessorHost().preCompactSelection(this, candidatesForCoproc,
1547               baseRequest);
1548           } else {
1549             try {
1550               override = user.getUGI().doAs(new PrivilegedExceptionAction<Boolean>() {
1551                 @Override
1552                 public Boolean run() throws Exception {
1553                   return getCoprocessorHost().preCompactSelection(thisStore, candidatesForCoproc,
1554                     baseRequest);
1555                 }
1556               });
1557             } catch (InterruptedException ie) {
1558               InterruptedIOException iioe = new InterruptedIOException();
1559               iioe.initCause(ie);
1560               throw iioe;
1561             }
1562           }
1563           if (override) {
1564             // Coprocessor is overriding normal file selection.
1565             compaction.forceSelect(new CompactionRequest(candidatesForCoproc));
1566           }
1567         }
1568 
1569         // Normal case - coprocessor is not overriding file selection.
1570         if (!compaction.hasSelection()) {
1571           boolean isUserCompaction = priority == Store.PRIORITY_USER;
1572           boolean mayUseOffPeak = offPeakHours.isOffPeakHour() &&
1573               offPeakCompactionTracker.compareAndSet(false, true);
1574           try {
1575             compaction.select(this.filesCompacting, isUserCompaction,
1576               mayUseOffPeak, forceMajor && filesCompacting.isEmpty());
1577           } catch (IOException e) {
1578             if (mayUseOffPeak) {
1579               offPeakCompactionTracker.set(false);
1580             }
1581             throw e;
1582           }
1583           assert compaction.hasSelection();
1584           if (mayUseOffPeak && !compaction.getRequest().isOffPeak()) {
1585             // Compaction policy doesn't want to take advantage of off-peak.
1586             offPeakCompactionTracker.set(false);
1587           }
1588         }
1589         if (this.getCoprocessorHost() != null) {
1590           if (user == null) {
1591             this.getCoprocessorHost().postCompactSelection(
1592               this, ImmutableList.copyOf(compaction.getRequest().getFiles()), baseRequest);
1593           } else {
1594             try {
1595               user.getUGI().doAs(new PrivilegedExceptionAction<Void>() {
1596                 @Override
1597                 public Void run() throws Exception {
1598                   getCoprocessorHost().postCompactSelection(
1599                     thisStore,ImmutableList.copyOf(compaction.getRequest().getFiles()),baseRequest);
1600                   return null;
1601                 }
1602               });
1603             } catch (InterruptedException ie) {
1604               InterruptedIOException iioe = new InterruptedIOException();
1605               iioe.initCause(ie);
1606               throw iioe;
1607             }
1608           }
1609         }
1610 
1611         // Selected files; see if we have a compaction with some custom base request.
1612         if (baseRequest != null) {
1613           // Update the request with what the system thinks the request should be;
1614           // its up to the request if it wants to listen.
1615           compaction.forceSelect(
1616               baseRequest.combineWith(compaction.getRequest()));
1617         }
1618         // Finally, we have the resulting files list. Check if we have any files at all.
1619         request = compaction.getRequest();
1620         final Collection<StoreFile> selectedFiles = request.getFiles();
1621         if (selectedFiles.isEmpty()) {
1622           return null;
1623         }
1624 
1625         addToCompactingFiles(selectedFiles);
1626 
1627         // If we're enqueuing a major, clear the force flag.
1628         this.forceMajor = this.forceMajor && !request.isMajor();
1629 
1630         // Set common request properties.
1631         // Set priority, either override value supplied by caller or from store.
1632         request.setPriority((priority != Store.NO_PRIORITY) ? priority : getCompactPriority());
1633         request.setDescription(getRegionInfo().getRegionNameAsString(), getColumnFamilyName());
1634       }
1635     } finally {
1636       this.lock.readLock().unlock();
1637     }
1638 
1639     LOG.debug(getRegionInfo().getEncodedName() + " - "  + getColumnFamilyName()
1640         + ": Initiating " + (request.isMajor() ? "major" : "minor") + " compaction"
1641         + (request.isAllFiles() ? " (all files)" : ""));
1642     this.region.reportCompactionRequestStart(request.isMajor());
1643     return compaction;
1644   }
1645 
1646   /** Adds the files to compacting files. filesCompacting must be locked. */
1647   private void addToCompactingFiles(final Collection<StoreFile> filesToAdd) {
1648     if (filesToAdd == null) return;
1649     // Check that we do not try to compact the same StoreFile twice.
1650     if (!Collections.disjoint(filesCompacting, filesToAdd)) {
1651       Preconditions.checkArgument(false, "%s overlaps with %s", filesToAdd, filesCompacting);
1652     }
1653     filesCompacting.addAll(filesToAdd);
1654     Collections.sort(filesCompacting, StoreFile.Comparators.SEQ_ID);
1655   }
1656 
1657   private void removeUnneededFiles() throws IOException {
1658     if (!conf.getBoolean("hbase.store.delete.expired.storefile", true)) return;
1659     if (getFamily().getMinVersions() > 0) {
1660       LOG.debug("Skipping expired store file removal due to min version being " +
1661           getFamily().getMinVersions());
1662       return;
1663     }
1664     this.lock.readLock().lock();
1665     Collection<StoreFile> delSfs = null;
1666     try {
1667       synchronized (filesCompacting) {
1668         long cfTtl = getStoreFileTtl();
1669         if (cfTtl != Long.MAX_VALUE) {
1670           delSfs = storeEngine.getStoreFileManager().getUnneededFiles(
1671               EnvironmentEdgeManager.currentTime() - cfTtl, filesCompacting);
1672           addToCompactingFiles(delSfs);
1673         }
1674       }
1675     } finally {
1676       this.lock.readLock().unlock();
1677     }
1678     if (delSfs == null || delSfs.isEmpty()) return;
1679 
1680     Collection<StoreFile> newFiles = new ArrayList<StoreFile>(); // No new files.
1681     writeCompactionWalRecord(delSfs, newFiles);
1682     replaceStoreFiles(delSfs, newFiles);
1683     completeCompaction(delSfs);
1684     LOG.info("Completed removal of " + delSfs.size() + " unnecessary (expired) file(s) in "
1685         + this + " of " + this.getRegionInfo().getRegionNameAsString()
1686         + "; total size for store is " + TraditionalBinaryPrefix.long2String(storeSize, "", 1));
1687   }
1688 
1689   @Override
1690   public void cancelRequestedCompaction(CompactionContext compaction) {
1691     finishCompactionRequest(compaction.getRequest());
1692   }
1693 
1694   private void finishCompactionRequest(CompactionRequest cr) {
1695     this.region.reportCompactionRequestEnd(cr.isMajor(), cr.getFiles().size(), cr.getSize());
1696     if (cr.isOffPeak()) {
1697       offPeakCompactionTracker.set(false);
1698       cr.setOffPeak(false);
1699     }
1700     synchronized (filesCompacting) {
1701       filesCompacting.removeAll(cr.getFiles());
1702     }
1703   }
1704 
1705   /**
1706    * Validates a store file by opening and closing it. In HFileV2 this should
1707    * not be an expensive operation.
1708    *
1709    * @param path the path to the store file
1710    */
1711   private void validateStoreFile(Path path)
1712       throws IOException {
1713     StoreFile storeFile = null;
1714     try {
1715       storeFile = createStoreFileAndReader(path);
1716     } catch (IOException e) {
1717       LOG.error("Failed to open store file : " + path
1718           + ", keeping it in tmp location", e);
1719       throw e;
1720     } finally {
1721       if (storeFile != null) {
1722         storeFile.closeReader(false);
1723       }
1724     }
1725   }
1726 
1727   /**
1728    * <p>It works by processing a compaction that's been written to disk.
1729    *
1730    * <p>It is usually invoked at the end of a compaction, but might also be
1731    * invoked at HStore startup, if the prior execution died midway through.
1732    *
1733    * <p>Moving the compacted TreeMap into place means:
1734    * <pre>
1735    * 1) Unload all replaced StoreFile, close and collect list to delete.
1736    * 2) Compute new store size
1737    * </pre>
1738    *
1739    * @param compactedFiles list of files that were compacted
1740    */
1741   @VisibleForTesting
1742   protected void completeCompaction(final Collection<StoreFile> compactedFiles)
1743     throws IOException {
1744     LOG.debug("Completing compaction...");
1745     this.storeSize = 0L;
1746     this.totalUncompressedBytes = 0L;
1747     for (StoreFile hsf : this.storeEngine.getStoreFileManager().getStorefiles()) {
1748       StoreFile.Reader r = hsf.getReader();
1749       if (r == null) {
1750         LOG.warn("StoreFile " + hsf + " has a null Reader");
1751         continue;
1752       }
1753       this.storeSize += r.length();
1754       this.totalUncompressedBytes += r.getTotalUncompressedBytes();
1755     }
1756   }
1757 
1758   /*
1759    * @param wantedVersions How many versions were asked for.
1760    * @return wantedVersions or this families' {@link HConstants#VERSIONS}.
1761    */
1762   int versionsToReturn(final int wantedVersions) {
1763     if (wantedVersions <= 0) {
1764       throw new IllegalArgumentException("Number of versions must be > 0");
1765     }
1766     // Make sure we do not return more than maximum versions for this store.
1767     int maxVersions = this.family.getMaxVersions();
1768     return wantedVersions > maxVersions ? maxVersions: wantedVersions;
1769   }
1770 
1771   /**
1772    * @param cell
1773    * @param oldestTimestamp
1774    * @return true if the cell is expired
1775    */
1776   static boolean isCellTTLExpired(final Cell cell, final long oldestTimestamp, final long now) {
1777     // Look for a TTL tag first. Use it instead of the family setting if
1778     // found. If a cell has multiple TTLs, resolve the conflict by using the
1779     // first tag encountered.
1780     Iterator<Tag> i = CellUtil.tagsIterator(cell);
1781     while (i.hasNext()) {
1782       Tag t = i.next();
1783       if (TagType.TTL_TAG_TYPE == t.getType()) {
1784         // Unlike in schema cell TTLs are stored in milliseconds, no need
1785         // to convert
1786         long ts = cell.getTimestamp();
1787         assert t.getValueLength() == Bytes.SIZEOF_LONG;
1788         long ttl = TagUtil.getValueAsLong(t);
1789         if (ts + ttl < now) {
1790           return true;
1791         }
1792         // Per cell TTLs cannot extend lifetime beyond family settings, so
1793         // fall through to check that
1794         break;
1795       }
1796     }
1797     return false;
1798   }
1799 
1800   @Override
1801   public boolean canSplit() {
1802     this.lock.readLock().lock();
1803     try {
1804       // Not split-able if we find a reference store file present in the store.
1805       boolean result = !hasReferences();
1806       if (!result && LOG.isDebugEnabled()) {
1807         LOG.debug("Cannot split region due to reference files being there");
1808       }
1809       return result;
1810     } finally {
1811       this.lock.readLock().unlock();
1812     }
1813   }
1814 
1815   @Override
1816   public byte[] getSplitPoint() {
1817     this.lock.readLock().lock();
1818     try {
1819       // Should already be enforced by the split policy!
1820       assert !this.getRegionInfo().isMetaRegion();
1821       // Not split-able if we find a reference store file present in the store.
1822       if (hasReferences()) {
1823         return null;
1824       }
1825       return this.storeEngine.getStoreFileManager().getSplitPoint();
1826     } catch(IOException e) {
1827       LOG.warn("Failed getting store size for " + this, e);
1828     } finally {
1829       this.lock.readLock().unlock();
1830     }
1831     return null;
1832   }
1833 
1834   @Override
1835   public long getLastCompactSize() {
1836     return this.lastCompactSize;
1837   }
1838 
1839   @Override
1840   public long getSize() {
1841     return storeSize;
1842   }
1843 
1844   @Override
1845   public void triggerMajorCompaction() {
1846     this.forceMajor = true;
1847   }
1848 
1849 
1850   //////////////////////////////////////////////////////////////////////////////
1851   // File administration
1852   //////////////////////////////////////////////////////////////////////////////
1853 
1854   @Override
1855   public KeyValueScanner getScanner(Scan scan,
1856       final NavigableSet<byte []> targetCols, long readPt) throws IOException {
1857     lock.readLock().lock();
1858     try {
1859       KeyValueScanner scanner = null;
1860       if (this.getCoprocessorHost() != null) {
1861         scanner = this.getCoprocessorHost().preStoreScannerOpen(this, scan, targetCols);
1862       }
1863       scanner = createScanner(scan, targetCols, readPt, scanner);
1864       return scanner;
1865     } finally {
1866       lock.readLock().unlock();
1867     }
1868   }
1869 
1870   protected KeyValueScanner createScanner(Scan scan, final NavigableSet<byte[]> targetCols,
1871       long readPt, KeyValueScanner scanner) throws IOException {
1872     if (scanner == null) {
1873       scanner = scan.isReversed() ? new ReversedStoreScanner(this,
1874           getScanInfo(), scan, targetCols, readPt) : new StoreScanner(this,
1875           getScanInfo(), scan, targetCols, readPt);
1876     }
1877     return scanner;
1878   }
1879 
1880   @Override
1881   public String toString() {
1882     return this.getColumnFamilyName();
1883   }
1884 
1885   @Override
1886   public int getStorefilesCount() {
1887     return this.storeEngine.getStoreFileManager().getStorefileCount();
1888   }
1889 
1890   @Override
1891   public long getStoreSizeUncompressed() {
1892     return this.totalUncompressedBytes;
1893   }
1894 
1895   @Override
1896   public long getStorefilesSize() {
1897     long size = 0;
1898     for (StoreFile s: this.storeEngine.getStoreFileManager().getStorefiles()) {
1899       StoreFile.Reader r = s.getReader();
1900       if (r == null) {
1901         LOG.warn("StoreFile " + s + " has a null Reader");
1902         continue;
1903       }
1904       size += r.length();
1905     }
1906     return size;
1907   }
1908 
1909   @Override
1910   public long getStorefilesIndexSize() {
1911     long size = 0;
1912     for (StoreFile s: this.storeEngine.getStoreFileManager().getStorefiles()) {
1913       StoreFile.Reader r = s.getReader();
1914       if (r == null) {
1915         LOG.warn("StoreFile " + s + " has a null Reader");
1916         continue;
1917       }
1918       size += r.indexSize();
1919     }
1920     return size;
1921   }
1922 
1923   @Override
1924   public long getTotalStaticIndexSize() {
1925     long size = 0;
1926     for (StoreFile s : this.storeEngine.getStoreFileManager().getStorefiles()) {
1927       StoreFile.Reader r = s.getReader();
1928       if (r == null) {
1929         continue;
1930       }
1931       size += r.getUncompressedDataIndexSize();
1932     }
1933     return size;
1934   }
1935 
1936   @Override
1937   public long getTotalStaticBloomSize() {
1938     long size = 0;
1939     for (StoreFile s : this.storeEngine.getStoreFileManager().getStorefiles()) {
1940       StoreFile.Reader r = s.getReader();
1941       if (r == null) {
1942         continue;
1943       }
1944       size += r.getTotalBloomSize();
1945     }
1946     return size;
1947   }
1948 
1949   @Override
1950   public long getMemStoreSize() {
1951     return this.memstore.size();
1952   }
1953 
1954   @Override
1955   public int getCompactPriority() {
1956     int priority = this.storeEngine.getStoreFileManager().getStoreCompactionPriority();
1957     if (priority == PRIORITY_USER) {
1958       LOG.warn("Compaction priority is USER despite there being no user compaction");
1959     }
1960     return priority;
1961   }
1962 
1963   @Override
1964   public boolean throttleCompaction(long compactionSize) {
1965     return storeEngine.getCompactionPolicy().throttleCompaction(compactionSize);
1966   }
1967 
1968   public HRegion getHRegion() {
1969     return this.region;
1970   }
1971 
1972   @Override
1973   public RegionCoprocessorHost getCoprocessorHost() {
1974     return this.region.getCoprocessorHost();
1975   }
1976 
1977   @Override
1978   public HRegionInfo getRegionInfo() {
1979     return this.fs.getRegionInfo();
1980   }
1981 
1982   @Override
1983   public boolean areWritesEnabled() {
1984     return this.region.areWritesEnabled();
1985   }
1986 
1987   @Override
1988   public long getSmallestReadPoint() {
1989     return this.region.getSmallestReadPoint();
1990   }
1991 
1992   /**
1993    * Used in tests. TODO: Remove
1994    *
1995    * Updates the value for the given row/family/qualifier. This function will always be seen as
1996    * atomic by other readers because it only puts a single KV to memstore. Thus no read/write
1997    * control necessary.
1998    * @param row row to update
1999    * @param f family to update
2000    * @param qualifier qualifier to update
2001    * @param newValue the new value to set into memstore
2002    * @return memstore size delta
2003    * @throws IOException
2004    */
2005   public long updateColumnValue(byte [] row, byte [] f,
2006                                 byte [] qualifier, long newValue)
2007       throws IOException {
2008 
2009     this.lock.readLock().lock();
2010     try {
2011       long now = EnvironmentEdgeManager.currentTime();
2012 
2013       return this.memstore.updateColumnValue(row,
2014           f,
2015           qualifier,
2016           newValue,
2017           now);
2018 
2019     } finally {
2020       this.lock.readLock().unlock();
2021     }
2022   }
2023 
2024   @Override
2025   public long upsert(Iterable<Cell> cells, long readpoint) throws IOException {
2026     this.lock.readLock().lock();
2027     try {
2028       return this.memstore.upsert(cells, readpoint);
2029     } finally {
2030       this.lock.readLock().unlock();
2031     }
2032   }
2033 
2034   @Override
2035   public StoreFlushContext createFlushContext(long cacheFlushId) {
2036     return new StoreFlusherImpl(cacheFlushId);
2037   }
2038 
2039   private final class StoreFlusherImpl implements StoreFlushContext {
2040 
2041     private long cacheFlushSeqNum;
2042     private MemStoreSnapshot snapshot;
2043     private List<Path> tempFiles;
2044     private List<Path> committedFiles;
2045     private long cacheFlushCount;
2046     private long cacheFlushSize;
2047 
2048     private StoreFlusherImpl(long cacheFlushSeqNum) {
2049       this.cacheFlushSeqNum = cacheFlushSeqNum;
2050     }
2051 
2052     /**
2053      * This is not thread safe. The caller should have a lock on the region or the store.
2054      * If necessary, the lock can be added with the patch provided in HBASE-10087
2055      */
2056     @Override
2057     public void prepare() {
2058       this.snapshot = memstore.snapshot();
2059       this.cacheFlushCount = snapshot.getCellsCount();
2060       this.cacheFlushSize = snapshot.getSize();
2061       committedFiles = new ArrayList<Path>(1);
2062     }
2063 
2064     @Override
2065     public void flushCache(MonitoredTask status) throws IOException {
2066       RegionServerServices rsService = region.getRegionServerServices();
2067       ThroughputController throughputController =
2068           rsService == null ? null : rsService.getFlushThroughputController();
2069       tempFiles = HStore.this.flushCache(cacheFlushSeqNum, snapshot, status, throughputController);
2070     }
2071 
2072     @Override
2073     public boolean commit(MonitoredTask status) throws IOException {
2074       if (this.tempFiles == null || this.tempFiles.isEmpty()) {
2075         return false;
2076       }
2077       List<StoreFile> storeFiles = new ArrayList<StoreFile>(this.tempFiles.size());
2078       for (Path storeFilePath : tempFiles) {
2079         try {
2080           storeFiles.add(HStore.this.commitFile(storeFilePath, cacheFlushSeqNum, status));
2081         } catch (IOException ex) {
2082           LOG.error("Failed to commit store file " + storeFilePath, ex);
2083           // Try to delete the files we have committed before.
2084           for (StoreFile sf : storeFiles) {
2085             Path pathToDelete = sf.getPath();
2086             try {
2087               sf.deleteReader();
2088             } catch (IOException deleteEx) {
2089               LOG.fatal("Failed to delete store file we committed, halting " + pathToDelete, ex);
2090               Runtime.getRuntime().halt(1);
2091             }
2092           }
2093           throw new IOException("Failed to commit the flush", ex);
2094         }
2095       }
2096 
2097       for (StoreFile sf : storeFiles) {
2098         if (HStore.this.getCoprocessorHost() != null) {
2099           HStore.this.getCoprocessorHost().postFlush(HStore.this, sf);
2100         }
2101         committedFiles.add(sf.getPath());
2102       }
2103 
2104       HStore.this.flushedCellsCount += cacheFlushCount;
2105       HStore.this.flushedCellsSize += cacheFlushSize;
2106 
2107       // Add new file to store files.  Clear snapshot too while we have the Store write lock.
2108       return HStore.this.updateStorefiles(storeFiles, snapshot.getId());
2109     }
2110 
2111     @Override
2112     public List<Path> getCommittedFiles() {
2113       return committedFiles;
2114     }
2115 
2116     /**
2117      * Similar to commit, but called in secondary region replicas for replaying the
2118      * flush cache from primary region. Adds the new files to the store, and drops the
2119      * snapshot depending on dropMemstoreSnapshot argument.
2120      * @param fileNames names of the flushed files
2121      * @param dropMemstoreSnapshot whether to drop the prepared memstore snapshot
2122      * @throws IOException
2123      */
2124     @Override
2125     public void replayFlush(List<String> fileNames, boolean dropMemstoreSnapshot)
2126         throws IOException {
2127       List<StoreFile> storeFiles = new ArrayList<StoreFile>(fileNames.size());
2128       for (String file : fileNames) {
2129         // open the file as a store file (hfile link, etc)
2130         StoreFileInfo storeFileInfo = fs.getStoreFileInfo(getColumnFamilyName(), file);
2131         StoreFile storeFile = createStoreFileAndReader(storeFileInfo);
2132         storeFiles.add(storeFile);
2133         HStore.this.storeSize += storeFile.getReader().length();
2134         HStore.this.totalUncompressedBytes += storeFile.getReader().getTotalUncompressedBytes();
2135         if (LOG.isInfoEnabled()) {
2136           LOG.info("Region: " + HStore.this.getRegionInfo().getEncodedName() +
2137             " added " + storeFile + ", entries=" + storeFile.getReader().getEntries() +
2138             ", sequenceid=" +  + storeFile.getReader().getSequenceID() +
2139             ", filesize=" + StringUtils.humanReadableInt(storeFile.getReader().length()));
2140         }
2141       }
2142 
2143       long snapshotId = -1; // -1 means do not drop
2144       if (dropMemstoreSnapshot && snapshot != null) {
2145         snapshotId = snapshot.getId();
2146       }
2147       HStore.this.updateStorefiles(storeFiles, snapshotId);
2148     }
2149 
2150     /**
2151      * Abort the snapshot preparation. Drops the snapshot if any.
2152      * @throws IOException
2153      */
2154     @Override
2155     public void abort() throws IOException {
2156       if (snapshot == null) {
2157         return;
2158       }
2159       HStore.this.updateStorefiles(new ArrayList<StoreFile>(0), snapshot.getId());
2160     }
2161   }
2162 
2163   @Override
2164   public boolean needsCompaction() {
2165     return this.storeEngine.needsCompaction(this.filesCompacting);
2166   }
2167 
2168   @Override
2169   public CacheConfig getCacheConfig() {
2170     return this.cacheConf;
2171   }
2172 
2173   public static final long FIXED_OVERHEAD =
2174       ClassSize.align(ClassSize.OBJECT + (16 * ClassSize.REFERENCE) + (10 * Bytes.SIZEOF_LONG)
2175               + (5 * Bytes.SIZEOF_INT) + (2 * Bytes.SIZEOF_BOOLEAN));
2176 
2177   public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD
2178       + ClassSize.OBJECT + ClassSize.REENTRANT_LOCK
2179       + ClassSize.CONCURRENT_SKIPLISTMAP
2180       + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY + ClassSize.OBJECT
2181       + ScanInfo.FIXED_OVERHEAD);
2182 
2183   @Override
2184   public long heapSize() {
2185     return DEEP_OVERHEAD + this.memstore.heapSize();
2186   }
2187 
2188   @Override
2189   public CellComparator getComparator() {
2190     return comparator;
2191   }
2192 
2193   @Override
2194   public ScanInfo getScanInfo() {
2195     return scanInfo;
2196   }
2197 
2198   /**
2199    * Set scan info, used by test
2200    * @param scanInfo new scan info to use for test
2201    */
2202   void setScanInfo(ScanInfo scanInfo) {
2203     this.scanInfo = scanInfo;
2204   }
2205 
2206   @Override
2207   public boolean hasTooManyStoreFiles() {
2208     return getStorefilesCount() > this.blockingFileCount;
2209   }
2210 
2211   @Override
2212   public long getFlushedCellsCount() {
2213     return flushedCellsCount;
2214   }
2215 
2216   @Override
2217   public long getFlushedCellsSize() {
2218     return flushedCellsSize;
2219   }
2220 
2221   @Override
2222   public long getCompactedCellsCount() {
2223     return compactedCellsCount;
2224   }
2225 
2226   @Override
2227   public long getCompactedCellsSize() {
2228     return compactedCellsSize;
2229   }
2230 
2231   @Override
2232   public long getMajorCompactedCellsCount() {
2233     return majorCompactedCellsCount;
2234   }
2235 
2236   @Override
2237   public long getMajorCompactedCellsSize() {
2238     return majorCompactedCellsSize;
2239   }
2240 
2241   /**
2242    * Returns the StoreEngine that is backing this concrete implementation of Store.
2243    * @return Returns the {@link StoreEngine} object used internally inside this HStore object.
2244    */
2245   @VisibleForTesting
2246   public StoreEngine<?, ?, ?, ?> getStoreEngine() {
2247     return this.storeEngine;
2248   }
2249 
2250   protected OffPeakHours getOffPeakHours() {
2251     return this.offPeakHours;
2252   }
2253 
2254   /**
2255    * {@inheritDoc}
2256    */
2257   @Override
2258   public void onConfigurationChange(Configuration conf) {
2259     this.conf = new CompoundConfiguration()
2260             .add(conf)
2261             .addBytesMap(family.getValues());
2262     this.storeEngine.compactionPolicy.setConf(conf);
2263     this.offPeakHours = OffPeakHours.getInstance(conf);
2264   }
2265 
2266   /**
2267    * {@inheritDoc}
2268    */
2269   @Override
2270   public void registerChildren(ConfigurationManager manager) {
2271     // No children to register
2272   }
2273 
2274   /**
2275    * {@inheritDoc}
2276    */
2277   @Override
2278   public void deregisterChildren(ConfigurationManager manager) {
2279     // No children to deregister
2280   }
2281 
2282   @Override
2283   public double getCompactionPressure() {
2284     return storeEngine.getStoreFileManager().getCompactionPressure();
2285   }
2286 
2287   @Override
2288   public boolean isPrimaryReplicaStore() {
2289 	   return getRegionInfo().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID;
2290   }
2291 
2292   @Override
2293   public void closeAndArchiveCompactedFiles() throws IOException {
2294     lock.readLock().lock();
2295     Collection<StoreFile> copyCompactedfiles = null;
2296     try {
2297       Collection<StoreFile> compactedfiles =
2298           this.getStoreEngine().getStoreFileManager().getCompactedfiles();
2299       if (compactedfiles != null && compactedfiles.size() != 0) {
2300         // Do a copy under read lock
2301         copyCompactedfiles = new ArrayList<StoreFile>(compactedfiles);
2302       } else {
2303         if (LOG.isTraceEnabled()) {
2304           LOG.trace("No compacted files to archive");
2305           return;
2306         }
2307       }
2308     } finally {
2309       lock.readLock().unlock();
2310     }
2311     if (copyCompactedfiles != null && !copyCompactedfiles.isEmpty()) {
2312       removeCompactedfiles(copyCompactedfiles);
2313     }
2314   }
2315 
2316   /**
2317    * Archives and removes the compacted files
2318    * @param compactedfiles The compacted files in this store that are not active in reads
2319    * @throws IOException
2320    */
2321   private void removeCompactedfiles(Collection<StoreFile> compactedfiles)
2322       throws IOException {
2323     final List<StoreFile> filesToRemove = new ArrayList<StoreFile>(compactedfiles.size());
2324     for (final StoreFile file : compactedfiles) {
2325       synchronized (file) {
2326         try {
2327           StoreFile.Reader r = file.getReader();
2328           if (r == null) {
2329             if (LOG.isDebugEnabled()) {
2330               LOG.debug("The file " + file + " was closed but still not archived.");
2331             }
2332             filesToRemove.add(file);
2333           }
2334           if (r != null && r.isCompactedAway() && !r.isReferencedInReads()) {
2335             // Even if deleting fails we need not bother as any new scanners won't be
2336             // able to use the compacted file as the status is already compactedAway
2337             if (LOG.isTraceEnabled()) {
2338               LOG.trace("Closing and archiving the file " + file.getPath());
2339             }
2340             r.close(true);
2341             // Just close and return
2342             filesToRemove.add(file);
2343           }
2344         } catch (Exception e) {
2345           LOG.error(
2346             "Exception while trying to close the compacted store file " + file.getPath().getName());
2347         }
2348       }
2349     }
2350     if (this.isPrimaryReplicaStore()) {
2351       // Only the primary region is allowed to move the file to archive.
2352       // The secondary region does not move the files to archive. Any active reads from
2353       // the secondary region will still work because the file as such has active readers on it.
2354       if (!filesToRemove.isEmpty()) {
2355         if (LOG.isDebugEnabled()) {
2356           LOG.debug("Moving the files " + filesToRemove + " to archive");
2357         }
2358         // Only if this is successful it has to be removed
2359         this.fs.removeStoreFiles(this.getFamily().getNameAsString(), filesToRemove);
2360       }
2361     }
2362     if (!filesToRemove.isEmpty()) {
2363       // Clear the compactedfiles from the store file manager
2364       clearCompactedfiles(filesToRemove);
2365     }
2366   }
2367 
2368   private void clearCompactedfiles(final List<StoreFile> filesToRemove) throws IOException {
2369     if (LOG.isTraceEnabled()) {
2370       LOG.trace("Clearing the compacted file " + filesToRemove + " from this store");
2371     }
2372     try {
2373       lock.writeLock().lock();
2374       this.getStoreEngine().getStoreFileManager().removeCompactedFiles(filesToRemove);
2375     } finally {
2376       lock.writeLock().unlock();
2377     }
2378   }
2379 }