View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import java.io.IOException;
22  import java.io.InterruptedIOException;
23  import java.net.InetSocketAddress;
24  import java.security.Key;
25  import java.security.KeyException;
26  import java.util.ArrayList;
27  import java.util.Collection;
28  import java.util.Collections;
29  import java.util.HashMap;
30  import java.util.HashSet;
31  import java.util.Iterator;
32  import java.util.List;
33  import java.util.NavigableSet;
34  import java.util.Set;
35  import java.util.concurrent.Callable;
36  import java.util.concurrent.CompletionService;
37  import java.util.concurrent.ConcurrentHashMap;
38  import java.util.concurrent.ExecutionException;
39  import java.util.concurrent.ExecutorCompletionService;
40  import java.util.concurrent.Future;
41  import java.util.concurrent.ThreadPoolExecutor;
42  import java.util.concurrent.atomic.AtomicBoolean;
43  import java.util.concurrent.locks.ReentrantReadWriteLock;
44  
45  import org.apache.commons.logging.Log;
46  import org.apache.commons.logging.LogFactory;
47  import org.apache.hadoop.conf.Configuration;
48  import org.apache.hadoop.fs.FileSystem;
49  import org.apache.hadoop.fs.Path;
50  import org.apache.hadoop.hbase.Cell;
51  import org.apache.hadoop.hbase.CellComparator;
52  import org.apache.hadoop.hbase.CellUtil;
53  import org.apache.hadoop.hbase.CompoundConfiguration;
54  import org.apache.hadoop.hbase.HColumnDescriptor;
55  import org.apache.hadoop.hbase.HConstants;
56  import org.apache.hadoop.hbase.HRegionInfo;
57  import org.apache.hadoop.hbase.KeyValue;
58  import org.apache.hadoop.hbase.TableName;
59  import org.apache.hadoop.hbase.Tag;
60  import org.apache.hadoop.hbase.TagType;
61  import org.apache.hadoop.hbase.classification.InterfaceAudience;
62  import org.apache.hadoop.hbase.client.Scan;
63  import org.apache.hadoop.hbase.conf.ConfigurationManager;
64  import org.apache.hadoop.hbase.io.compress.Compression;
65  import org.apache.hadoop.hbase.io.crypto.Cipher;
66  import org.apache.hadoop.hbase.io.crypto.Encryption;
67  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
68  import org.apache.hadoop.hbase.io.hfile.HFile;
69  import org.apache.hadoop.hbase.io.hfile.HFileContext;
70  import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
71  import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
72  import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
73  import org.apache.hadoop.hbase.io.hfile.HFileScanner;
74  import org.apache.hadoop.hbase.io.hfile.InvalidHFileException;
75  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
76  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
77  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
78  import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
79  import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
80  import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
81  import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
82  import org.apache.hadoop.hbase.regionserver.compactions.OffPeakHours;
83  import org.apache.hadoop.hbase.regionserver.compactions.CompactionThroughputController;
84  import org.apache.hadoop.hbase.regionserver.wal.WALUtil;
85  import org.apache.hadoop.hbase.security.EncryptionUtil;
86  import org.apache.hadoop.hbase.security.User;
87  import org.apache.hadoop.hbase.util.Bytes;
88  import org.apache.hadoop.hbase.util.ChecksumType;
89  import org.apache.hadoop.hbase.util.ClassSize;
90  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
91  import org.apache.hadoop.hbase.util.Pair;
92  import org.apache.hadoop.hbase.util.ReflectionUtils;
93  import org.apache.hadoop.ipc.RemoteException;
94  import org.apache.hadoop.util.StringUtils;
95  import org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix;
96  
97  import com.google.common.annotations.VisibleForTesting;
98  import com.google.common.base.Preconditions;
99  import com.google.common.collect.ImmutableCollection;
100 import com.google.common.collect.ImmutableList;
101 import com.google.common.collect.Lists;
102 import com.google.common.collect.Sets;
103 
104 /**
105  * A Store holds a column family in a Region.  Its a memstore and a set of zero
106  * or more StoreFiles, which stretch backwards over time.
107  *
108  * <p>There's no reason to consider append-logging at this level; all logging
109  * and locking is handled at the HRegion level.  Store just provides
110  * services to manage sets of StoreFiles.  One of the most important of those
111  * services is compaction services where files are aggregated once they pass
112  * a configurable threshold.
113  *
114  * <p>Locking and transactions are handled at a higher level.  This API should
115  * not be called directly but by an HRegion manager.
116  */
117 @InterfaceAudience.Private
118 public class HStore implements Store {
119   private static final String MEMSTORE_CLASS_NAME = "hbase.regionserver.memstore.class";
120   public static final String COMPACTCHECKER_INTERVAL_MULTIPLIER_KEY =
121       "hbase.server.compactchecker.interval.multiplier";
122   public static final String BLOCKING_STOREFILES_KEY = "hbase.hstore.blockingStoreFiles";
123   public static final int DEFAULT_COMPACTCHECKER_INTERVAL_MULTIPLIER = 1000;
124   public static final int DEFAULT_BLOCKING_STOREFILE_COUNT = 7;
125 
126   private static final Log LOG = LogFactory.getLog(HStore.class);
127 
128   protected final MemStore memstore;
129   // This stores directory in the filesystem.
130   protected final HRegion region;
131   private final HColumnDescriptor family;
132   private final HRegionFileSystem fs;
133   protected Configuration conf;
134   protected CacheConfig cacheConf;
135   private long lastCompactSize = 0;
136   volatile boolean forceMajor = false;
137   /* how many bytes to write between status checks */
138   static int closeCheckInterval = 0;
139   private volatile long storeSize = 0L;
140   private volatile long totalUncompressedBytes = 0L;
141 
142   /**
143    * RWLock for store operations.
144    * Locked in shared mode when the list of component stores is looked at:
145    *   - all reads/writes to table data
146    *   - checking for split
147    * Locked in exclusive mode when the list of component stores is modified:
148    *   - closing
149    *   - completing a compaction
150    */
151   final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
152   private final boolean verifyBulkLoads;
153 
154   private ScanInfo scanInfo;
155 
156   // TODO: ideally, this should be part of storeFileManager, as we keep passing this to it.
157   final List<StoreFile> filesCompacting = Lists.newArrayList();
158 
159   // All access must be synchronized.
160   private final Set<ChangedReadersObserver> changedReaderObservers =
161     Collections.newSetFromMap(new ConcurrentHashMap<ChangedReadersObserver, Boolean>());
162 
163   protected final int blocksize;
164   private HFileDataBlockEncoder dataBlockEncoder;
165 
166   /** Checksum configuration */
167   protected ChecksumType checksumType;
168   protected int bytesPerChecksum;
169 
170   // Comparing KeyValues
171   private final CellComparator comparator;
172 
173   final StoreEngine<?, ?, ?, ?> storeEngine;
174 
175   private static final AtomicBoolean offPeakCompactionTracker = new AtomicBoolean();
176   private volatile OffPeakHours offPeakHours;
177 
178   private static final int DEFAULT_FLUSH_RETRIES_NUMBER = 10;
179   private int flushRetriesNumber;
180   private int pauseTime;
181 
182   private long blockingFileCount;
183   private int compactionCheckMultiplier;
184 
185   protected Encryption.Context cryptoContext = Encryption.Context.NONE;
186 
187   private volatile long flushedCellsCount = 0;
188   private volatile long compactedCellsCount = 0;
189   private volatile long majorCompactedCellsCount = 0;
190   private volatile long flushedCellsSize = 0;
191   private volatile long compactedCellsSize = 0;
192   private volatile long majorCompactedCellsSize = 0;
193 
194   /**
195    * Constructor
196    * @param region
197    * @param family HColumnDescriptor for this column
198    * @param confParam configuration object
199    * failed.  Can be null.
200    * @throws IOException
201    */
202   protected HStore(final HRegion region, final HColumnDescriptor family,
203       final Configuration confParam) throws IOException {
204 
205     this.fs = region.getRegionFileSystem();
206 
207     // Assemble the store's home directory and Ensure it exists.
208     fs.createStoreDir(family.getNameAsString());
209     this.region = region;
210     this.family = family;
211     // 'conf' renamed to 'confParam' b/c we use this.conf in the constructor
212     // CompoundConfiguration will look for keys in reverse order of addition, so we'd
213     // add global config first, then table and cf overrides, then cf metadata.
214     this.conf = new CompoundConfiguration()
215       .add(confParam)
216       .addStringMap(region.getTableDesc().getConfiguration())
217       .addStringMap(family.getConfiguration())
218       .addBytesMap(family.getValues());
219     this.blocksize = family.getBlocksize();
220 
221     this.dataBlockEncoder =
222         new HFileDataBlockEncoderImpl(family.getDataBlockEncoding());
223 
224     this.comparator = region.getCellCompartor();
225     // used by ScanQueryMatcher
226     long timeToPurgeDeletes =
227         Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0);
228     LOG.trace("Time to purge deletes set to " + timeToPurgeDeletes +
229         "ms in store " + this);
230     // Get TTL
231     long ttl = determineTTLFromFamily(family);
232     // Why not just pass a HColumnDescriptor in here altogether?  Even if have
233     // to clone it?
234     scanInfo = new ScanInfo(family, ttl, timeToPurgeDeletes, this.comparator);
235     String className = conf.get(MEMSTORE_CLASS_NAME, DefaultMemStore.class.getName());
236     this.memstore = ReflectionUtils.instantiateWithCustomCtor(className, new Class[] {
237         Configuration.class, CellComparator.class }, new Object[] { conf, this.comparator });
238     this.offPeakHours = OffPeakHours.getInstance(conf);
239 
240     // Setting up cache configuration for this family
241     createCacheConf(family);
242 
243     this.verifyBulkLoads = conf.getBoolean("hbase.hstore.bulkload.verify", false);
244 
245     this.blockingFileCount =
246         conf.getInt(BLOCKING_STOREFILES_KEY, DEFAULT_BLOCKING_STOREFILE_COUNT);
247     this.compactionCheckMultiplier = conf.getInt(
248         COMPACTCHECKER_INTERVAL_MULTIPLIER_KEY, DEFAULT_COMPACTCHECKER_INTERVAL_MULTIPLIER);
249     if (this.compactionCheckMultiplier <= 0) {
250       LOG.error("Compaction check period multiplier must be positive, setting default: "
251           + DEFAULT_COMPACTCHECKER_INTERVAL_MULTIPLIER);
252       this.compactionCheckMultiplier = DEFAULT_COMPACTCHECKER_INTERVAL_MULTIPLIER;
253     }
254 
255     if (HStore.closeCheckInterval == 0) {
256       HStore.closeCheckInterval = conf.getInt(
257           "hbase.hstore.close.check.interval", 10*1000*1000 /* 10 MB */);
258     }
259 
260     this.storeEngine = createStoreEngine(this, this.conf, this.comparator);
261     this.storeEngine.getStoreFileManager().loadFiles(loadStoreFiles());
262 
263     // Initialize checksum type from name. The names are CRC32, CRC32C, etc.
264     this.checksumType = getChecksumType(conf);
265     // initilize bytes per checksum
266     this.bytesPerChecksum = getBytesPerChecksum(conf);
267     flushRetriesNumber = conf.getInt(
268         "hbase.hstore.flush.retries.number", DEFAULT_FLUSH_RETRIES_NUMBER);
269     pauseTime = conf.getInt(HConstants.HBASE_SERVER_PAUSE, HConstants.DEFAULT_HBASE_SERVER_PAUSE);
270     if (flushRetriesNumber <= 0) {
271       throw new IllegalArgumentException(
272           "hbase.hstore.flush.retries.number must be > 0, not "
273               + flushRetriesNumber);
274     }
275 
276     // Crypto context for new store files
277     String cipherName = family.getEncryptionType();
278     if (cipherName != null) {
279       Cipher cipher;
280       Key key;
281       byte[] keyBytes = family.getEncryptionKey();
282       if (keyBytes != null) {
283         // Family provides specific key material
284         String masterKeyName = conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY,
285           User.getCurrent().getShortName());
286         try {
287           // First try the master key
288           key = EncryptionUtil.unwrapKey(conf, masterKeyName, keyBytes);
289         } catch (KeyException e) {
290           // If the current master key fails to unwrap, try the alternate, if
291           // one is configured
292           if (LOG.isDebugEnabled()) {
293             LOG.debug("Unable to unwrap key with current master key '" + masterKeyName + "'");
294           }
295           String alternateKeyName =
296             conf.get(HConstants.CRYPTO_MASTERKEY_ALTERNATE_NAME_CONF_KEY);
297           if (alternateKeyName != null) {
298             try {
299               key = EncryptionUtil.unwrapKey(conf, alternateKeyName, keyBytes);
300             } catch (KeyException ex) {
301               throw new IOException(ex);
302             }
303           } else {
304             throw new IOException(e);
305           }
306         }
307         // Use the algorithm the key wants
308         cipher = Encryption.getCipher(conf, key.getAlgorithm());
309         if (cipher == null) {
310           throw new RuntimeException("Cipher '" + key.getAlgorithm() + "' is not available");
311         }
312         // Fail if misconfigured
313         // We use the encryption type specified in the column schema as a sanity check on
314         // what the wrapped key is telling us
315         if (!cipher.getName().equalsIgnoreCase(cipherName)) {
316           throw new RuntimeException("Encryption for family '" + family.getNameAsString() +
317             "' configured with type '" + cipherName +
318             "' but key specifies algorithm '" + cipher.getName() + "'");
319         }
320       } else {
321         // Family does not provide key material, create a random key
322         cipher = Encryption.getCipher(conf, cipherName);
323         if (cipher == null) {
324           throw new RuntimeException("Cipher '" + cipherName + "' is not available");
325         }
326         key = cipher.getRandomKey();
327       }
328       cryptoContext = Encryption.newContext(conf);
329       cryptoContext.setCipher(cipher);
330       cryptoContext.setKey(key);
331     }
332   }
333 
334   /**
335    * Creates the cache config.
336    * @param family The current column family.
337    */
338   protected void createCacheConf(final HColumnDescriptor family) {
339     this.cacheConf = new CacheConfig(conf, family);
340   }
341 
342   /**
343    * Creates the store engine configured for the given Store.
344    * @param store The store. An unfortunate dependency needed due to it
345    *              being passed to coprocessors via the compactor.
346    * @param conf Store configuration.
347    * @param kvComparator KVComparator for storeFileManager.
348    * @return StoreEngine to use.
349    */
350   protected StoreEngine<?, ?, ?, ?> createStoreEngine(Store store, Configuration conf,
351       CellComparator kvComparator) throws IOException {
352     return StoreEngine.create(store, conf, comparator);
353   }
354 
355   /**
356    * @param family
357    * @return TTL in seconds of the specified family
358    */
359   public static long determineTTLFromFamily(final HColumnDescriptor family) {
360     // HCD.getTimeToLive returns ttl in seconds.  Convert to milliseconds.
361     long ttl = family.getTimeToLive();
362     if (ttl == HConstants.FOREVER) {
363       // Default is unlimited ttl.
364       ttl = Long.MAX_VALUE;
365     } else if (ttl == -1) {
366       ttl = Long.MAX_VALUE;
367     } else {
368       // Second -> ms adjust for user data
369       ttl *= 1000;
370     }
371     return ttl;
372   }
373 
374   @Override
375   public String getColumnFamilyName() {
376     return this.family.getNameAsString();
377   }
378 
379   @Override
380   public TableName getTableName() {
381     return this.getRegionInfo().getTable();
382   }
383 
384   @Override
385   public FileSystem getFileSystem() {
386     return this.fs.getFileSystem();
387   }
388 
389   public HRegionFileSystem getRegionFileSystem() {
390     return this.fs;
391   }
392 
393   /* Implementation of StoreConfigInformation */
394   @Override
395   public long getStoreFileTtl() {
396     // TTL only applies if there's no MIN_VERSIONs setting on the column.
397     return (this.scanInfo.getMinVersions() == 0) ? this.scanInfo.getTtl() : Long.MAX_VALUE;
398   }
399 
400   @Override
401   public long getMemstoreFlushSize() {
402     // TODO: Why is this in here?  The flushsize of the region rather than the store?  St.Ack
403     return this.region.memstoreFlushSize;
404   }
405 
406   @Override
407   public long getFlushableSize() {
408     return this.memstore.getFlushableSize();
409   }
410 
411   @Override
412   public long getSnapshotSize() {
413     return this.memstore.getSnapshotSize();
414   }
415 
416   @Override
417   public long getCompactionCheckMultiplier() {
418     return this.compactionCheckMultiplier;
419   }
420 
421   @Override
422   public long getBlockingFileCount() {
423     return blockingFileCount;
424   }
425   /* End implementation of StoreConfigInformation */
426 
427   /**
428    * Returns the configured bytesPerChecksum value.
429    * @param conf The configuration
430    * @return The bytesPerChecksum that is set in the configuration
431    */
432   public static int getBytesPerChecksum(Configuration conf) {
433     return conf.getInt(HConstants.BYTES_PER_CHECKSUM,
434                        HFile.DEFAULT_BYTES_PER_CHECKSUM);
435   }
436 
437   /**
438    * Returns the configured checksum algorithm.
439    * @param conf The configuration
440    * @return The checksum algorithm that is set in the configuration
441    */
442   public static ChecksumType getChecksumType(Configuration conf) {
443     String checksumName = conf.get(HConstants.CHECKSUM_TYPE_NAME);
444     if (checksumName == null) {
445       return ChecksumType.getDefaultChecksumType();
446     } else {
447       return ChecksumType.nameToType(checksumName);
448     }
449   }
450 
451   /**
452    * @return how many bytes to write between status checks
453    */
454   public static int getCloseCheckInterval() {
455     return closeCheckInterval;
456   }
457 
458   @Override
459   public HColumnDescriptor getFamily() {
460     return this.family;
461   }
462 
463   /**
464    * @return The maximum sequence id in all store files. Used for log replay.
465    */
466   @Override
467   public long getMaxSequenceId() {
468     return StoreFile.getMaxSequenceIdInList(this.getStorefiles());
469   }
470 
471   @Override
472   public long getMaxMemstoreTS() {
473     return StoreFile.getMaxMemstoreTSInList(this.getStorefiles());
474   }
475 
476   /**
477    * @param tabledir {@link Path} to where the table is being stored
478    * @param hri {@link HRegionInfo} for the region.
479    * @param family {@link HColumnDescriptor} describing the column family
480    * @return Path to family/Store home directory.
481    */
482   @Deprecated
483   public static Path getStoreHomedir(final Path tabledir,
484       final HRegionInfo hri, final byte[] family) {
485     return getStoreHomedir(tabledir, hri.getEncodedName(), family);
486   }
487 
488   /**
489    * @param tabledir {@link Path} to where the table is being stored
490    * @param encodedName Encoded region name.
491    * @param family {@link HColumnDescriptor} describing the column family
492    * @return Path to family/Store home directory.
493    */
494   @Deprecated
495   public static Path getStoreHomedir(final Path tabledir,
496       final String encodedName, final byte[] family) {
497     return new Path(tabledir, new Path(encodedName, Bytes.toString(family)));
498   }
499 
500   @Override
501   public HFileDataBlockEncoder getDataBlockEncoder() {
502     return dataBlockEncoder;
503   }
504 
505   /**
506    * Should be used only in tests.
507    * @param blockEncoder the block delta encoder to use
508    */
509   void setDataBlockEncoderInTest(HFileDataBlockEncoder blockEncoder) {
510     this.dataBlockEncoder = blockEncoder;
511   }
512 
513   /**
514    * Creates an unsorted list of StoreFile loaded in parallel
515    * from the given directory.
516    * @throws IOException
517    */
518   private List<StoreFile> loadStoreFiles() throws IOException {
519     Collection<StoreFileInfo> files = fs.getStoreFiles(getColumnFamilyName());
520     return openStoreFiles(files);
521   }
522 
523   private List<StoreFile> openStoreFiles(Collection<StoreFileInfo> files) throws IOException {
524     if (files == null || files.size() == 0) {
525       return new ArrayList<StoreFile>();
526     }
527     // initialize the thread pool for opening store files in parallel..
528     ThreadPoolExecutor storeFileOpenerThreadPool =
529       this.region.getStoreFileOpenAndCloseThreadPool("StoreFileOpenerThread-" +
530           this.getColumnFamilyName());
531     CompletionService<StoreFile> completionService =
532       new ExecutorCompletionService<StoreFile>(storeFileOpenerThreadPool);
533 
534     int totalValidStoreFile = 0;
535     for (final StoreFileInfo storeFileInfo: files) {
536       // open each store file in parallel
537       completionService.submit(new Callable<StoreFile>() {
538         @Override
539         public StoreFile call() throws IOException {
540           StoreFile storeFile = createStoreFileAndReader(storeFileInfo);
541           return storeFile;
542         }
543       });
544       totalValidStoreFile++;
545     }
546 
547     ArrayList<StoreFile> results = new ArrayList<StoreFile>(files.size());
548     IOException ioe = null;
549     try {
550       for (int i = 0; i < totalValidStoreFile; i++) {
551         try {
552           Future<StoreFile> future = completionService.take();
553           StoreFile storeFile = future.get();
554           long length = storeFile.getReader().length();
555           this.storeSize += length;
556           this.totalUncompressedBytes +=
557               storeFile.getReader().getTotalUncompressedBytes();
558           if (LOG.isDebugEnabled()) {
559             LOG.debug("loaded " + storeFile.toStringDetailed());
560           }
561           results.add(storeFile);
562         } catch (InterruptedException e) {
563           if (ioe == null) ioe = new InterruptedIOException(e.getMessage());
564         } catch (ExecutionException e) {
565           if (ioe == null) ioe = new IOException(e.getCause());
566         }
567       }
568     } finally {
569       storeFileOpenerThreadPool.shutdownNow();
570     }
571     if (ioe != null) {
572       // close StoreFile readers
573       for (StoreFile file : results) {
574         try {
575           if (file != null) file.closeReader(true);
576         } catch (IOException e) {
577           LOG.warn(e.getMessage());
578         }
579       }
580       throw ioe;
581     }
582 
583     return results;
584   }
585 
586   /**
587    * Checks the underlying store files, and opens the files that  have not
588    * been opened, and removes the store file readers for store files no longer
589    * available. Mainly used by secondary region replicas to keep up to date with
590    * the primary region files.
591    * @throws IOException
592    */
593   @Override
594   public void refreshStoreFiles() throws IOException {
595     Collection<StoreFileInfo> newFiles = fs.getStoreFiles(getColumnFamilyName());
596     refreshStoreFilesInternal(newFiles);
597   }
598 
599   @Override
600   public void refreshStoreFiles(Collection<String> newFiles) throws IOException {
601     List<StoreFileInfo> storeFiles = new ArrayList<StoreFileInfo>(newFiles.size());
602     for (String file : newFiles) {
603       storeFiles.add(fs.getStoreFileInfo(getColumnFamilyName(), file));
604     }
605     refreshStoreFilesInternal(storeFiles);
606   }
607 
608   /**
609    * Checks the underlying store files, and opens the files that  have not
610    * been opened, and removes the store file readers for store files no longer
611    * available. Mainly used by secondary region replicas to keep up to date with
612    * the primary region files.
613    * @throws IOException
614    */
615   private void refreshStoreFilesInternal(Collection<StoreFileInfo> newFiles) throws IOException {
616     StoreFileManager sfm = storeEngine.getStoreFileManager();
617     Collection<StoreFile> currentFiles = sfm.getStorefiles();
618     if (currentFiles == null) currentFiles = new ArrayList<StoreFile>(0);
619 
620     if (newFiles == null) newFiles = new ArrayList<StoreFileInfo>(0);
621 
622     HashMap<StoreFileInfo, StoreFile> currentFilesSet = new HashMap<StoreFileInfo, StoreFile>(currentFiles.size());
623     for (StoreFile sf : currentFiles) {
624       currentFilesSet.put(sf.getFileInfo(), sf);
625     }
626     HashSet<StoreFileInfo> newFilesSet = new HashSet<StoreFileInfo>(newFiles);
627 
628     Set<StoreFileInfo> toBeAddedFiles = Sets.difference(newFilesSet, currentFilesSet.keySet());
629     Set<StoreFileInfo> toBeRemovedFiles = Sets.difference(currentFilesSet.keySet(), newFilesSet);
630 
631     if (toBeAddedFiles.isEmpty() && toBeRemovedFiles.isEmpty()) {
632       return;
633     }
634 
635     LOG.info("Refreshing store files for region " + this.getRegionInfo().getRegionNameAsString()
636       + " files to add: " + toBeAddedFiles + " files to remove: " + toBeRemovedFiles);
637 
638     Set<StoreFile> toBeRemovedStoreFiles = new HashSet<StoreFile>(toBeRemovedFiles.size());
639     for (StoreFileInfo sfi : toBeRemovedFiles) {
640       toBeRemovedStoreFiles.add(currentFilesSet.get(sfi));
641     }
642 
643     // try to open the files
644     List<StoreFile> openedFiles = openStoreFiles(toBeAddedFiles);
645 
646     // propogate the file changes to the underlying store file manager
647     replaceStoreFiles(toBeRemovedStoreFiles, openedFiles); //won't throw an exception
648 
649     // Advance the memstore read point to be at least the new store files seqIds so that
650     // readers might pick it up. This assumes that the store is not getting any writes (otherwise
651     // in-flight transactions might be made visible)
652     if (!toBeAddedFiles.isEmpty()) {
653       region.getMVCC().advanceMemstoreReadPointIfNeeded(this.getMaxSequenceId());
654     }
655 
656     // notify scanners, close file readers, and recompute store size
657     completeCompaction(toBeRemovedStoreFiles, false);
658   }
659 
660   private StoreFile createStoreFileAndReader(final Path p) throws IOException {
661     StoreFileInfo info = new StoreFileInfo(conf, this.getFileSystem(), p);
662     return createStoreFileAndReader(info);
663   }
664 
665   private StoreFile createStoreFileAndReader(final StoreFileInfo info)
666       throws IOException {
667     info.setRegionCoprocessorHost(this.region.getCoprocessorHost());
668     StoreFile storeFile = new StoreFile(this.getFileSystem(), info, this.conf, this.cacheConf,
669       this.family.getBloomFilterType());
670     storeFile.createReader();
671     return storeFile;
672   }
673 
674   @Override
675   public Pair<Long, Cell> add(final Cell cell) {
676     lock.readLock().lock();
677     try {
678        return this.memstore.add(cell);
679     } finally {
680       lock.readLock().unlock();
681     }
682   }
683 
684   @Override
685   public long timeOfOldestEdit() {
686     return memstore.timeOfOldestEdit();
687   }
688 
689   /**
690    * Adds a value to the memstore
691    *
692    * @param kv
693    * @return memstore size delta
694    */
695   protected long delete(final KeyValue kv) {
696     lock.readLock().lock();
697     try {
698       return this.memstore.delete(kv);
699     } finally {
700       lock.readLock().unlock();
701     }
702   }
703 
704   @Override
705   public void rollback(final Cell cell) {
706     lock.readLock().lock();
707     try {
708       this.memstore.rollback(cell);
709     } finally {
710       lock.readLock().unlock();
711     }
712   }
713 
714   /**
715    * @return All store files.
716    */
717   @Override
718   public Collection<StoreFile> getStorefiles() {
719     return this.storeEngine.getStoreFileManager().getStorefiles();
720   }
721 
722   @Override
723   public void assertBulkLoadHFileOk(Path srcPath) throws IOException {
724     HFile.Reader reader  = null;
725     try {
726       LOG.info("Validating hfile at " + srcPath + " for inclusion in "
727           + "store " + this + " region " + this.getRegionInfo().getRegionNameAsString());
728       reader = HFile.createReader(srcPath.getFileSystem(conf),
729           srcPath, cacheConf, conf);
730       reader.loadFileInfo();
731 
732       byte[] firstKey = reader.getFirstRowKey();
733       Preconditions.checkState(firstKey != null, "First key can not be null");
734       Cell lk = reader.getLastKey();
735       Preconditions.checkState(lk != null, "Last key can not be null");
736       byte[] lastKey =  CellUtil.cloneRow(lk);
737 
738       LOG.debug("HFile bounds: first=" + Bytes.toStringBinary(firstKey) +
739           " last=" + Bytes.toStringBinary(lastKey));
740       LOG.debug("Region bounds: first=" +
741           Bytes.toStringBinary(getRegionInfo().getStartKey()) +
742           " last=" + Bytes.toStringBinary(getRegionInfo().getEndKey()));
743 
744       if (!this.getRegionInfo().containsRange(firstKey, lastKey)) {
745         throw new WrongRegionException(
746             "Bulk load file " + srcPath.toString() + " does not fit inside region "
747             + this.getRegionInfo().getRegionNameAsString());
748       }
749 
750       if(reader.length() > conf.getLong(HConstants.HREGION_MAX_FILESIZE,
751           HConstants.DEFAULT_MAX_FILE_SIZE)) {
752         LOG.warn("Trying to bulk load hfile " + srcPath.toString() + " with size: " +
753             reader.length() + " bytes can be problematic as it may lead to oversplitting.");
754       }
755 
756       if (verifyBulkLoads) {
757         long verificationStartTime = EnvironmentEdgeManager.currentTime();
758         LOG.info("Full verification started for bulk load hfile: " + srcPath.toString());
759         Cell prevCell = null;
760         HFileScanner scanner = reader.getScanner(false, false, false);
761         scanner.seekTo();
762         do {
763           Cell cell = scanner.getCell();
764           if (prevCell != null) {
765             if (comparator.compareRows(prevCell, cell) > 0) {
766               throw new InvalidHFileException("Previous row is greater than"
767                   + " current row: path=" + srcPath + " previous="
768                   + CellUtil.getCellKeyAsString(prevCell) + " current="
769                   + CellUtil.getCellKeyAsString(cell));
770             }
771             if (CellComparator.compareFamilies(prevCell, cell) != 0) {
772               throw new InvalidHFileException("Previous key had different"
773                   + " family compared to current key: path=" + srcPath
774                   + " previous="
775                   + Bytes.toStringBinary(prevCell.getFamilyArray(), prevCell.getFamilyOffset(),
776                       prevCell.getFamilyLength())
777                   + " current="
778                   + Bytes.toStringBinary(cell.getFamilyArray(), cell.getFamilyOffset(),
779                       cell.getFamilyLength()));
780             }
781           }
782           prevCell = cell;
783         } while (scanner.next());
784       LOG.info("Full verification complete for bulk load hfile: " + srcPath.toString()
785          + " took " + (EnvironmentEdgeManager.currentTime() - verificationStartTime)
786          + " ms");
787       }
788     } finally {
789       if (reader != null) reader.close();
790     }
791   }
792 
793   @Override
794   public Path bulkLoadHFile(String srcPathStr, long seqNum) throws IOException {
795     Path srcPath = new Path(srcPathStr);
796     Path dstPath = fs.bulkLoadStoreFile(getColumnFamilyName(), srcPath, seqNum);
797 
798     LOG.info("Loaded HFile " + srcPath + " into store '" + getColumnFamilyName() + "' as "
799         + dstPath + " - updating store file list.");
800 
801     StoreFile sf = createStoreFileAndReader(dstPath);
802     bulkLoadHFile(sf);
803 
804     LOG.info("Successfully loaded store file " + srcPath + " into store " + this
805         + " (new location: " + dstPath + ")");
806 
807     return dstPath;
808   }
809 
810   @Override
811   public void bulkLoadHFile(StoreFileInfo fileInfo) throws IOException {
812     StoreFile sf = createStoreFileAndReader(fileInfo);
813     bulkLoadHFile(sf);
814   }
815 
816   private void bulkLoadHFile(StoreFile sf) throws IOException {
817     StoreFile.Reader r = sf.getReader();
818     this.storeSize += r.length();
819     this.totalUncompressedBytes += r.getTotalUncompressedBytes();
820 
821     // Append the new storefile into the list
822     this.lock.writeLock().lock();
823     try {
824       this.storeEngine.getStoreFileManager().insertNewFiles(Lists.newArrayList(sf));
825     } finally {
826       // We need the lock, as long as we are updating the storeFiles
827       // or changing the memstore. Let us release it before calling
828       // notifyChangeReadersObservers. See HBASE-4485 for a possible
829       // deadlock scenario that could have happened if continue to hold
830       // the lock.
831       this.lock.writeLock().unlock();
832     }
833     notifyChangedReadersObservers();
834     LOG.info("Loaded HFile " + sf.getFileInfo() + " into store '" + getColumnFamilyName());
835     if (LOG.isTraceEnabled()) {
836       String traceMessage = "BULK LOAD time,size,store size,store files ["
837           + EnvironmentEdgeManager.currentTime() + "," + r.length() + "," + storeSize
838           + "," + storeEngine.getStoreFileManager().getStorefileCount() + "]";
839       LOG.trace(traceMessage);
840     }
841   }
842 
843   @Override
844   public ImmutableCollection<StoreFile> close() throws IOException {
845     this.lock.writeLock().lock();
846     try {
847       // Clear so metrics doesn't find them.
848       ImmutableCollection<StoreFile> result = storeEngine.getStoreFileManager().clearFiles();
849 
850       if (!result.isEmpty()) {
851         // initialize the thread pool for closing store files in parallel.
852         ThreadPoolExecutor storeFileCloserThreadPool = this.region
853             .getStoreFileOpenAndCloseThreadPool("StoreFileCloserThread-"
854                 + this.getColumnFamilyName());
855 
856         // close each store file in parallel
857         CompletionService<Void> completionService =
858           new ExecutorCompletionService<Void>(storeFileCloserThreadPool);
859         for (final StoreFile f : result) {
860           completionService.submit(new Callable<Void>() {
861             @Override
862             public Void call() throws IOException {
863               f.closeReader(true);
864               return null;
865             }
866           });
867         }
868 
869         IOException ioe = null;
870         try {
871           for (int i = 0; i < result.size(); i++) {
872             try {
873               Future<Void> future = completionService.take();
874               future.get();
875             } catch (InterruptedException e) {
876               if (ioe == null) {
877                 ioe = new InterruptedIOException();
878                 ioe.initCause(e);
879               }
880             } catch (ExecutionException e) {
881               if (ioe == null) ioe = new IOException(e.getCause());
882             }
883           }
884         } finally {
885           storeFileCloserThreadPool.shutdownNow();
886         }
887         if (ioe != null) throw ioe;
888       }
889       LOG.info("Closed " + this);
890       return result;
891     } finally {
892       this.lock.writeLock().unlock();
893     }
894   }
895 
896   /**
897    * Snapshot this stores memstore. Call before running
898    * {@link #flushCache(long, MemStoreSnapshot, MonitoredTask)}
899    *  so it has some work to do.
900    */
901   void snapshot() {
902     this.lock.writeLock().lock();
903     try {
904       this.memstore.snapshot();
905     } finally {
906       this.lock.writeLock().unlock();
907     }
908   }
909 
910   /**
911    * Write out current snapshot.  Presumes {@link #snapshot()} has been called previously.
912    * @param logCacheFlushId flush sequence number
913    * @param snapshot
914    * @param status
915    * @return The path name of the tmp file to which the store was flushed
916    * @throws IOException
917    */
918   protected List<Path> flushCache(final long logCacheFlushId, MemStoreSnapshot snapshot,
919       MonitoredTask status) throws IOException {
920     // If an exception happens flushing, we let it out without clearing
921     // the memstore snapshot.  The old snapshot will be returned when we say
922     // 'snapshot', the next time flush comes around.
923     // Retry after catching exception when flushing, otherwise server will abort
924     // itself
925     StoreFlusher flusher = storeEngine.getStoreFlusher();
926     IOException lastException = null;
927     for (int i = 0; i < flushRetriesNumber; i++) {
928       try {
929         List<Path> pathNames = flusher.flushSnapshot(snapshot, logCacheFlushId, status);
930         Path lastPathName = null;
931         try {
932           for (Path pathName : pathNames) {
933             lastPathName = pathName;
934             validateStoreFile(pathName);
935           }
936           return pathNames;
937         } catch (Exception e) {
938           LOG.warn("Failed validating store file " + lastPathName + ", retrying num=" + i, e);
939           if (e instanceof IOException) {
940             lastException = (IOException) e;
941           } else {
942             lastException = new IOException(e);
943           }
944         }
945       } catch (IOException e) {
946         LOG.warn("Failed flushing store file, retrying num=" + i, e);
947         lastException = e;
948       }
949       if (lastException != null && i < (flushRetriesNumber - 1)) {
950         try {
951           Thread.sleep(pauseTime);
952         } catch (InterruptedException e) {
953           IOException iie = new InterruptedIOException();
954           iie.initCause(e);
955           throw iie;
956         }
957       }
958     }
959     throw lastException;
960   }
961 
962   /*
963    * @param path The pathname of the tmp file into which the store was flushed
964    * @param logCacheFlushId
965    * @param status
966    * @return StoreFile created.
967    * @throws IOException
968    */
969   private StoreFile commitFile(final Path path, final long logCacheFlushId, MonitoredTask status)
970       throws IOException {
971     // Write-out finished successfully, move into the right spot
972     Path dstPath = fs.commitStoreFile(getColumnFamilyName(), path);
973 
974     status.setStatus("Flushing " + this + ": reopening flushed file");
975     StoreFile sf = createStoreFileAndReader(dstPath);
976 
977     StoreFile.Reader r = sf.getReader();
978     this.storeSize += r.length();
979     this.totalUncompressedBytes += r.getTotalUncompressedBytes();
980 
981     if (LOG.isInfoEnabled()) {
982       LOG.info("Added " + sf + ", entries=" + r.getEntries() +
983         ", sequenceid=" + logCacheFlushId +
984         ", filesize=" + TraditionalBinaryPrefix.long2String(r.length(), "", 1));
985     }
986     return sf;
987   }
988 
989   @Override
990   public StoreFile.Writer createWriterInTmp(long maxKeyCount, Compression.Algorithm compression,
991                                             boolean isCompaction, boolean includeMVCCReadpoint,
992                                             boolean includesTag)
993       throws IOException {
994     return createWriterInTmp(maxKeyCount, compression, isCompaction, includeMVCCReadpoint,
995         includesTag, false);
996   }
997 
998   /*
999    * @param maxKeyCount
1000    * @param compression Compression algorithm to use
1001    * @param isCompaction whether we are creating a new file in a compaction
1002    * @param includesMVCCReadPoint - whether to include MVCC or not
1003    * @param includesTag - includesTag or not
1004    * @return Writer for a new StoreFile in the tmp dir.
1005    */
1006   @Override
1007   public StoreFile.Writer createWriterInTmp(long maxKeyCount, Compression.Algorithm compression,
1008       boolean isCompaction, boolean includeMVCCReadpoint, boolean includesTag,
1009       boolean shouldDropBehind)
1010   throws IOException {
1011     final CacheConfig writerCacheConf;
1012     if (isCompaction) {
1013       // Don't cache data on write on compactions.
1014       writerCacheConf = new CacheConfig(cacheConf);
1015       writerCacheConf.setCacheDataOnWrite(false);
1016     } else {
1017       writerCacheConf = cacheConf;
1018     }
1019     InetSocketAddress[] favoredNodes = null;
1020     if (region.getRegionServerServices() != null) {
1021       favoredNodes = region.getRegionServerServices().getFavoredNodesForRegion(
1022           region.getRegionInfo().getEncodedName());
1023     }
1024     HFileContext hFileContext = createFileContext(compression, includeMVCCReadpoint, includesTag,
1025       cryptoContext);
1026     StoreFile.Writer w = new StoreFile.WriterBuilder(conf, writerCacheConf,
1027         this.getFileSystem())
1028             .withFilePath(fs.createTempName())
1029             .withComparator(comparator)
1030             .withBloomType(family.getBloomFilterType())
1031             .withMaxKeyCount(maxKeyCount)
1032             .withFavoredNodes(favoredNodes)
1033             .withFileContext(hFileContext)
1034             .withShouldDropCacheBehind(shouldDropBehind)
1035             .build();
1036     return w;
1037   }
1038 
1039   private HFileContext createFileContext(Compression.Algorithm compression,
1040       boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context cryptoContext) {
1041     if (compression == null) {
1042       compression = HFile.DEFAULT_COMPRESSION_ALGORITHM;
1043     }
1044     HFileContext hFileContext = new HFileContextBuilder()
1045                                 .withIncludesMvcc(includeMVCCReadpoint)
1046                                 .withIncludesTags(includesTag)
1047                                 .withCompression(compression)
1048                                 .withCompressTags(family.isCompressTags())
1049                                 .withChecksumType(checksumType)
1050                                 .withBytesPerCheckSum(bytesPerChecksum)
1051                                 .withBlockSize(blocksize)
1052                                 .withHBaseCheckSum(true)
1053                                 .withDataBlockEncoding(family.getDataBlockEncoding())
1054                                 .withEncryptionContext(cryptoContext)
1055                                 .withCreateTime(EnvironmentEdgeManager.currentTime())
1056                                 .build();
1057     return hFileContext;
1058   }
1059 
1060 
1061   /*
1062    * Change storeFiles adding into place the Reader produced by this new flush.
1063    * @param sfs Store files
1064    * @param snapshotId
1065    * @throws IOException
1066    * @return Whether compaction is required.
1067    */
1068   private boolean updateStorefiles(final List<StoreFile> sfs, final long snapshotId)
1069       throws IOException {
1070     this.lock.writeLock().lock();
1071     try {
1072       this.storeEngine.getStoreFileManager().insertNewFiles(sfs);
1073       if (snapshotId > 0) {
1074         this.memstore.clearSnapshot(snapshotId);
1075       }
1076     } finally {
1077       // We need the lock, as long as we are updating the storeFiles
1078       // or changing the memstore. Let us release it before calling
1079       // notifyChangeReadersObservers. See HBASE-4485 for a possible
1080       // deadlock scenario that could have happened if continue to hold
1081       // the lock.
1082       this.lock.writeLock().unlock();
1083     }
1084 
1085     // Tell listeners of the change in readers.
1086     notifyChangedReadersObservers();
1087 
1088     if (LOG.isTraceEnabled()) {
1089       long totalSize = 0;
1090       for (StoreFile sf : sfs) {
1091         totalSize += sf.getReader().length();
1092       }
1093       String traceMessage = "FLUSH time,count,size,store size,store files ["
1094           + EnvironmentEdgeManager.currentTime() + "," + sfs.size() + "," + totalSize
1095           + "," + storeSize + "," + storeEngine.getStoreFileManager().getStorefileCount() + "]";
1096       LOG.trace(traceMessage);
1097     }
1098     return needsCompaction();
1099   }
1100 
1101   /*
1102    * Notify all observers that set of Readers has changed.
1103    * @throws IOException
1104    */
1105   private void notifyChangedReadersObservers() throws IOException {
1106     for (ChangedReadersObserver o: this.changedReaderObservers) {
1107       o.updateReaders();
1108     }
1109   }
1110 
1111   /**
1112    * Get all scanners with no filtering based on TTL (that happens further down
1113    * the line).
1114    * @return all scanners for this store
1115    */
1116   @Override
1117   public List<KeyValueScanner> getScanners(boolean cacheBlocks, boolean isGet,
1118       boolean usePread, boolean isCompaction, ScanQueryMatcher matcher, byte[] startRow,
1119       byte[] stopRow, long readPt) throws IOException {
1120     Collection<StoreFile> storeFilesToScan;
1121     List<KeyValueScanner> memStoreScanners;
1122     this.lock.readLock().lock();
1123     try {
1124       storeFilesToScan =
1125           this.storeEngine.getStoreFileManager().getFilesForScanOrGet(isGet, startRow, stopRow);
1126       memStoreScanners = this.memstore.getScanners(readPt);
1127     } finally {
1128       this.lock.readLock().unlock();
1129     }
1130 
1131     // First the store file scanners
1132 
1133     // TODO this used to get the store files in descending order,
1134     // but now we get them in ascending order, which I think is
1135     // actually more correct, since memstore get put at the end.
1136     List<StoreFileScanner> sfScanners = StoreFileScanner.getScannersForStoreFiles(storeFilesToScan,
1137         cacheBlocks, usePread, isCompaction, false, matcher, readPt);
1138     List<KeyValueScanner> scanners =
1139       new ArrayList<KeyValueScanner>(sfScanners.size()+1);
1140     scanners.addAll(sfScanners);
1141     // Then the memstore scanners
1142     scanners.addAll(memStoreScanners);
1143     return scanners;
1144   }
1145 
1146   @Override
1147   public void addChangedReaderObserver(ChangedReadersObserver o) {
1148     this.changedReaderObservers.add(o);
1149   }
1150 
1151   @Override
1152   public void deleteChangedReaderObserver(ChangedReadersObserver o) {
1153     // We don't check if observer present; it may not be (legitimately)
1154     this.changedReaderObservers.remove(o);
1155   }
1156 
1157   //////////////////////////////////////////////////////////////////////////////
1158   // Compaction
1159   //////////////////////////////////////////////////////////////////////////////
1160 
1161   /**
1162    * Compact the StoreFiles.  This method may take some time, so the calling
1163    * thread must be able to block for long periods.
1164    *
1165    * <p>During this time, the Store can work as usual, getting values from
1166    * StoreFiles and writing new StoreFiles from the memstore.
1167    *
1168    * Existing StoreFiles are not destroyed until the new compacted StoreFile is
1169    * completely written-out to disk.
1170    *
1171    * <p>The compactLock prevents multiple simultaneous compactions.
1172    * The structureLock prevents us from interfering with other write operations.
1173    *
1174    * <p>We don't want to hold the structureLock for the whole time, as a compact()
1175    * can be lengthy and we want to allow cache-flushes during this period.
1176    *
1177    * <p> Compaction event should be idempotent, since there is no IO Fencing for
1178    * the region directory in hdfs. A region server might still try to complete the
1179    * compaction after it lost the region. That is why the following events are carefully
1180    * ordered for a compaction:
1181    *  1. Compaction writes new files under region/.tmp directory (compaction output)
1182    *  2. Compaction atomically moves the temporary file under region directory
1183    *  3. Compaction appends a WAL edit containing the compaction input and output files.
1184    *  Forces sync on WAL.
1185    *  4. Compaction deletes the input files from the region directory.
1186    *
1187    * Failure conditions are handled like this:
1188    *  - If RS fails before 2, compaction wont complete. Even if RS lives on and finishes
1189    *  the compaction later, it will only write the new data file to the region directory.
1190    *  Since we already have this data, this will be idempotent but we will have a redundant
1191    *  copy of the data.
1192    *  - If RS fails between 2 and 3, the region will have a redundant copy of the data. The
1193    *  RS that failed won't be able to finish snyc() for WAL because of lease recovery in WAL.
1194    *  - If RS fails after 3, the region region server who opens the region will pick up the
1195    *  the compaction marker from the WAL and replay it by removing the compaction input files.
1196    *  Failed RS can also attempt to delete those files, but the operation will be idempotent
1197    *
1198    * See HBASE-2231 for details.
1199    *
1200    * @param compaction compaction details obtained from requestCompaction()
1201    * @throws IOException
1202    * @return Storefile we compacted into or null if we failed or opted out early.
1203    */
1204   @Override
1205   public List<StoreFile> compact(CompactionContext compaction,
1206       CompactionThroughputController throughputController) throws IOException {
1207     assert compaction != null;
1208     List<StoreFile> sfs = null;
1209     CompactionRequest cr = compaction.getRequest();
1210     try {
1211       // Do all sanity checking in here if we have a valid CompactionRequest
1212       // because we need to clean up after it on the way out in a finally
1213       // block below
1214       long compactionStartTime = EnvironmentEdgeManager.currentTime();
1215       assert compaction.hasSelection();
1216       Collection<StoreFile> filesToCompact = cr.getFiles();
1217       assert !filesToCompact.isEmpty();
1218       synchronized (filesCompacting) {
1219         // sanity check: we're compacting files that this store knows about
1220         // TODO: change this to LOG.error() after more debugging
1221         Preconditions.checkArgument(filesCompacting.containsAll(filesToCompact));
1222       }
1223 
1224       // Ready to go. Have list of files to compact.
1225       LOG.info("Starting compaction of " + filesToCompact.size() + " file(s) in "
1226           + this + " of " + this.getRegionInfo().getRegionNameAsString()
1227           + " into tmpdir=" + fs.getTempDir() + ", totalSize="
1228           + TraditionalBinaryPrefix.long2String(cr.getSize(), "", 1));
1229 
1230       // Commence the compaction.
1231       List<Path> newFiles = compaction.compact(throughputController);
1232 
1233       // TODO: get rid of this!
1234       if (!this.conf.getBoolean("hbase.hstore.compaction.complete", true)) {
1235         LOG.warn("hbase.hstore.compaction.complete is set to false");
1236         sfs = new ArrayList<StoreFile>(newFiles.size());
1237         for (Path newFile : newFiles) {
1238           // Create storefile around what we wrote with a reader on it.
1239           StoreFile sf = createStoreFileAndReader(newFile);
1240           sf.closeReader(true);
1241           sfs.add(sf);
1242         }
1243         return sfs;
1244       }
1245       // Do the steps necessary to complete the compaction.
1246       sfs = moveCompatedFilesIntoPlace(cr, newFiles);
1247       writeCompactionWalRecord(filesToCompact, sfs);
1248       replaceStoreFiles(filesToCompact, sfs);
1249       if (cr.isMajor()) {
1250         majorCompactedCellsCount += getCompactionProgress().totalCompactingKVs;
1251         majorCompactedCellsSize += getCompactionProgress().totalCompactedSize;
1252       } else {
1253         compactedCellsCount += getCompactionProgress().totalCompactingKVs;
1254         compactedCellsSize += getCompactionProgress().totalCompactedSize;
1255       }
1256       // At this point the store will use new files for all new scanners.
1257       completeCompaction(filesToCompact, true); // Archive old files & update store size.
1258 
1259       logCompactionEndMessage(cr, sfs, compactionStartTime);
1260       return sfs;
1261     } finally {
1262       finishCompactionRequest(cr);
1263     }
1264   }
1265 
1266   private List<StoreFile> moveCompatedFilesIntoPlace(
1267       CompactionRequest cr, List<Path> newFiles) throws IOException {
1268     List<StoreFile> sfs = new ArrayList<StoreFile>(newFiles.size());
1269     for (Path newFile : newFiles) {
1270       assert newFile != null;
1271       StoreFile sf = moveFileIntoPlace(newFile);
1272       if (this.getCoprocessorHost() != null) {
1273         this.getCoprocessorHost().postCompact(this, sf, cr);
1274       }
1275       assert sf != null;
1276       sfs.add(sf);
1277     }
1278     return sfs;
1279   }
1280 
1281   // Package-visible for tests
1282   StoreFile moveFileIntoPlace(final Path newFile) throws IOException {
1283     validateStoreFile(newFile);
1284     // Move the file into the right spot
1285     Path destPath = fs.commitStoreFile(getColumnFamilyName(), newFile);
1286     return createStoreFileAndReader(destPath);
1287   }
1288 
1289   /**
1290    * Writes the compaction WAL record.
1291    * @param filesCompacted Files compacted (input).
1292    * @param newFiles Files from compaction.
1293    */
1294   private void writeCompactionWalRecord(Collection<StoreFile> filesCompacted,
1295       Collection<StoreFile> newFiles) throws IOException {
1296     if (region.getWAL() == null) return;
1297     List<Path> inputPaths = new ArrayList<Path>(filesCompacted.size());
1298     for (StoreFile f : filesCompacted) {
1299       inputPaths.add(f.getPath());
1300     }
1301     List<Path> outputPaths = new ArrayList<Path>(newFiles.size());
1302     for (StoreFile f : newFiles) {
1303       outputPaths.add(f.getPath());
1304     }
1305     HRegionInfo info = this.region.getRegionInfo();
1306     CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(info,
1307         family.getName(), inputPaths, outputPaths, fs.getStoreDir(getFamily().getNameAsString()));
1308     WALUtil.writeCompactionMarker(region.getWAL(), this.region.getTableDesc(),
1309         this.region.getRegionInfo(), compactionDescriptor, this.region.getSequenceId());
1310   }
1311 
1312   @VisibleForTesting
1313   void replaceStoreFiles(final Collection<StoreFile> compactedFiles,
1314       final Collection<StoreFile> result) throws IOException {
1315     this.lock.writeLock().lock();
1316     try {
1317       this.storeEngine.getStoreFileManager().addCompactionResults(compactedFiles, result);
1318       filesCompacting.removeAll(compactedFiles); // safe bc: lock.writeLock();
1319     } finally {
1320       this.lock.writeLock().unlock();
1321     }
1322   }
1323 
1324   /**
1325    * Log a very elaborate compaction completion message.
1326    * @param cr Request.
1327    * @param sfs Resulting files.
1328    * @param compactionStartTime Start time.
1329    */
1330   private void logCompactionEndMessage(
1331       CompactionRequest cr, List<StoreFile> sfs, long compactionStartTime) {
1332     long now = EnvironmentEdgeManager.currentTime();
1333     StringBuilder message = new StringBuilder(
1334       "Completed" + (cr.isMajor() ? " major" : "") + " compaction of "
1335       + cr.getFiles().size() + (cr.isAllFiles() ? " (all)" : "") + " file(s) in "
1336       + this + " of " + this.getRegionInfo().getRegionNameAsString() + " into ");
1337     if (sfs.isEmpty()) {
1338       message.append("none, ");
1339     } else {
1340       for (StoreFile sf: sfs) {
1341         message.append(sf.getPath().getName());
1342         message.append("(size=");
1343         message.append(TraditionalBinaryPrefix.long2String(sf.getReader().length(), "", 1));
1344         message.append("), ");
1345       }
1346     }
1347     message.append("total size for store is ")
1348       .append(StringUtils.TraditionalBinaryPrefix.long2String(storeSize, "", 1))
1349       .append(". This selection was in queue for ")
1350       .append(StringUtils.formatTimeDiff(compactionStartTime, cr.getSelectionTime()))
1351       .append(", and took ").append(StringUtils.formatTimeDiff(now, compactionStartTime))
1352       .append(" to execute.");
1353     LOG.info(message.toString());
1354     if (LOG.isTraceEnabled()) {
1355       int fileCount = storeEngine.getStoreFileManager().getStorefileCount();
1356       long resultSize = 0;
1357       for (StoreFile sf : sfs) {
1358         resultSize += sf.getReader().length();
1359       }
1360       String traceMessage = "COMPACTION start,end,size out,files in,files out,store size,"
1361         + "store files [" + compactionStartTime + "," + now + "," + resultSize + ","
1362           + cr.getFiles().size() + "," + sfs.size() + "," +  storeSize + "," + fileCount + "]";
1363       LOG.trace(traceMessage);
1364     }
1365   }
1366 
1367   /**
1368    * Call to complete a compaction. Its for the case where we find in the WAL a compaction
1369    * that was not finished.  We could find one recovering a WAL after a regionserver crash.
1370    * See HBASE-2231.
1371    * @param compaction
1372    */
1373   @Override
1374   public void replayCompactionMarker(CompactionDescriptor compaction,
1375       boolean pickCompactionFiles, boolean removeFiles)
1376       throws IOException {
1377     LOG.debug("Completing compaction from the WAL marker");
1378     List<String> compactionInputs = compaction.getCompactionInputList();
1379     List<String> compactionOutputs = Lists.newArrayList(compaction.getCompactionOutputList());
1380 
1381     // The Compaction Marker is written after the compaction is completed,
1382     // and the files moved into the region/family folder.
1383     //
1384     // If we crash after the entry is written, we may not have removed the
1385     // input files, but the output file is present.
1386     // (The unremoved input files will be removed by this function)
1387     //
1388     // If we scan the directory and the file is not present, it can mean that:
1389     //   - The file was manually removed by the user
1390     //   - The file was removed as consequence of subsequent compaction
1391     // so, we can't do anything with the "compaction output list" because those
1392     // files have already been loaded when opening the region (by virtue of
1393     // being in the store's folder) or they may be missing due to a compaction.
1394 
1395     String familyName = this.getColumnFamilyName();
1396     List<String> inputFiles = new ArrayList<String>(compactionInputs.size());
1397     for (String compactionInput : compactionInputs) {
1398       Path inputPath = fs.getStoreFilePath(familyName, compactionInput);
1399       inputFiles.add(inputPath.getName());
1400     }
1401 
1402     //some of the input files might already be deleted
1403     List<StoreFile> inputStoreFiles = new ArrayList<StoreFile>(compactionInputs.size());
1404     for (StoreFile sf : this.getStorefiles()) {
1405       if (inputFiles.contains(sf.getPath().getName())) {
1406         inputStoreFiles.add(sf);
1407       }
1408     }
1409 
1410     // check whether we need to pick up the new files
1411     List<StoreFile> outputStoreFiles = new ArrayList<StoreFile>(compactionOutputs.size());
1412 
1413     if (pickCompactionFiles) {
1414       for (StoreFile sf : this.getStorefiles()) {
1415         compactionOutputs.remove(sf.getPath().getName());
1416       }
1417       for (String compactionOutput : compactionOutputs) {
1418         StoreFileInfo storeFileInfo = fs.getStoreFileInfo(getColumnFamilyName(), compactionOutput);
1419         StoreFile storeFile = createStoreFileAndReader(storeFileInfo);
1420         outputStoreFiles.add(storeFile);
1421       }
1422     }
1423 
1424     if (!inputStoreFiles.isEmpty() || !outputStoreFiles.isEmpty()) {
1425       LOG.info("Replaying compaction marker, replacing input files: " +
1426           inputStoreFiles + " with output files : " + outputStoreFiles);
1427       this.replaceStoreFiles(inputStoreFiles, outputStoreFiles);
1428       this.completeCompaction(inputStoreFiles, removeFiles);
1429     }
1430   }
1431 
1432   /**
1433    * This method tries to compact N recent files for testing.
1434    * Note that because compacting "recent" files only makes sense for some policies,
1435    * e.g. the default one, it assumes default policy is used. It doesn't use policy,
1436    * but instead makes a compaction candidate list by itself.
1437    * @param N Number of files.
1438    */
1439   public void compactRecentForTestingAssumingDefaultPolicy(int N) throws IOException {
1440     List<StoreFile> filesToCompact;
1441     boolean isMajor;
1442 
1443     this.lock.readLock().lock();
1444     try {
1445       synchronized (filesCompacting) {
1446         filesToCompact = Lists.newArrayList(storeEngine.getStoreFileManager().getStorefiles());
1447         if (!filesCompacting.isEmpty()) {
1448           // exclude all files older than the newest file we're currently
1449           // compacting. this allows us to preserve contiguity (HBASE-2856)
1450           StoreFile last = filesCompacting.get(filesCompacting.size() - 1);
1451           int idx = filesToCompact.indexOf(last);
1452           Preconditions.checkArgument(idx != -1);
1453           filesToCompact.subList(0, idx + 1).clear();
1454         }
1455         int count = filesToCompact.size();
1456         if (N > count) {
1457           throw new RuntimeException("Not enough files");
1458         }
1459 
1460         filesToCompact = filesToCompact.subList(count - N, count);
1461         isMajor = (filesToCompact.size() == storeEngine.getStoreFileManager().getStorefileCount());
1462         filesCompacting.addAll(filesToCompact);
1463         Collections.sort(filesCompacting, StoreFile.Comparators.SEQ_ID);
1464       }
1465     } finally {
1466       this.lock.readLock().unlock();
1467     }
1468 
1469     try {
1470       // Ready to go. Have list of files to compact.
1471       List<Path> newFiles = ((DefaultCompactor)this.storeEngine.getCompactor())
1472           .compactForTesting(filesToCompact, isMajor);
1473       for (Path newFile: newFiles) {
1474         // Move the compaction into place.
1475         StoreFile sf = moveFileIntoPlace(newFile);
1476         if (this.getCoprocessorHost() != null) {
1477           this.getCoprocessorHost().postCompact(this, sf, null);
1478         }
1479         replaceStoreFiles(filesToCompact, Lists.newArrayList(sf));
1480         completeCompaction(filesToCompact, true);
1481       }
1482     } finally {
1483       synchronized (filesCompacting) {
1484         filesCompacting.removeAll(filesToCompact);
1485       }
1486     }
1487   }
1488 
1489   @Override
1490   public boolean hasReferences() {
1491     return StoreUtils.hasReferences(this.storeEngine.getStoreFileManager().getStorefiles());
1492   }
1493 
1494   @Override
1495   public CompactionProgress getCompactionProgress() {
1496     return this.storeEngine.getCompactor().getProgress();
1497   }
1498 
1499   @Override
1500   public boolean isMajorCompaction() throws IOException {
1501     for (StoreFile sf : this.storeEngine.getStoreFileManager().getStorefiles()) {
1502       // TODO: what are these reader checks all over the place?
1503       if (sf.getReader() == null) {
1504         LOG.debug("StoreFile " + sf + " has null Reader");
1505         return false;
1506       }
1507     }
1508     return storeEngine.getCompactionPolicy().isMajorCompaction(
1509         this.storeEngine.getStoreFileManager().getStorefiles());
1510   }
1511 
1512   @Override
1513   public CompactionContext requestCompaction() throws IOException {
1514     return requestCompaction(Store.NO_PRIORITY, null);
1515   }
1516 
1517   @Override
1518   public CompactionContext requestCompaction(int priority, CompactionRequest baseRequest)
1519       throws IOException {
1520     // don't even select for compaction if writes are disabled
1521     if (!this.areWritesEnabled()) {
1522       return null;
1523     }
1524 
1525     // Before we do compaction, try to get rid of unneeded files to simplify things.
1526     removeUnneededFiles();
1527 
1528     CompactionContext compaction = storeEngine.createCompaction();
1529     CompactionRequest request = null;
1530     this.lock.readLock().lock();
1531     try {
1532       synchronized (filesCompacting) {
1533         // First, see if coprocessor would want to override selection.
1534         if (this.getCoprocessorHost() != null) {
1535           List<StoreFile> candidatesForCoproc = compaction.preSelect(this.filesCompacting);
1536           boolean override = this.getCoprocessorHost().preCompactSelection(
1537               this, candidatesForCoproc, baseRequest);
1538           if (override) {
1539             // Coprocessor is overriding normal file selection.
1540             compaction.forceSelect(new CompactionRequest(candidatesForCoproc));
1541           }
1542         }
1543 
1544         // Normal case - coprocessor is not overriding file selection.
1545         if (!compaction.hasSelection()) {
1546           boolean isUserCompaction = priority == Store.PRIORITY_USER;
1547           boolean mayUseOffPeak = offPeakHours.isOffPeakHour() &&
1548               offPeakCompactionTracker.compareAndSet(false, true);
1549           try {
1550             compaction.select(this.filesCompacting, isUserCompaction,
1551               mayUseOffPeak, forceMajor && filesCompacting.isEmpty());
1552           } catch (IOException e) {
1553             if (mayUseOffPeak) {
1554               offPeakCompactionTracker.set(false);
1555             }
1556             throw e;
1557           }
1558           assert compaction.hasSelection();
1559           if (mayUseOffPeak && !compaction.getRequest().isOffPeak()) {
1560             // Compaction policy doesn't want to take advantage of off-peak.
1561             offPeakCompactionTracker.set(false);
1562           }
1563         }
1564         if (this.getCoprocessorHost() != null) {
1565           this.getCoprocessorHost().postCompactSelection(
1566               this, ImmutableList.copyOf(compaction.getRequest().getFiles()), baseRequest);
1567         }
1568 
1569         // Selected files; see if we have a compaction with some custom base request.
1570         if (baseRequest != null) {
1571           // Update the request with what the system thinks the request should be;
1572           // its up to the request if it wants to listen.
1573           compaction.forceSelect(
1574               baseRequest.combineWith(compaction.getRequest()));
1575         }
1576         // Finally, we have the resulting files list. Check if we have any files at all.
1577         request = compaction.getRequest();
1578         final Collection<StoreFile> selectedFiles = request.getFiles();
1579         if (selectedFiles.isEmpty()) {
1580           return null;
1581         }
1582 
1583         addToCompactingFiles(selectedFiles);
1584 
1585         // If we're enqueuing a major, clear the force flag.
1586         this.forceMajor = this.forceMajor && !request.isMajor();
1587 
1588         // Set common request properties.
1589         // Set priority, either override value supplied by caller or from store.
1590         request.setPriority((priority != Store.NO_PRIORITY) ? priority : getCompactPriority());
1591         request.setDescription(getRegionInfo().getRegionNameAsString(), getColumnFamilyName());
1592       }
1593     } finally {
1594       this.lock.readLock().unlock();
1595     }
1596 
1597     LOG.debug(getRegionInfo().getEncodedName() + " - "  + getColumnFamilyName()
1598         + ": Initiating " + (request.isMajor() ? "major" : "minor") + " compaction"
1599         + (request.isAllFiles() ? " (all files)" : ""));
1600     this.region.reportCompactionRequestStart(request.isMajor());
1601     return compaction;
1602   }
1603 
1604   /** Adds the files to compacting files. filesCompacting must be locked. */
1605   private void addToCompactingFiles(final Collection<StoreFile> filesToAdd) {
1606     if (filesToAdd == null) return;
1607     // Check that we do not try to compact the same StoreFile twice.
1608     if (!Collections.disjoint(filesCompacting, filesToAdd)) {
1609       Preconditions.checkArgument(false, "%s overlaps with %s", filesToAdd, filesCompacting);
1610     }
1611     filesCompacting.addAll(filesToAdd);
1612     Collections.sort(filesCompacting, StoreFile.Comparators.SEQ_ID);
1613   }
1614 
1615   private void removeUnneededFiles() throws IOException {
1616     if (!conf.getBoolean("hbase.store.delete.expired.storefile", true)) return;
1617     if (getFamily().getMinVersions() > 0) {
1618       LOG.debug("Skipping expired store file removal due to min version being " +
1619           getFamily().getMinVersions());
1620       return;
1621     }
1622     this.lock.readLock().lock();
1623     Collection<StoreFile> delSfs = null;
1624     try {
1625       synchronized (filesCompacting) {
1626         long cfTtl = getStoreFileTtl();
1627         if (cfTtl != Long.MAX_VALUE) {
1628           delSfs = storeEngine.getStoreFileManager().getUnneededFiles(
1629               EnvironmentEdgeManager.currentTime() - cfTtl, filesCompacting);
1630           addToCompactingFiles(delSfs);
1631         }
1632       }
1633     } finally {
1634       this.lock.readLock().unlock();
1635     }
1636     if (delSfs == null || delSfs.isEmpty()) return;
1637 
1638     Collection<StoreFile> newFiles = new ArrayList<StoreFile>(); // No new files.
1639     writeCompactionWalRecord(delSfs, newFiles);
1640     replaceStoreFiles(delSfs, newFiles);
1641     completeCompaction(delSfs);
1642     LOG.info("Completed removal of " + delSfs.size() + " unnecessary (expired) file(s) in "
1643         + this + " of " + this.getRegionInfo().getRegionNameAsString()
1644         + "; total size for store is " + TraditionalBinaryPrefix.long2String(storeSize, "", 1));
1645   }
1646 
1647   @Override
1648   public void cancelRequestedCompaction(CompactionContext compaction) {
1649     finishCompactionRequest(compaction.getRequest());
1650   }
1651 
1652   private void finishCompactionRequest(CompactionRequest cr) {
1653     this.region.reportCompactionRequestEnd(cr.isMajor(), cr.getFiles().size(), cr.getSize());
1654     if (cr.isOffPeak()) {
1655       offPeakCompactionTracker.set(false);
1656       cr.setOffPeak(false);
1657     }
1658     synchronized (filesCompacting) {
1659       filesCompacting.removeAll(cr.getFiles());
1660     }
1661   }
1662 
1663   /**
1664    * Validates a store file by opening and closing it. In HFileV2 this should
1665    * not be an expensive operation.
1666    *
1667    * @param path the path to the store file
1668    */
1669   private void validateStoreFile(Path path)
1670       throws IOException {
1671     StoreFile storeFile = null;
1672     try {
1673       storeFile = createStoreFileAndReader(path);
1674     } catch (IOException e) {
1675       LOG.error("Failed to open store file : " + path
1676           + ", keeping it in tmp location", e);
1677       throw e;
1678     } finally {
1679       if (storeFile != null) {
1680         storeFile.closeReader(false);
1681       }
1682     }
1683   }
1684 
1685   /**
1686    * <p>It works by processing a compaction that's been written to disk.
1687    *
1688    * <p>It is usually invoked at the end of a compaction, but might also be
1689    * invoked at HStore startup, if the prior execution died midway through.
1690    *
1691    * <p>Moving the compacted TreeMap into place means:
1692    * <pre>
1693    * 1) Unload all replaced StoreFile, close and collect list to delete.
1694    * 2) Compute new store size
1695    * </pre>
1696    *
1697    * @param compactedFiles list of files that were compacted
1698    */
1699   @VisibleForTesting
1700   protected void completeCompaction(final Collection<StoreFile> compactedFiles)
1701     throws IOException {
1702     completeCompaction(compactedFiles, true);
1703   }
1704 
1705 
1706   /**
1707    * <p>It works by processing a compaction that's been written to disk.
1708    *
1709    * <p>It is usually invoked at the end of a compaction, but might also be
1710    * invoked at HStore startup, if the prior execution died midway through.
1711    *
1712    * <p>Moving the compacted TreeMap into place means:
1713    * <pre>
1714    * 1) Unload all replaced StoreFile, close and collect list to delete.
1715    * 2) Compute new store size
1716    * </pre>
1717    *
1718    * @param compactedFiles list of files that were compacted
1719    */
1720   @VisibleForTesting
1721   protected void completeCompaction(final Collection<StoreFile> compactedFiles, boolean removeFiles)
1722       throws IOException {
1723     try {
1724       // Do not delete old store files until we have sent out notification of
1725       // change in case old files are still being accessed by outstanding scanners.
1726       // Don't do this under writeLock; see HBASE-4485 for a possible deadlock
1727       // scenario that could have happened if continue to hold the lock.
1728       notifyChangedReadersObservers();
1729       // At this point the store will use new files for all scanners.
1730 
1731       // let the archive util decide if we should archive or delete the files
1732       LOG.debug("Removing store files after compaction...");
1733       for (StoreFile compactedFile : compactedFiles) {
1734         compactedFile.closeReader(true);
1735       }
1736       if (removeFiles) {
1737         this.fs.removeStoreFiles(this.getColumnFamilyName(), compactedFiles);
1738       }
1739     } catch (IOException e) {
1740       e = e instanceof RemoteException ?
1741                 ((RemoteException)e).unwrapRemoteException() : e;
1742       LOG.error("Failed removing compacted files in " + this +
1743         ". Files we were trying to remove are " + compactedFiles.toString() +
1744         "; some of them may have been already removed", e);
1745     }
1746 
1747     // 4. Compute new store size
1748     this.storeSize = 0L;
1749     this.totalUncompressedBytes = 0L;
1750     for (StoreFile hsf : this.storeEngine.getStoreFileManager().getStorefiles()) {
1751       StoreFile.Reader r = hsf.getReader();
1752       if (r == null) {
1753         LOG.warn("StoreFile " + hsf + " has a null Reader");
1754         continue;
1755       }
1756       this.storeSize += r.length();
1757       this.totalUncompressedBytes += r.getTotalUncompressedBytes();
1758     }
1759   }
1760 
1761   /*
1762    * @param wantedVersions How many versions were asked for.
1763    * @return wantedVersions or this families' {@link HConstants#VERSIONS}.
1764    */
1765   int versionsToReturn(final int wantedVersions) {
1766     if (wantedVersions <= 0) {
1767       throw new IllegalArgumentException("Number of versions must be > 0");
1768     }
1769     // Make sure we do not return more than maximum versions for this store.
1770     int maxVersions = this.family.getMaxVersions();
1771     return wantedVersions > maxVersions ? maxVersions: wantedVersions;
1772   }
1773 
1774   /**
1775    * @param cell
1776    * @param oldestTimestamp
1777    * @return true if the cell is expired
1778    */
1779   static boolean isCellTTLExpired(final Cell cell, final long oldestTimestamp, final long now) {
1780     // Do not create an Iterator or Tag objects unless the cell actually has tags.
1781     if (cell.getTagsLength() > 0) {
1782       // Look for a TTL tag first. Use it instead of the family setting if
1783       // found. If a cell has multiple TTLs, resolve the conflict by using the
1784       // first tag encountered.
1785       Iterator<Tag> i = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(),
1786         cell.getTagsLength());
1787       while (i.hasNext()) {
1788         Tag t = i.next();
1789         if (TagType.TTL_TAG_TYPE == t.getType()) {
1790           // Unlike in schema cell TTLs are stored in milliseconds, no need
1791           // to convert
1792           long ts = cell.getTimestamp();
1793           assert t.getTagLength() == Bytes.SIZEOF_LONG;
1794           long ttl = Bytes.toLong(t.getBuffer(), t.getTagOffset(), t.getTagLength());
1795           if (ts + ttl < now) {
1796             return true;
1797           }
1798           // Per cell TTLs cannot extend lifetime beyond family settings, so
1799           // fall through to check that
1800           break;
1801         }
1802       }
1803     }
1804     return false;
1805   }
1806 
1807   @Override
1808   public boolean canSplit() {
1809     this.lock.readLock().lock();
1810     try {
1811       // Not split-able if we find a reference store file present in the store.
1812       boolean result = !hasReferences();
1813       if (!result && LOG.isDebugEnabled()) {
1814         LOG.debug("Cannot split region due to reference files being there");
1815       }
1816       return result;
1817     } finally {
1818       this.lock.readLock().unlock();
1819     }
1820   }
1821 
1822   @Override
1823   public byte[] getSplitPoint() {
1824     this.lock.readLock().lock();
1825     try {
1826       // Should already be enforced by the split policy!
1827       assert !this.getRegionInfo().isMetaRegion();
1828       // Not split-able if we find a reference store file present in the store.
1829       if (hasReferences()) {
1830         return null;
1831       }
1832       return this.storeEngine.getStoreFileManager().getSplitPoint();
1833     } catch(IOException e) {
1834       LOG.warn("Failed getting store size for " + this, e);
1835     } finally {
1836       this.lock.readLock().unlock();
1837     }
1838     return null;
1839   }
1840 
1841   @Override
1842   public long getLastCompactSize() {
1843     return this.lastCompactSize;
1844   }
1845 
1846   @Override
1847   public long getSize() {
1848     return storeSize;
1849   }
1850 
1851   @Override
1852   public void triggerMajorCompaction() {
1853     this.forceMajor = true;
1854   }
1855 
1856 
1857   //////////////////////////////////////////////////////////////////////////////
1858   // File administration
1859   //////////////////////////////////////////////////////////////////////////////
1860 
1861   @Override
1862   public KeyValueScanner getScanner(Scan scan,
1863       final NavigableSet<byte []> targetCols, long readPt) throws IOException {
1864     lock.readLock().lock();
1865     try {
1866       KeyValueScanner scanner = null;
1867       if (this.getCoprocessorHost() != null) {
1868         scanner = this.getCoprocessorHost().preStoreScannerOpen(this, scan, targetCols);
1869       }
1870       scanner = createScanner(scan, targetCols, readPt, scanner);
1871       return scanner;
1872     } finally {
1873       lock.readLock().unlock();
1874     }
1875   }
1876 
1877   protected KeyValueScanner createScanner(Scan scan, final NavigableSet<byte[]> targetCols,
1878       long readPt, KeyValueScanner scanner) throws IOException {
1879     if (scanner == null) {
1880       scanner = scan.isReversed() ? new ReversedStoreScanner(this,
1881           getScanInfo(), scan, targetCols, readPt) : new StoreScanner(this,
1882           getScanInfo(), scan, targetCols, readPt);
1883     }
1884     return scanner;
1885   }
1886 
1887   @Override
1888   public String toString() {
1889     return this.getColumnFamilyName();
1890   }
1891 
1892   @Override
1893   public int getStorefilesCount() {
1894     return this.storeEngine.getStoreFileManager().getStorefileCount();
1895   }
1896 
1897   @Override
1898   public long getStoreSizeUncompressed() {
1899     return this.totalUncompressedBytes;
1900   }
1901 
1902   @Override
1903   public long getStorefilesSize() {
1904     long size = 0;
1905     for (StoreFile s: this.storeEngine.getStoreFileManager().getStorefiles()) {
1906       StoreFile.Reader r = s.getReader();
1907       if (r == null) {
1908         LOG.warn("StoreFile " + s + " has a null Reader");
1909         continue;
1910       }
1911       size += r.length();
1912     }
1913     return size;
1914   }
1915 
1916   @Override
1917   public long getStorefilesIndexSize() {
1918     long size = 0;
1919     for (StoreFile s: this.storeEngine.getStoreFileManager().getStorefiles()) {
1920       StoreFile.Reader r = s.getReader();
1921       if (r == null) {
1922         LOG.warn("StoreFile " + s + " has a null Reader");
1923         continue;
1924       }
1925       size += r.indexSize();
1926     }
1927     return size;
1928   }
1929 
1930   @Override
1931   public long getTotalStaticIndexSize() {
1932     long size = 0;
1933     for (StoreFile s : this.storeEngine.getStoreFileManager().getStorefiles()) {
1934       size += s.getReader().getUncompressedDataIndexSize();
1935     }
1936     return size;
1937   }
1938 
1939   @Override
1940   public long getTotalStaticBloomSize() {
1941     long size = 0;
1942     for (StoreFile s : this.storeEngine.getStoreFileManager().getStorefiles()) {
1943       StoreFile.Reader r = s.getReader();
1944       size += r.getTotalBloomSize();
1945     }
1946     return size;
1947   }
1948 
1949   @Override
1950   public long getMemStoreSize() {
1951     return this.memstore.size();
1952   }
1953 
1954   @Override
1955   public int getCompactPriority() {
1956     int priority = this.storeEngine.getStoreFileManager().getStoreCompactionPriority();
1957     if (priority == PRIORITY_USER) {
1958       LOG.warn("Compaction priority is USER despite there being no user compaction");
1959     }
1960     return priority;
1961   }
1962 
1963   @Override
1964   public boolean throttleCompaction(long compactionSize) {
1965     return storeEngine.getCompactionPolicy().throttleCompaction(compactionSize);
1966   }
1967 
1968   public HRegion getHRegion() {
1969     return this.region;
1970   }
1971 
1972   @Override
1973   public RegionCoprocessorHost getCoprocessorHost() {
1974     return this.region.getCoprocessorHost();
1975   }
1976 
1977   @Override
1978   public HRegionInfo getRegionInfo() {
1979     return this.fs.getRegionInfo();
1980   }
1981 
1982   @Override
1983   public boolean areWritesEnabled() {
1984     return this.region.areWritesEnabled();
1985   }
1986 
1987   @Override
1988   public long getSmallestReadPoint() {
1989     return this.region.getSmallestReadPoint();
1990   }
1991 
1992   /**
1993    * Used in tests. TODO: Remove
1994    *
1995    * Updates the value for the given row/family/qualifier. This function will always be seen as
1996    * atomic by other readers because it only puts a single KV to memstore. Thus no read/write
1997    * control necessary.
1998    * @param row row to update
1999    * @param f family to update
2000    * @param qualifier qualifier to update
2001    * @param newValue the new value to set into memstore
2002    * @return memstore size delta
2003    * @throws IOException
2004    */
2005   public long updateColumnValue(byte [] row, byte [] f,
2006                                 byte [] qualifier, long newValue)
2007       throws IOException {
2008 
2009     this.lock.readLock().lock();
2010     try {
2011       long now = EnvironmentEdgeManager.currentTime();
2012 
2013       return this.memstore.updateColumnValue(row,
2014           f,
2015           qualifier,
2016           newValue,
2017           now);
2018 
2019     } finally {
2020       this.lock.readLock().unlock();
2021     }
2022   }
2023 
2024   @Override
2025   public long upsert(Iterable<Cell> cells, long readpoint) throws IOException {
2026     this.lock.readLock().lock();
2027     try {
2028       return this.memstore.upsert(cells, readpoint);
2029     } finally {
2030       this.lock.readLock().unlock();
2031     }
2032   }
2033 
2034   @Override
2035   public StoreFlushContext createFlushContext(long cacheFlushId) {
2036     return new StoreFlusherImpl(cacheFlushId);
2037   }
2038 
2039   private final class StoreFlusherImpl implements StoreFlushContext {
2040 
2041     private long cacheFlushSeqNum;
2042     private MemStoreSnapshot snapshot;
2043     private List<Path> tempFiles;
2044     private List<Path> committedFiles;
2045     private long cacheFlushCount;
2046     private long cacheFlushSize;
2047 
2048     private StoreFlusherImpl(long cacheFlushSeqNum) {
2049       this.cacheFlushSeqNum = cacheFlushSeqNum;
2050     }
2051 
2052     /**
2053      * This is not thread safe. The caller should have a lock on the region or the store.
2054      * If necessary, the lock can be added with the patch provided in HBASE-10087
2055      */
2056     @Override
2057     public void prepare() {
2058       this.snapshot = memstore.snapshot();
2059       this.cacheFlushCount = snapshot.getCellsCount();
2060       this.cacheFlushSize = snapshot.getSize();
2061       committedFiles = new ArrayList<Path>(1);
2062     }
2063 
2064     @Override
2065     public void flushCache(MonitoredTask status) throws IOException {
2066       tempFiles = HStore.this.flushCache(cacheFlushSeqNum, snapshot, status);
2067     }
2068 
2069     @Override
2070     public boolean commit(MonitoredTask status) throws IOException {
2071       if (this.tempFiles == null || this.tempFiles.isEmpty()) {
2072         return false;
2073       }
2074       List<StoreFile> storeFiles = new ArrayList<StoreFile>(this.tempFiles.size());
2075       for (Path storeFilePath : tempFiles) {
2076         try {
2077           storeFiles.add(HStore.this.commitFile(storeFilePath, cacheFlushSeqNum, status));
2078         } catch (IOException ex) {
2079           LOG.error("Failed to commit store file " + storeFilePath, ex);
2080           // Try to delete the files we have committed before.
2081           for (StoreFile sf : storeFiles) {
2082             Path pathToDelete = sf.getPath();
2083             try {
2084               sf.deleteReader();
2085             } catch (IOException deleteEx) {
2086               LOG.fatal("Failed to delete store file we committed, halting " + pathToDelete, ex);
2087               Runtime.getRuntime().halt(1);
2088             }
2089           }
2090           throw new IOException("Failed to commit the flush", ex);
2091         }
2092       }
2093 
2094       for (StoreFile sf : storeFiles) {
2095         if (HStore.this.getCoprocessorHost() != null) {
2096           HStore.this.getCoprocessorHost().postFlush(HStore.this, sf);
2097         }
2098         committedFiles.add(sf.getPath());
2099       }
2100 
2101       HStore.this.flushedCellsCount += cacheFlushCount;
2102       HStore.this.flushedCellsSize += cacheFlushSize;
2103 
2104       // Add new file to store files.  Clear snapshot too while we have the Store write lock.
2105       return HStore.this.updateStorefiles(storeFiles, snapshot.getId());
2106     }
2107 
2108     @Override
2109     public List<Path> getCommittedFiles() {
2110       return committedFiles;
2111     }
2112 
2113     /**
2114      * Similar to commit, but called in secondary region replicas for replaying the
2115      * flush cache from primary region. Adds the new files to the store, and drops the
2116      * snapshot depending on dropMemstoreSnapshot argument.
2117      * @param fileNames names of the flushed files
2118      * @param dropMemstoreSnapshot whether to drop the prepared memstore snapshot
2119      * @throws IOException
2120      */
2121     @Override
2122     public void replayFlush(List<String> fileNames, boolean dropMemstoreSnapshot)
2123         throws IOException {
2124       List<StoreFile> storeFiles = new ArrayList<StoreFile>(fileNames.size());
2125       for (String file : fileNames) {
2126         // open the file as a store file (hfile link, etc)
2127         StoreFileInfo storeFileInfo = fs.getStoreFileInfo(getColumnFamilyName(), file);
2128         StoreFile storeFile = createStoreFileAndReader(storeFileInfo);
2129         storeFiles.add(storeFile);
2130         HStore.this.storeSize += storeFile.getReader().length();
2131         HStore.this.totalUncompressedBytes += storeFile.getReader().getTotalUncompressedBytes();
2132         if (LOG.isInfoEnabled()) {
2133           LOG.info("Region: " + HStore.this.getRegionInfo().getEncodedName() +
2134             " added " + storeFile + ", entries=" + storeFile.getReader().getEntries() +
2135             ", sequenceid=" +  + storeFile.getReader().getSequenceID() +
2136             ", filesize=" + StringUtils.humanReadableInt(storeFile.getReader().length()));
2137         }
2138       }
2139 
2140       long snapshotId = -1; // -1 means do not drop
2141       if (dropMemstoreSnapshot && snapshot != null) {
2142         snapshotId = snapshot.getId();
2143       }
2144       HStore.this.updateStorefiles(storeFiles, snapshotId);
2145     }
2146 
2147     /**
2148      * Abort the snapshot preparation. Drops the snapshot if any.
2149      * @throws IOException
2150      */
2151     @Override
2152     public void abort() throws IOException {
2153       if (snapshot == null) {
2154         return;
2155       }
2156       HStore.this.updateStorefiles(new ArrayList<StoreFile>(0), snapshot.getId());
2157     }
2158   }
2159 
2160   @Override
2161   public boolean needsCompaction() {
2162     return this.storeEngine.needsCompaction(this.filesCompacting);
2163   }
2164 
2165   @Override
2166   public CacheConfig getCacheConfig() {
2167     return this.cacheConf;
2168   }
2169 
2170   public static final long FIXED_OVERHEAD =
2171       ClassSize.align(ClassSize.OBJECT + (16 * ClassSize.REFERENCE) + (10 * Bytes.SIZEOF_LONG)
2172               + (5 * Bytes.SIZEOF_INT) + (2 * Bytes.SIZEOF_BOOLEAN));
2173 
2174   public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD
2175       + ClassSize.OBJECT + ClassSize.REENTRANT_LOCK
2176       + ClassSize.CONCURRENT_SKIPLISTMAP
2177       + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY + ClassSize.OBJECT
2178       + ScanInfo.FIXED_OVERHEAD);
2179 
2180   @Override
2181   public long heapSize() {
2182     return DEEP_OVERHEAD + this.memstore.heapSize();
2183   }
2184 
2185   @Override
2186   public CellComparator getComparator() {
2187     return comparator;
2188   }
2189 
2190   @Override
2191   public ScanInfo getScanInfo() {
2192     return scanInfo;
2193   }
2194 
2195   /**
2196    * Set scan info, used by test
2197    * @param scanInfo new scan info to use for test
2198    */
2199   void setScanInfo(ScanInfo scanInfo) {
2200     this.scanInfo = scanInfo;
2201   }
2202 
2203   @Override
2204   public boolean hasTooManyStoreFiles() {
2205     return getStorefilesCount() > this.blockingFileCount;
2206   }
2207 
2208   @Override
2209   public long getFlushedCellsCount() {
2210     return flushedCellsCount;
2211   }
2212 
2213   @Override
2214   public long getFlushedCellsSize() {
2215     return flushedCellsSize;
2216   }
2217 
2218   @Override
2219   public long getCompactedCellsCount() {
2220     return compactedCellsCount;
2221   }
2222 
2223   @Override
2224   public long getCompactedCellsSize() {
2225     return compactedCellsSize;
2226   }
2227 
2228   @Override
2229   public long getMajorCompactedCellsCount() {
2230     return majorCompactedCellsCount;
2231   }
2232 
2233   @Override
2234   public long getMajorCompactedCellsSize() {
2235     return majorCompactedCellsSize;
2236   }
2237 
2238   /**
2239    * Returns the StoreEngine that is backing this concrete implementation of Store.
2240    * @return Returns the {@link StoreEngine} object used internally inside this HStore object.
2241    */
2242   @VisibleForTesting
2243   public StoreEngine<?, ?, ?, ?> getStoreEngine() {
2244     return this.storeEngine;
2245   }
2246 
2247   protected OffPeakHours getOffPeakHours() {
2248     return this.offPeakHours;
2249   }
2250 
2251   /**
2252    * {@inheritDoc}
2253    */
2254   @Override
2255   public void onConfigurationChange(Configuration conf) {
2256     this.conf = new CompoundConfiguration()
2257             .add(conf)
2258             .addBytesMap(family.getValues());
2259     this.storeEngine.compactionPolicy.setConf(conf);
2260     this.offPeakHours = OffPeakHours.getInstance(conf);
2261   }
2262 
2263   /**
2264    * {@inheritDoc}
2265    */
2266   @Override
2267   public void registerChildren(ConfigurationManager manager) {
2268     // No children to register
2269   }
2270 
2271   /**
2272    * {@inheritDoc}
2273    */
2274   @Override
2275   public void deregisterChildren(ConfigurationManager manager) {
2276     // No children to deregister
2277   }
2278 
2279   @Override
2280   public double getCompactionPressure() {
2281     return storeEngine.getStoreFileManager().getCompactionPressure();
2282   }
2283 }