View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  import java.io.IOException;
21  import java.util.Collection;
22  import java.util.List;
23  import java.util.NavigableSet;
24  
25  import org.apache.hadoop.hbase.classification.InterfaceAudience;
26  import org.apache.hadoop.hbase.classification.InterfaceStability;
27  import org.apache.hadoop.fs.FileSystem;
28  import org.apache.hadoop.fs.Path;
29  import org.apache.hadoop.hbase.Cell;
30  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
31  import org.apache.hadoop.hbase.HColumnDescriptor;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.KeyValue;
34  import org.apache.hadoop.hbase.TableName;
35  import org.apache.hadoop.hbase.client.Scan;
36  import org.apache.hadoop.hbase.conf.PropagatingConfigurationObserver;
37  import org.apache.hadoop.hbase.io.HeapSize;
38  import org.apache.hadoop.hbase.io.compress.Compression;
39  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
40  import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
41  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
42  import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
43  import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
44  import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
45  import org.apache.hadoop.hbase.regionserver.compactions.CompactionThroughputController;
46  import org.apache.hadoop.hbase.util.Pair;
47  
48  /**
49   * Interface for objects that hold a column family in a Region. Its a memstore and a set of zero or
50   * more StoreFiles, which stretch backwards over time.
51   */
52  @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC)
53  @InterfaceStability.Evolving
54  public interface Store extends HeapSize, StoreConfigInformation, PropagatingConfigurationObserver {
55  
56    /* The default priority for user-specified compaction requests.
57     * The user gets top priority unless we have blocking compactions. (Pri <= 0)
58     */ int PRIORITY_USER = 1;
59    int NO_PRIORITY = Integer.MIN_VALUE;
60  
61    // General Accessors
62    KeyValue.KVComparator getComparator();
63  
64    Collection<StoreFile> getStorefiles();
65  
66    /**
67     * Close all the readers We don't need to worry about subsequent requests because the Region
68     * holds a write lock that will prevent any more reads or writes.
69     * @return the {@link StoreFile StoreFiles} that were previously being used.
70     * @throws IOException on failure
71     */
72    Collection<StoreFile> close() throws IOException;
73  
74    /**
75     * Return a scanner for both the memstore and the HStore files. Assumes we are not in a
76     * compaction.
77     * @param scan Scan to apply when scanning the stores
78     * @param targetCols columns to scan
79     * @return a scanner over the current key values
80     * @throws IOException on failure
81     */
82    KeyValueScanner getScanner(Scan scan, final NavigableSet<byte[]> targetCols, long readPt)
83        throws IOException;
84  
85    /**
86     * Get all scanners with no filtering based on TTL (that happens further down
87     * the line).
88     * @param cacheBlocks
89     * @param isGet
90     * @param usePread
91     * @param isCompaction
92     * @param matcher
93     * @param startRow
94     * @param stopRow
95     * @param readPt
96     * @return all scanners for this store
97     */
98    List<KeyValueScanner> getScanners(
99      boolean cacheBlocks,
100     boolean isGet,
101     boolean usePread,
102     boolean isCompaction,
103     ScanQueryMatcher matcher,
104     byte[] startRow,
105     byte[] stopRow,
106     long readPt
107   ) throws IOException;
108 
109   ScanInfo getScanInfo();
110 
111   /**
112    * Adds or replaces the specified KeyValues.
113    * <p>
114    * For each KeyValue specified, if a cell with the same row, family, and qualifier exists in
115    * MemStore, it will be replaced. Otherwise, it will just be inserted to MemStore.
116    * <p>
117    * This operation is atomic on each KeyValue (row/family/qualifier) but not necessarily atomic
118    * across all of them.
119    * @param cells
120    * @param readpoint readpoint below which we can safely remove duplicate KVs
121    * @return memstore size delta
122    * @throws IOException
123    */
124   long upsert(Iterable<Cell> cells, long readpoint) throws IOException;
125 
126   /**
127    * Adds a value to the memstore
128    * @param cell
129    * @return memstore size delta & newly added KV which maybe different than the passed in KV
130    */
131   Pair<Long, Cell> add(Cell cell);
132 
133   /**
134    * When was the last edit done in the memstore
135    */
136   long timeOfOldestEdit();
137 
138   /**
139    * Removes a Cell from the memstore. The Cell is removed only if its key & memstoreTS match the
140    * key & memstoreTS value of the cell parameter.
141    * @param cell
142    */
143   void rollback(final Cell cell);
144 
145   /**
146    * Find the key that matches <i>row</i> exactly, or the one that immediately precedes it. WARNING:
147    * Only use this method on a table where writes occur with strictly increasing timestamps. This
148    * method assumes this pattern of writes in order to make it reasonably performant. Also our
149    * search is dependent on the axiom that deletes are for cells that are in the container that
150    * follows whether a memstore snapshot or a storefile, not for the current container: i.e. we'll
151    * see deletes before we come across cells we are to delete. Presumption is that the
152    * memstore#kvset is processed before memstore#snapshot and so on.
153    * @param row The row key of the targeted row.
154    * @return Found Cell or null if none found.
155    * @throws IOException
156    */
157   Cell getRowKeyAtOrBefore(final byte[] row) throws IOException;
158 
159   FileSystem getFileSystem();
160 
161   /*
162    * @param maxKeyCount
163    * @param compression Compression algorithm to use
164    * @param isCompaction whether we are creating a new file in a compaction
165    * @param includeMVCCReadpoint whether we should out the MVCC readpoint
166    * @return Writer for a new StoreFile in the tmp dir.
167    */
168   StoreFile.Writer createWriterInTmp(
169     long maxKeyCount,
170     Compression.Algorithm compression,
171     boolean isCompaction,
172     boolean includeMVCCReadpoint,
173     boolean includesTags
174   ) throws IOException;
175 
176   // Compaction oriented methods
177 
178   boolean throttleCompaction(long compactionSize);
179 
180   /**
181    * getter for CompactionProgress object
182    * @return CompactionProgress object; can be null
183    */
184   CompactionProgress getCompactionProgress();
185 
186   CompactionContext requestCompaction() throws IOException;
187 
188   CompactionContext requestCompaction(int priority, CompactionRequest baseRequest)
189       throws IOException;
190 
191   void cancelRequestedCompaction(CompactionContext compaction);
192 
193   List<StoreFile> compact(CompactionContext compaction,
194       CompactionThroughputController throughputController) throws IOException;
195 
196   /**
197    * @return true if we should run a major compaction.
198    */
199   boolean isMajorCompaction() throws IOException;
200 
201   void triggerMajorCompaction();
202 
203   /**
204    * See if there's too much store files in this store
205    * @return true if number of store files is greater than the number defined in minFilesToCompact
206    */
207   boolean needsCompaction();
208 
209   int getCompactPriority();
210 
211   StoreFlushContext createFlushContext(long cacheFlushId);
212 
213   /**
214    * Call to complete a compaction. Its for the case where we find in the WAL a compaction
215    * that was not finished.  We could find one recovering a WAL after a regionserver crash.
216    * See HBASE-2331.
217    * @param compaction the descriptor for compaction
218    * @param pickCompactionFiles whether or not pick up the new compaction output files and
219    * add it to the store
220    * @param removeFiles whether to remove/archive files from filesystem
221    */
222   void replayCompactionMarker(CompactionDescriptor compaction, boolean pickCompactionFiles,
223       boolean removeFiles)
224       throws IOException;
225 
226   // Split oriented methods
227 
228   boolean canSplit();
229 
230   /**
231    * Determines if Store should be split
232    * @return byte[] if store should be split, null otherwise.
233    */
234   byte[] getSplitPoint();
235 
236   // Bulk Load methods
237 
238   /**
239    * This throws a WrongRegionException if the HFile does not fit in this region, or an
240    * InvalidHFileException if the HFile is not valid.
241    */
242   void assertBulkLoadHFileOk(Path srcPath) throws IOException;
243 
244   /**
245    * This method should only be called from Region. It is assumed that the ranges of values in the
246    * HFile fit within the stores assigned region. (assertBulkLoadHFileOk checks this)
247    *
248    * @param srcPathStr
249    * @param sequenceId sequence Id associated with the HFile
250    */
251   Path bulkLoadHFile(String srcPathStr, long sequenceId) throws IOException;
252 
253   // General accessors into the state of the store
254   // TODO abstract some of this out into a metrics class
255 
256   /**
257    * @return <tt>true</tt> if the store has any underlying reference files to older HFiles
258    */
259   boolean hasReferences();
260 
261   /**
262    * @return The size of this store's memstore, in bytes
263    */
264   long getMemStoreSize();
265 
266   /**
267    * @return The amount of memory we could flush from this memstore; usually this is equal to
268    * {@link #getMemStoreSize()} unless we are carrying snapshots and then it will be the size of
269    * outstanding snapshots.
270    */
271   long getFlushableSize();
272 
273   /**
274    * Returns the memstore snapshot size
275    * @return size of the memstore snapshot
276    */
277   long getSnapshotSize();
278 
279   HColumnDescriptor getFamily();
280 
281   /**
282    * @return The maximum sequence id in all store files.
283    */
284   long getMaxSequenceId();
285 
286   /**
287    * @return The maximum memstoreTS in all store files.
288    */
289   long getMaxMemstoreTS();
290 
291   /**
292    * @return the data block encoder
293    */
294   HFileDataBlockEncoder getDataBlockEncoder();
295 
296   /** @return aggregate size of all HStores used in the last compaction */
297   long getLastCompactSize();
298 
299   /** @return aggregate size of HStore */
300   long getSize();
301 
302   /**
303    * @return Count of store files
304    */
305   int getStorefilesCount();
306 
307   /**
308    * @return The size of the store files, in bytes, uncompressed.
309    */
310   long getStoreSizeUncompressed();
311 
312   /**
313    * @return The size of the store files, in bytes.
314    */
315   long getStorefilesSize();
316 
317   /**
318    * @return The size of the store file indexes, in bytes.
319    */
320   long getStorefilesIndexSize();
321 
322   /**
323    * Returns the total size of all index blocks in the data block indexes, including the root level,
324    * intermediate levels, and the leaf level for multi-level indexes, or just the root level for
325    * single-level indexes.
326    * @return the total size of block indexes in the store
327    */
328   long getTotalStaticIndexSize();
329 
330   /**
331    * Returns the total byte size of all Bloom filter bit arrays. For compound Bloom filters even the
332    * Bloom blocks currently not loaded into the block cache are counted.
333    * @return the total size of all Bloom filters in the store
334    */
335   long getTotalStaticBloomSize();
336 
337   // Test-helper methods
338 
339   /**
340    * Used for tests.
341    * @return cache configuration for this Store.
342    */
343   CacheConfig getCacheConfig();
344 
345   /**
346    * @return the parent region info hosting this store
347    */
348   HRegionInfo getRegionInfo();
349 
350   RegionCoprocessorHost getCoprocessorHost();
351 
352   boolean areWritesEnabled();
353 
354   /**
355    * @return The smallest mvcc readPoint across all the scanners in this
356    * region. Writes older than this readPoint, are included  in every
357    * read operation.
358    */
359   long getSmallestReadPoint();
360 
361   String getColumnFamilyName();
362 
363   TableName getTableName();
364 
365   /**
366    * @return The number of cells flushed to disk
367    */
368   long getFlushedCellsCount();
369 
370   /**
371    * @return The total size of data flushed to disk, in bytes
372    */
373   long getFlushedCellsSize();
374 
375   /**
376    * @return The number of cells processed during minor compactions
377    */
378   long getCompactedCellsCount();
379 
380   /**
381    * @return The total amount of data processed during minor compactions, in bytes
382    */
383   long getCompactedCellsSize();
384 
385   /**
386    * @return The number of cells processed during major compactions
387    */
388   long getMajorCompactedCellsCount();
389 
390   /**
391    * @return The total amount of data processed during major compactions, in bytes
392    */
393   long getMajorCompactedCellsSize();
394 
395   /*
396    * @param o Observer who wants to know about changes in set of Readers
397    */
398   void addChangedReaderObserver(ChangedReadersObserver o);
399 
400   /*
401    * @param o Observer no longer interested in changes in set of Readers.
402    */
403   void deleteChangedReaderObserver(ChangedReadersObserver o);
404 
405   /**
406    * @return Whether this store has too many store files.
407    */
408   boolean hasTooManyStoreFiles();
409 
410   /**
411    * Checks the underlying store files, and opens the files that  have not
412    * been opened, and removes the store file readers for store files no longer
413    * available. Mainly used by secondary region replicas to keep up to date with
414    * the primary region files.
415    * @throws IOException
416    */
417   void refreshStoreFiles() throws IOException;
418 
419   /**
420    * This value can represent the degree of emergency of compaction for this store. It should be
421    * greater than or equal to 0.0, any value greater than 1.0 means we have too many store files.
422    * <ul>
423    * <li>if getStorefilesCount &lt;= getMinFilesToCompact, return 0.0</li>
424    * <li>return (getStorefilesCount - getMinFilesToCompact) / (blockingFileCount -
425    * getMinFilesToCompact)</li>
426    * </ul>
427    * <p>
428    * And for striped stores, we should calculate this value by the files in each stripe separately
429    * and return the maximum value.
430    * <p>
431    * It is similar to {@link #getCompactPriority()} except that it is more suitable to use in a
432    * linear formula.
433    */
434   double getCompactionPressure();
435 
436    /**
437     * Replaces the store files that the store has with the given files. Mainly used by
438     * secondary region replicas to keep up to date with
439     * the primary region files.
440     * @throws IOException
441     */
442   void refreshStoreFiles(Collection<String> newFiles) throws IOException;
443 
444   void bulkLoadHFile(StoreFileInfo fileInfo) throws IOException;
445 }