View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import org.apache.commons.logging.Log;
22  import org.apache.commons.logging.LogFactory;
23  import org.apache.hadoop.hbase.classification.InterfaceAudience;
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
26  
27  /**
28   * Configuration class for stripe store and compactions.
29   * See {@link StripeStoreFileManager} for general documentation.
30   * See getters for the description of each setting.
31   */
32  @InterfaceAudience.Private
33  public class StripeStoreConfig {
34    private static final Log LOG = LogFactory.getLog(StripeStoreConfig.class);
35  
36    /** The maximum number of files to compact within a stripe; same as for regular compaction. */
37    public static final String MAX_FILES_KEY = "hbase.store.stripe.compaction.maxFiles";
38    /** The minimum number of files to compact within a stripe; same as for regular compaction. */
39    public static final String MIN_FILES_KEY = "hbase.store.stripe.compaction.minFiles";
40  
41    /**  The minimum number of files to compact when compacting L0; same as minFiles for regular
42     * compaction. Given that L0 causes unnecessary overwriting of the data, should be higher than
43     * regular minFiles. */
44    public static final String MIN_FILES_L0_KEY = "hbase.store.stripe.compaction.minFilesL0";
45  
46    /** The size the stripe should achieve to be considered for splitting into multiple stripes.
47     Stripe will be split when it can be fully compacted, and it is above this size. */
48    public static final String SIZE_TO_SPLIT_KEY = "hbase.store.stripe.sizeToSplit";
49    /** The target count of new stripes to produce when splitting a stripe. A floating point
50     number, default is 2. Values less than 1 will be converted to 1/x. Non-whole numbers will
51     produce unbalanced splits, which may be good for some cases. In this case the "smaller" of
52     the new stripes will always be the rightmost one. If the stripe is bigger than sizeToSplit
53     when splitting, this will be adjusted by a whole increment. */
54    public static final String SPLIT_PARTS_KEY = "hbase.store.stripe.splitPartCount";
55    /** The initial stripe count to create. If the row distribution is roughly the same over time,
56     it's good to set this to a count of stripes that is expected to be achieved in most regions,
57     to get this count from the outset and prevent unnecessary splitting. */
58    public static final String INITIAL_STRIPE_COUNT_KEY = "hbase.store.stripe.initialStripeCount";
59  
60    /** Whether to flush memstore to L0 files, or directly to stripes. */
61    public static final String FLUSH_TO_L0_KEY = "hbase.store.stripe.compaction.flushToL0";
62  
63    /** When splitting region, the maximum size imbalance to allow in an attempt to split at a
64     stripe boundary, so that no files go to both regions. Most users won't need to change that. */
65    public static final String MAX_REGION_SPLIT_IMBALANCE_KEY =
66        "hbase.store.stripe.region.split.max.imbalance";
67  
68  
69    private final float maxRegionSplitImbalance;
70    private final int level0CompactMinFiles;
71    private final int stripeCompactMinFiles;
72    private final int stripeCompactMaxFiles;
73  
74    private final int initialCount;
75    private final long sizeToSplitAt;
76    private final float splitPartCount;
77    private final boolean flushIntoL0;
78    private final long splitPartSize; // derived from sizeToSplitAt and splitPartCount
79  
80    private static final double EPSILON = 0.001; // good enough for this, not a real epsilon.
81    public StripeStoreConfig(Configuration config, StoreConfigInformation sci) {
82      this.level0CompactMinFiles = config.getInt(MIN_FILES_L0_KEY, 4);
83      this.flushIntoL0 = config.getBoolean(FLUSH_TO_L0_KEY, false);
84      int minMinFiles = flushIntoL0 ? 3 : 4; // make sure not to compact tiny files too often.
85      int minFiles = config.getInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, -1);
86      this.stripeCompactMinFiles = config.getInt(MIN_FILES_KEY, Math.max(minMinFiles, minFiles));
87      this.stripeCompactMaxFiles = config.getInt(MAX_FILES_KEY,
88          config.getInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_KEY, 10));
89      this.maxRegionSplitImbalance = getFloat(config, MAX_REGION_SPLIT_IMBALANCE_KEY, 1.5f, true);
90  
91      float splitPartCount = getFloat(config, SPLIT_PARTS_KEY, 2f, true);
92      if (Math.abs(splitPartCount - 1.0) < EPSILON) {
93        LOG.error("Split part count cannot be 1 (" + splitPartCount + "), using the default");
94        splitPartCount = 2f;
95      }
96      this.splitPartCount = splitPartCount;
97      // Arbitrary default split size - 4 times the size of one L0 compaction.
98      // If we flush into L0 there's no split compaction, but for default value it is ok.
99      double flushSize = sci.getMemstoreFlushSize();
100     if (flushSize == 0) {
101       flushSize = 128 * 1024 * 1024;
102     }
103     long defaultSplitSize = (long)(flushSize * getLevel0MinFiles() * 4 * splitPartCount);
104     this.sizeToSplitAt = config.getLong(SIZE_TO_SPLIT_KEY, defaultSplitSize);
105     int initialCount = config.getInt(INITIAL_STRIPE_COUNT_KEY, 1);
106     if (initialCount == 0) {
107       LOG.error("Initial stripe count is 0, using the default");
108       initialCount = 1;
109     }
110     this.initialCount = initialCount;
111     this.splitPartSize = (long)(this.sizeToSplitAt / this.splitPartCount);
112   }
113 
114   private static float getFloat(
115       Configuration config, String key, float defaultValue, boolean moreThanOne) {
116     float value = config.getFloat(key, defaultValue);
117     if (value < EPSILON) {
118       LOG.warn(String.format(
119           "%s is set to 0 or negative; using default value of %f", key, defaultValue));
120       value = defaultValue;
121     } else if ((value > 1f) != moreThanOne) {
122       value = 1f / value;
123     }
124     return value;
125   }
126 
127   public float getMaxSplitImbalance() {
128     return this.maxRegionSplitImbalance;
129   }
130 
131   public int getLevel0MinFiles() {
132     return level0CompactMinFiles;
133   }
134 
135   public int getStripeCompactMinFiles() {
136     return stripeCompactMinFiles;
137   }
138 
139   public int getStripeCompactMaxFiles() {
140     return stripeCompactMaxFiles;
141   }
142 
143   public boolean isUsingL0Flush() {
144     return flushIntoL0;
145   }
146 
147   public long getSplitSize() {
148     return sizeToSplitAt;
149   }
150 
151   public int getInitialCount() {
152     return initialCount;
153   }
154 
155   public float getSplitCount() {
156     return splitPartCount;
157   }
158 
159   /**
160    * @return the desired size of the target stripe when splitting, in bytes.
161    *         Derived from {@link #getSplitSize()} and {@link #getSplitCount()}.
162    */
163   public long getSplitPartSize() {
164     return splitPartSize;
165   }
166 }