001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import org.apache.hadoop.conf.Configuration; 021import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration; 022import org.apache.yetus.audience.InterfaceAudience; 023import org.slf4j.Logger; 024import org.slf4j.LoggerFactory; 025 026/** 027 * Configuration class for stripe store and compactions. See {@link StripeStoreFileManager} for 028 * general documentation. See getters for the description of each setting. 029 */ 030@InterfaceAudience.Private 031public class StripeStoreConfig { 032 private static final Logger LOG = LoggerFactory.getLogger(StripeStoreConfig.class); 033 034 /** The maximum number of files to compact within a stripe; same as for regular compaction. */ 035 public static final String MAX_FILES_KEY = "hbase.store.stripe.compaction.maxFiles"; 036 /** The minimum number of files to compact within a stripe; same as for regular compaction. */ 037 public static final String MIN_FILES_KEY = "hbase.store.stripe.compaction.minFiles"; 038 039 /** 040 * The minimum number of files to compact when compacting L0; same as minFiles for regular 041 * compaction. Given that L0 causes unnecessary overwriting of the data, should be higher than 042 * regular minFiles. 043 */ 044 public static final String MIN_FILES_L0_KEY = "hbase.store.stripe.compaction.minFilesL0"; 045 046 /** 047 * The size the stripe should achieve to be considered for splitting into multiple stripes. Stripe 048 * will be split when it can be fully compacted, and it is above this size. 049 */ 050 public static final String SIZE_TO_SPLIT_KEY = "hbase.store.stripe.sizeToSplit"; 051 /** 052 * The target count of new stripes to produce when splitting a stripe. A floating point number, 053 * default is 2. Values less than 1 will be converted to 1/x. Non-whole numbers will produce 054 * unbalanced splits, which may be good for some cases. In this case the "smaller" of the new 055 * stripes will always be the rightmost one. If the stripe is bigger than sizeToSplit when 056 * splitting, this will be adjusted by a whole increment. 057 */ 058 public static final String SPLIT_PARTS_KEY = "hbase.store.stripe.splitPartCount"; 059 /** 060 * The initial stripe count to create. If the row distribution is roughly the same over time, it's 061 * good to set this to a count of stripes that is expected to be achieved in most regions, to get 062 * this count from the outset and prevent unnecessary splitting. 063 */ 064 public static final String INITIAL_STRIPE_COUNT_KEY = "hbase.store.stripe.initialStripeCount"; 065 066 /** Whether to flush memstore to L0 files, or directly to stripes. */ 067 public static final String FLUSH_TO_L0_KEY = "hbase.store.stripe.compaction.flushToL0"; 068 069 /** 070 * When splitting region, the maximum size imbalance to allow in an attempt to split at a stripe 071 * boundary, so that no files go to both regions. Most users won't need to change that. 072 */ 073 public static final String MAX_REGION_SPLIT_IMBALANCE_KEY = 074 "hbase.store.stripe.region.split.max.imbalance"; 075 076 private final float maxRegionSplitImbalance; 077 private final int level0CompactMinFiles; 078 private final int stripeCompactMinFiles; 079 private final int stripeCompactMaxFiles; 080 081 private final int initialCount; 082 private final long sizeToSplitAt; 083 private final float splitPartCount; 084 private final boolean flushIntoL0; 085 private final long splitPartSize; // derived from sizeToSplitAt and splitPartCount 086 087 private static final double EPSILON = 0.001; // good enough for this, not a real epsilon. 088 089 public StripeStoreConfig(Configuration config, StoreConfigInformation sci) { 090 this.level0CompactMinFiles = config.getInt(MIN_FILES_L0_KEY, 4); 091 this.flushIntoL0 = config.getBoolean(FLUSH_TO_L0_KEY, false); 092 int minMinFiles = flushIntoL0 ? 3 : 4; // make sure not to compact tiny files too often. 093 int minFiles = config.getInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, -1); 094 this.stripeCompactMinFiles = config.getInt(MIN_FILES_KEY, Math.max(minMinFiles, minFiles)); 095 this.stripeCompactMaxFiles = config.getInt(MAX_FILES_KEY, 096 config.getInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_KEY, 10)); 097 this.maxRegionSplitImbalance = getFloat(config, MAX_REGION_SPLIT_IMBALANCE_KEY, 1.5f, true); 098 099 float splitPartCount = getFloat(config, SPLIT_PARTS_KEY, 2f, true); 100 if (Math.abs(splitPartCount - 1.0) < EPSILON) { 101 LOG.error("Split part count cannot be 1 (" + splitPartCount + "), using the default"); 102 splitPartCount = 2f; 103 } 104 this.splitPartCount = splitPartCount; 105 // Arbitrary default split size - 4 times the size of one L0 compaction. 106 // If we flush into L0 there's no split compaction, but for default value it is ok. 107 double flushSize = sci.getMemStoreFlushSize(); 108 if (flushSize == 0) { 109 flushSize = 128 * 1024 * 1024; 110 } 111 long defaultSplitSize = (long) (flushSize * getLevel0MinFiles() * 4 * splitPartCount); 112 this.sizeToSplitAt = config.getLong(SIZE_TO_SPLIT_KEY, defaultSplitSize); 113 int initialCount = config.getInt(INITIAL_STRIPE_COUNT_KEY, 1); 114 if (initialCount == 0) { 115 LOG.error("Initial stripe count is 0, using the default"); 116 initialCount = 1; 117 } 118 this.initialCount = initialCount; 119 this.splitPartSize = (long) (this.sizeToSplitAt / this.splitPartCount); 120 } 121 122 private static float getFloat(Configuration config, String key, float defaultValue, 123 boolean moreThanOne) { 124 float value = config.getFloat(key, defaultValue); 125 if (value < EPSILON) { 126 LOG.warn( 127 String.format("%s is set to 0 or negative; using default value of %f", key, defaultValue)); 128 value = defaultValue; 129 } else if ((value > 1f) != moreThanOne) { 130 value = 1f / value; 131 } 132 return value; 133 } 134 135 public float getMaxSplitImbalance() { 136 return this.maxRegionSplitImbalance; 137 } 138 139 public int getLevel0MinFiles() { 140 return level0CompactMinFiles; 141 } 142 143 public int getStripeCompactMinFiles() { 144 return stripeCompactMinFiles; 145 } 146 147 public int getStripeCompactMaxFiles() { 148 return stripeCompactMaxFiles; 149 } 150 151 public boolean isUsingL0Flush() { 152 return flushIntoL0; 153 } 154 155 public long getSplitSize() { 156 return sizeToSplitAt; 157 } 158 159 public int getInitialCount() { 160 return initialCount; 161 } 162 163 public float getSplitCount() { 164 return splitPartCount; 165 } 166 167 /** 168 * @return the desired size of the target stripe when splitting, in bytes. Derived from 169 * {@link #getSplitSize()} and {@link #getSplitCount()}. 170 */ 171 public long getSplitPartSize() { 172 return splitPartSize; 173 } 174}