1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.util;
21
22 import java.io.DataInput;
23 import java.io.IOException;
24
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.hbase.classification.InterfaceAudience;
28 import org.apache.hadoop.conf.Configuration;
29 import org.apache.hadoop.hbase.KeyValue;
30 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
31 import org.apache.hadoop.hbase.io.hfile.HFile;
32 import org.apache.hadoop.hbase.regionserver.BloomType;
33
34
35
36
37
38 @InterfaceAudience.Private
39 public final class BloomFilterFactory {
40
41 private static final Log LOG =
42 LogFactory.getLog(BloomFilterFactory.class.getName());
43
44
45 private BloomFilterFactory() {}
46
47
48
49
50
51 public static final String IO_STOREFILE_BLOOM_ERROR_RATE =
52 "io.storefile.bloom.error.rate";
53
54
55
56
57
58 public static final String IO_STOREFILE_BLOOM_MAX_FOLD =
59 "io.storefile.bloom.max.fold";
60
61
62
63
64
65 public static final String IO_STOREFILE_BLOOM_MAX_KEYS =
66 "io.storefile.bloom.max.keys";
67
68
69 public static final String IO_STOREFILE_BLOOM_ENABLED =
70 "io.storefile.bloom.enabled";
71
72
73 public static final String IO_STOREFILE_DELETEFAMILY_BLOOM_ENABLED =
74 "io.storefile.delete.family.bloom.enabled";
75
76
77
78
79
80 public static final String IO_STOREFILE_BLOOM_BLOCK_SIZE =
81 "io.storefile.bloom.block.size";
82
83
84 private static final int MAX_ALLOWED_FOLD_FACTOR = 7;
85
86
87
88
89
90
91
92
93
94
95
96
97 public static BloomFilter
98 createFromMeta(DataInput meta, HFile.Reader reader)
99 throws IllegalArgumentException, IOException {
100 int version = meta.readInt();
101 switch (version) {
102 case ByteBloomFilter.VERSION:
103
104
105
106 return new ByteBloomFilter(meta);
107
108 case CompoundBloomFilterBase.VERSION:
109 return new CompoundBloomFilter(meta, reader);
110
111 default:
112 throw new IllegalArgumentException(
113 "Bad bloom filter format version " + version
114 );
115 }
116 }
117
118
119
120
121
122 public static boolean isGeneralBloomEnabled(Configuration conf) {
123 return conf.getBoolean(IO_STOREFILE_BLOOM_ENABLED, true);
124 }
125
126
127
128
129 public static boolean isDeleteFamilyBloomEnabled(Configuration conf) {
130 return conf.getBoolean(IO_STOREFILE_DELETEFAMILY_BLOOM_ENABLED, true);
131 }
132
133
134
135
136 public static float getErrorRate(Configuration conf) {
137 return conf.getFloat(IO_STOREFILE_BLOOM_ERROR_RATE, (float) 0.01);
138 }
139
140
141
142
143 public static int getMaxFold(Configuration conf) {
144 return conf.getInt(IO_STOREFILE_BLOOM_MAX_FOLD, MAX_ALLOWED_FOLD_FACTOR);
145 }
146
147
148 public static int getBloomBlockSize(Configuration conf) {
149 return conf.getInt(IO_STOREFILE_BLOOM_BLOCK_SIZE, 128 * 1024);
150 }
151
152
153
154
155 public static int getMaxKeys(Configuration conf) {
156 return conf.getInt(IO_STOREFILE_BLOOM_MAX_KEYS, 128 * 1000 * 1000);
157 }
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172 public static BloomFilterWriter createGeneralBloomAtWrite(Configuration conf,
173 CacheConfig cacheConf, BloomType bloomType, int maxKeys,
174 HFile.Writer writer) {
175 if (!isGeneralBloomEnabled(conf)) {
176 LOG.trace("Bloom filters are disabled by configuration for "
177 + writer.getPath()
178 + (conf == null ? " (configuration is null)" : ""));
179 return null;
180 } else if (bloomType == BloomType.NONE) {
181 LOG.trace("Bloom filter is turned off for the column family");
182 return null;
183 }
184
185 float err = getErrorRate(conf);
186
187
188
189
190
191 if (bloomType == BloomType.ROWCOL) {
192 err = (float) (1 - Math.sqrt(1 - err));
193 }
194
195 int maxFold = conf.getInt(IO_STOREFILE_BLOOM_MAX_FOLD,
196 MAX_ALLOWED_FOLD_FACTOR);
197
198
199
200 CompoundBloomFilterWriter bloomWriter = new CompoundBloomFilterWriter(getBloomBlockSize(conf),
201 err, Hash.getHashType(conf), maxFold, cacheConf.shouldCacheBloomsOnWrite(),
202 bloomType == BloomType.ROWCOL ? KeyValue.COMPARATOR : KeyValue.RAW_COMPARATOR);
203 writer.addInlineBlockWriter(bloomWriter);
204 return bloomWriter;
205 }
206
207
208
209
210
211
212
213
214
215
216
217
218 public static BloomFilterWriter createDeleteBloomAtWrite(Configuration conf,
219 CacheConfig cacheConf, int maxKeys, HFile.Writer writer) {
220 if (!isDeleteFamilyBloomEnabled(conf)) {
221 LOG.info("Delete Bloom filters are disabled by configuration for "
222 + writer.getPath()
223 + (conf == null ? " (configuration is null)" : ""));
224 return null;
225 }
226
227 float err = getErrorRate(conf);
228
229 int maxFold = getMaxFold(conf);
230
231 CompoundBloomFilterWriter bloomWriter = new CompoundBloomFilterWriter(getBloomBlockSize(conf),
232 err, Hash.getHashType(conf), maxFold, cacheConf.shouldCacheBloomsOnWrite(),
233 KeyValue.RAW_COMPARATOR);
234 writer.addInlineBlockWriter(bloomWriter);
235 return bloomWriter;
236 }
237 };