1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver.compactions;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.List;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.hbase.classification.InterfaceAudience;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
31 import org.apache.hadoop.hbase.regionserver.StoreFile;
32
33
34
35
36
37
38
39 @InterfaceAudience.Private
40 public class ExploringCompactionPolicy extends RatioBasedCompactionPolicy {
41 private static final Log LOG = LogFactory.getLog(ExploringCompactionPolicy.class);
42
43
44
45
46
47
48 public ExploringCompactionPolicy(final Configuration conf,
49 final StoreConfigInformation storeConfigInfo) {
50 super(conf, storeConfigInfo);
51 }
52
53 @Override
54 final ArrayList<StoreFile> applyCompactionPolicy(final ArrayList<StoreFile> candidates,
55 final boolean mayUseOffPeak, final boolean mightBeStuck) throws IOException {
56 return new ArrayList<StoreFile>(applyCompactionPolicy(candidates, mightBeStuck,
57 mayUseOffPeak, comConf.getMinFilesToCompact(), comConf.getMaxFilesToCompact()));
58 }
59
60 public List<StoreFile> applyCompactionPolicy(final List<StoreFile> candidates,
61 boolean mightBeStuck, boolean mayUseOffPeak, int minFiles, int maxFiles) {
62
63 final double currentRatio = mayUseOffPeak
64 ? comConf.getCompactionRatioOffPeak() : comConf.getCompactionRatio();
65
66
67 List<StoreFile> bestSelection = new ArrayList<StoreFile>(0);
68 List<StoreFile> smallest = mightBeStuck ? new ArrayList<StoreFile>(0) : null;
69 long bestSize = 0;
70 long smallestSize = Long.MAX_VALUE;
71
72 int opts = 0, optsInRatio = 0, bestStart = -1;
73
74 for (int start = 0; start < candidates.size(); start++) {
75
76 for (int currentEnd = start + minFiles - 1;
77 currentEnd < candidates.size(); currentEnd++) {
78 List<StoreFile> potentialMatchFiles = candidates.subList(start, currentEnd + 1);
79
80
81 if (potentialMatchFiles.size() < minFiles) {
82 continue;
83 }
84 if (potentialMatchFiles.size() > maxFiles) {
85 continue;
86 }
87
88
89
90 long size = getTotalStoreSize(potentialMatchFiles);
91
92
93
94 if (mightBeStuck && size < smallestSize) {
95 smallest = potentialMatchFiles;
96 smallestSize = size;
97 }
98
99 if (size > comConf.getMaxCompactSize()) {
100 continue;
101 }
102
103 ++opts;
104 if (size >= comConf.getMinCompactSize()
105 && !filesInRatio(potentialMatchFiles, currentRatio)) {
106 continue;
107 }
108
109 ++optsInRatio;
110 if (isBetterSelection(bestSelection, bestSize, potentialMatchFiles, size, mightBeStuck)) {
111 bestSelection = potentialMatchFiles;
112 bestSize = size;
113 bestStart = start;
114 }
115 }
116 }
117 if (bestSelection.size() == 0 && mightBeStuck) {
118 LOG.debug("Exploring compaction algorithm has selected " + smallest.size()
119 + " files of size "+ smallestSize + " because the store might be stuck");
120 return new ArrayList<StoreFile>(smallest);
121 }
122 LOG.debug("Exploring compaction algorithm has selected " + bestSelection.size()
123 + " files of size " + bestSize + " starting at candidate #" + bestStart +
124 " after considering " + opts + " permutations with " + optsInRatio + " in ratio");
125 return new ArrayList<StoreFile>(bestSelection);
126 }
127
128 private boolean isBetterSelection(List<StoreFile> bestSelection,
129 long bestSize, List<StoreFile> selection, long size, boolean mightBeStuck) {
130 if (mightBeStuck && bestSize > 0 && size > 0) {
131
132
133
134
135 final double REPLACE_IF_BETTER_BY = 1.05;
136 double thresholdQuality = ((double)bestSelection.size() / bestSize) * REPLACE_IF_BETTER_BY;
137 return thresholdQuality < ((double)selection.size() / size);
138 }
139
140 return selection.size() > bestSelection.size()
141 || (selection.size() == bestSelection.size() && size < bestSize);
142 }
143
144
145
146
147
148
149 private long getTotalStoreSize(final List<StoreFile> potentialMatchFiles) {
150 long size = 0;
151
152 for (StoreFile s:potentialMatchFiles) {
153 size += s.getReader().length();
154 }
155 return size;
156 }
157
158
159
160
161
162
163
164
165
166 private boolean filesInRatio(final List<StoreFile> files, final double currentRatio) {
167 if (files.size() < 2) {
168 return true;
169 }
170
171 long totalFileSize = getTotalStoreSize(files);
172
173 for (StoreFile file : files) {
174 long singleFileSize = file.getReader().length();
175 long sumAllOtherFileSizes = totalFileSize - singleFileSize;
176
177 if (singleFileSize > sumAllOtherFileSizes * currentRatio) {
178 return false;
179 }
180 }
181 return true;
182 }
183 }