View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.Collection;
24  import java.util.Collections;
25  import java.util.Iterator;
26  import java.util.List;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.Cell;
32  import org.apache.hadoop.hbase.CellComparator;
33  import org.apache.hadoop.hbase.KeyValue;
34  import org.apache.hadoop.hbase.classification.InterfaceAudience;
35  import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
36  
37  import com.google.common.collect.ImmutableCollection;
38  import com.google.common.collect.ImmutableList;
39  import com.google.common.collect.Lists;
40  
41  /**
42   * Default implementation of StoreFileManager. Not thread-safe.
43   */
44  @InterfaceAudience.Private
45  class DefaultStoreFileManager implements StoreFileManager {
46    private static final Log LOG = LogFactory.getLog(DefaultStoreFileManager.class);
47  
48    private final CellComparator kvComparator;
49    private final CompactionConfiguration comConf;
50    private final int blockingFileCount;
51  
52    /**
53     * List of store files inside this store. This is an immutable list that
54     * is atomically replaced when its contents change.
55     */
56    private volatile ImmutableList<StoreFile> storefiles = null;
57    /**
58     * List of compacted files inside this store that needs to be excluded in reads
59     * because further new reads will be using only the newly created files out of compaction.
60     * These compacted files will be deleted/cleared once all the existing readers on these
61     * compacted files are done.
62     */
63    private volatile List<StoreFile> compactedfiles = null;
64  
65    public DefaultStoreFileManager(CellComparator kvComparator, Configuration conf,
66        CompactionConfiguration comConf) {
67      this.kvComparator = kvComparator;
68      this.comConf = comConf;
69      this.blockingFileCount =
70          conf.getInt(HStore.BLOCKING_STOREFILES_KEY, HStore.DEFAULT_BLOCKING_STOREFILE_COUNT);
71    }
72  
73    @Override
74    public void loadFiles(List<StoreFile> storeFiles) {
75      sortAndSetStoreFiles(storeFiles);
76    }
77  
78    @Override
79    public final Collection<StoreFile> getStorefiles() {
80      // TODO: I can return a null list of StoreFiles? That'll mess up clients. St.Ack 20151111
81      return storefiles;
82    }
83  
84    @Override
85    public Collection<StoreFile> getCompactedfiles() {
86      return compactedfiles;
87    }
88  
89    @Override
90    public void insertNewFiles(Collection<StoreFile> sfs) throws IOException {
91      ArrayList<StoreFile> newFiles = new ArrayList<StoreFile>(storefiles);
92      newFiles.addAll(sfs);
93      sortAndSetStoreFiles(newFiles);
94    }
95  
96    @Override
97    public ImmutableCollection<StoreFile> clearFiles() {
98      ImmutableList<StoreFile> result = storefiles;
99      storefiles = ImmutableList.of();
100     return result;
101   }
102 
103   @Override
104   public Collection<StoreFile> clearCompactedFiles() {
105     List<StoreFile> result = compactedfiles;
106     compactedfiles = new ArrayList<StoreFile>();
107     return result;
108   }
109 
110   @Override
111   public final int getStorefileCount() {
112     return storefiles.size();
113   }
114 
115   @Override
116   public void addCompactionResults(
117     Collection<StoreFile> newCompactedfiles, Collection<StoreFile> results) {
118     ArrayList<StoreFile> newStoreFiles = Lists.newArrayList(storefiles);
119     newStoreFiles.removeAll(newCompactedfiles);
120     if (!results.isEmpty()) {
121       newStoreFiles.addAll(results);
122     }
123     sortAndSetStoreFiles(newStoreFiles);
124     ArrayList<StoreFile> updatedCompactedfiles = null;
125     if (this.compactedfiles != null) {
126       updatedCompactedfiles = new ArrayList<StoreFile>(this.compactedfiles);
127       updatedCompactedfiles.addAll(newCompactedfiles);
128     } else {
129       updatedCompactedfiles = new ArrayList<StoreFile>(newCompactedfiles);
130     }
131     markCompactedAway(newCompactedfiles);
132     this.compactedfiles = sortCompactedfiles(updatedCompactedfiles);
133   }
134 
135   // Mark the files as compactedAway once the storefiles and compactedfiles list is finalised
136   // Let a background thread close the actual reader on these compacted files and also
137   // ensure to evict the blocks from block cache so that they are no longer in
138   // cache
139   private void markCompactedAway(Collection<StoreFile> compactedFiles) {
140     for (StoreFile file : compactedFiles) {
141       file.markCompactedAway();
142     }
143   }
144 
145   @Override
146   public void removeCompactedFiles(Collection<StoreFile> removedCompactedfiles) throws IOException {
147     ArrayList<StoreFile> updatedCompactedfiles = null;
148     if (this.compactedfiles != null) {
149       updatedCompactedfiles = new ArrayList<StoreFile>(this.compactedfiles);
150       updatedCompactedfiles.removeAll(removedCompactedfiles);
151       this.compactedfiles = sortCompactedfiles(updatedCompactedfiles);
152     }
153   }
154 
155   @Override
156   public final Iterator<StoreFile> getCandidateFilesForRowKeyBefore(final KeyValue targetKey) {
157     return new ArrayList<StoreFile>(Lists.reverse(this.storefiles)).iterator();
158   }
159 
160   @Override
161   public Iterator<StoreFile> updateCandidateFilesForRowKeyBefore(
162       Iterator<StoreFile> candidateFiles, final KeyValue targetKey, final Cell candidate) {
163     // Default store has nothing useful to do here.
164     // TODO: move this comment when implementing Level:
165     // Level store can trim the list by range, removing all the files which cannot have
166     // any useful candidates less than "candidate".
167     return candidateFiles;
168   }
169 
170   @Override
171   public final byte[] getSplitPoint() throws IOException {
172     if (this.storefiles.isEmpty()) {
173       return null;
174     }
175     return StoreUtils.getLargestFile(this.storefiles).getFileSplitPoint(this.kvComparator);
176   }
177 
178   @Override
179   public final Collection<StoreFile> getFilesForScanOrGet(boolean isGet,
180       byte[] startRow, byte[] stopRow) {
181     // We cannot provide any useful input and already have the files sorted by seqNum.
182     return getStorefiles();
183   }
184 
185   @Override
186   public int getStoreCompactionPriority() {
187     int priority = blockingFileCount - storefiles.size();
188     return (priority == HStore.PRIORITY_USER) ? priority + 1 : priority;
189   }
190 
191   @Override
192   public Collection<StoreFile> getUnneededFiles(long maxTs, List<StoreFile> filesCompacting) {
193     Collection<StoreFile> expiredStoreFiles = null;
194     ImmutableList<StoreFile> files = storefiles;
195     // 1) We can never get rid of the last file which has the maximum seqid.
196     // 2) Files that are not the latest can't become one due to (1), so the rest are fair game.
197     for (int i = 0; i < files.size() - 1; ++i) {
198       StoreFile sf = files.get(i);
199       long fileTs = sf.getReader().getMaxTimestamp();
200       if (fileTs < maxTs && !filesCompacting.contains(sf)) {
201         LOG.info("Found an expired store file: " + sf.getPath()
202             + " whose maxTimeStamp is " + fileTs + ", which is below " + maxTs);
203         if (expiredStoreFiles == null) {
204           expiredStoreFiles = new ArrayList<StoreFile>();
205         }
206         expiredStoreFiles.add(sf);
207       }
208     }
209     return expiredStoreFiles;
210   }
211 
212   private void sortAndSetStoreFiles(List<StoreFile> storeFiles) {
213     Collections.sort(storeFiles, StoreFile.Comparators.SEQ_ID);
214     storefiles = ImmutableList.copyOf(storeFiles);
215   }
216 
217   private List<StoreFile> sortCompactedfiles(List<StoreFile> storefiles) {
218     // Sorting may not be really needed here for the compacted files?
219     Collections.sort(storefiles, StoreFile.Comparators.SEQ_ID);
220     return new ArrayList<StoreFile>(storefiles);
221   }
222 
223   @Override
224   public double getCompactionPressure() {
225     int storefileCount = getStorefileCount();
226     int minFilesToCompact = comConf.getMinFilesToCompact();
227     if (storefileCount <= minFilesToCompact) {
228       return 0.0;
229     }
230     return (double) (storefileCount - minFilesToCompact) / (blockingFileCount - minFilesToCompact);
231   }
232 }
233