View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.Collection;
24  import java.util.Collections;
25  import java.util.Comparator;
26  import java.util.Iterator;
27  import java.util.List;
28  
29  import com.google.common.collect.ImmutableCollection;
30  import com.google.common.collect.ImmutableList;
31  import com.google.common.collect.Lists;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.conf.Configuration;
36  import org.apache.hadoop.hbase.Cell;
37  import org.apache.hadoop.hbase.CellComparator;
38  import org.apache.hadoop.hbase.KeyValue;
39  import org.apache.hadoop.hbase.classification.InterfaceAudience;
40  import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
41  
42  /**
43   * Default implementation of StoreFileManager. Not thread-safe.
44   */
45  @InterfaceAudience.Private
46  class DefaultStoreFileManager implements StoreFileManager {
47    private static final Log LOG = LogFactory.getLog(DefaultStoreFileManager.class);
48  
49    private final CellComparator kvComparator;
50    private final CompactionConfiguration comConf;
51    private final int blockingFileCount;
52    private final Comparator<StoreFile> storeFileComparator;
53    /**
54     * List of store files inside this store. This is an immutable list that
55     * is atomically replaced when its contents change.
56     */
57    private volatile ImmutableList<StoreFile> storefiles = null;
58    /**
59     * List of compacted files inside this store that needs to be excluded in reads
60     * because further new reads will be using only the newly created files out of compaction.
61     * These compacted files will be deleted/cleared once all the existing readers on these
62     * compacted files are done.
63     */
64    private volatile List<StoreFile> compactedfiles = null;
65  
66    public DefaultStoreFileManager(CellComparator kvComparator,
67        Comparator<StoreFile> storeFileComparator, Configuration conf,
68        CompactionConfiguration comConf) {
69      this.kvComparator = kvComparator;
70      this.storeFileComparator = storeFileComparator;
71      this.comConf = comConf;
72      this.blockingFileCount =
73          conf.getInt(HStore.BLOCKING_STOREFILES_KEY, HStore.DEFAULT_BLOCKING_STOREFILE_COUNT);
74    }
75  
76    @Override
77    public void loadFiles(List<StoreFile> storeFiles) {
78      sortAndSetStoreFiles(storeFiles);
79    }
80  
81    @Override
82    public final Collection<StoreFile> getStorefiles() {
83      // TODO: I can return a null list of StoreFiles? That'll mess up clients. St.Ack 20151111
84      return storefiles;
85    }
86  
87    @Override
88    public Collection<StoreFile> getCompactedfiles() {
89      return compactedfiles;
90    }
91  
92    @Override
93    public void insertNewFiles(Collection<StoreFile> sfs) throws IOException {
94      ArrayList<StoreFile> newFiles = new ArrayList<StoreFile>(storefiles);
95      newFiles.addAll(sfs);
96      sortAndSetStoreFiles(newFiles);
97    }
98  
99    @Override
100   public ImmutableCollection<StoreFile> clearFiles() {
101     ImmutableList<StoreFile> result = storefiles;
102     storefiles = ImmutableList.of();
103     return result;
104   }
105 
106   @Override
107   public Collection<StoreFile> clearCompactedFiles() {
108     List<StoreFile> result = compactedfiles;
109     compactedfiles = new ArrayList<StoreFile>();
110     return result;
111   }
112 
113   @Override
114   public final int getStorefileCount() {
115     return storefiles.size();
116   }
117 
118   @Override
119   public void addCompactionResults(
120     Collection<StoreFile> newCompactedfiles, Collection<StoreFile> results) {
121     ArrayList<StoreFile> newStoreFiles = Lists.newArrayList(storefiles);
122     newStoreFiles.removeAll(newCompactedfiles);
123     if (!results.isEmpty()) {
124       newStoreFiles.addAll(results);
125     }
126     sortAndSetStoreFiles(newStoreFiles);
127     ArrayList<StoreFile> updatedCompactedfiles = null;
128     if (this.compactedfiles != null) {
129       updatedCompactedfiles = new ArrayList<StoreFile>(this.compactedfiles);
130       updatedCompactedfiles.addAll(newCompactedfiles);
131     } else {
132       updatedCompactedfiles = new ArrayList<StoreFile>(newCompactedfiles);
133     }
134     markCompactedAway(newCompactedfiles);
135     this.compactedfiles = sortCompactedfiles(updatedCompactedfiles);
136   }
137 
138   // Mark the files as compactedAway once the storefiles and compactedfiles list is finalised
139   // Let a background thread close the actual reader on these compacted files and also
140   // ensure to evict the blocks from block cache so that they are no longer in
141   // cache
142   private void markCompactedAway(Collection<StoreFile> compactedFiles) {
143     for (StoreFile file : compactedFiles) {
144       file.markCompactedAway();
145     }
146   }
147 
148   @Override
149   public void removeCompactedFiles(Collection<StoreFile> removedCompactedfiles) throws IOException {
150     ArrayList<StoreFile> updatedCompactedfiles = null;
151     if (this.compactedfiles != null) {
152       updatedCompactedfiles = new ArrayList<StoreFile>(this.compactedfiles);
153       updatedCompactedfiles.removeAll(removedCompactedfiles);
154       this.compactedfiles = sortCompactedfiles(updatedCompactedfiles);
155     }
156   }
157 
158   @Override
159   public final Iterator<StoreFile> getCandidateFilesForRowKeyBefore(final KeyValue targetKey) {
160     return new ArrayList<StoreFile>(Lists.reverse(this.storefiles)).iterator();
161   }
162 
163   @Override
164   public Iterator<StoreFile> updateCandidateFilesForRowKeyBefore(
165       Iterator<StoreFile> candidateFiles, final KeyValue targetKey, final Cell candidate) {
166     // Default store has nothing useful to do here.
167     // TODO: move this comment when implementing Level:
168     // Level store can trim the list by range, removing all the files which cannot have
169     // any useful candidates less than "candidate".
170     return candidateFiles;
171   }
172 
173   @Override
174   public final byte[] getSplitPoint() throws IOException {
175     if (this.storefiles.isEmpty()) {
176       return null;
177     }
178     return StoreUtils.getLargestFile(this.storefiles).getFileSplitPoint(this.kvComparator);
179   }
180 
181   @Override
182   public final Collection<StoreFile> getFilesForScanOrGet(boolean isGet,
183       byte[] startRow, byte[] stopRow) {
184     // We cannot provide any useful input and already have the files sorted by seqNum.
185     return getStorefiles();
186   }
187 
188   @Override
189   public int getStoreCompactionPriority() {
190     int priority = blockingFileCount - storefiles.size();
191     return (priority == HStore.PRIORITY_USER) ? priority + 1 : priority;
192   }
193 
194   @Override
195   public Collection<StoreFile> getUnneededFiles(long maxTs, List<StoreFile> filesCompacting) {
196     Collection<StoreFile> expiredStoreFiles = null;
197     ImmutableList<StoreFile> files = storefiles;
198     // 1) We can never get rid of the last file which has the maximum seqid.
199     // 2) Files that are not the latest can't become one due to (1), so the rest are fair game.
200     for (int i = 0; i < files.size() - 1; ++i) {
201       StoreFile sf = files.get(i);
202       long fileTs = sf.getReader().getMaxTimestamp();
203       if (fileTs < maxTs && !filesCompacting.contains(sf)) {
204         LOG.info("Found an expired store file: " + sf.getPath()
205             + " whose maxTimeStamp is " + fileTs + ", which is below " + maxTs);
206         if (expiredStoreFiles == null) {
207           expiredStoreFiles = new ArrayList<StoreFile>();
208         }
209         expiredStoreFiles.add(sf);
210       }
211     }
212     return expiredStoreFiles;
213   }
214 
215   private void sortAndSetStoreFiles(List<StoreFile> storeFiles) {
216     Collections.sort(storeFiles, storeFileComparator);
217     storefiles = ImmutableList.copyOf(storeFiles);
218   }
219 
220   private List<StoreFile> sortCompactedfiles(List<StoreFile> storefiles) {
221     // Sorting may not be really needed here for the compacted files?
222     Collections.sort(storefiles, storeFileComparator);
223     return new ArrayList<StoreFile>(storefiles);
224   }
225 
226   @Override
227   public double getCompactionPressure() {
228     int storefileCount = getStorefileCount();
229     int minFilesToCompact = comConf.getMinFilesToCompact();
230     if (storefileCount <= minFilesToCompact) {
231       return 0.0;
232     }
233     return (double) (storefileCount - minFilesToCompact) / (blockingFileCount - minFilesToCompact);
234   }
235 
236   @Override
237   public Comparator<StoreFile> getStoreFileComparator() {
238     return storeFileComparator;
239   }
240 }
241