001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import java.io.IOException;
021import java.util.Collection;
022import java.util.Comparator;
023import java.util.Iterator;
024import java.util.List;
025import java.util.Optional;
026import java.util.stream.Collectors;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.Cell;
029import org.apache.hadoop.hbase.CellComparator;
030import org.apache.hadoop.hbase.KeyValue;
031import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
032import org.apache.yetus.audience.InterfaceAudience;
033import org.slf4j.Logger;
034import org.slf4j.LoggerFactory;
035
036import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableCollection;
037import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList;
038import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
039
040/**
041 * Default implementation of StoreFileManager. Not thread-safe.
042 */
043@InterfaceAudience.Private
044class DefaultStoreFileManager implements StoreFileManager {
045  private static final Logger LOG = LoggerFactory.getLogger(DefaultStoreFileManager.class);
046
047  private final CellComparator cellComparator;
048  private final CompactionConfiguration comConf;
049  private final int blockingFileCount;
050  private final Comparator<HStoreFile> storeFileComparator;
051  /**
052   * List of store files inside this store. This is an immutable list that is atomically replaced
053   * when its contents change.
054   */
055  private volatile ImmutableList<HStoreFile> storefiles = ImmutableList.of();
056  /**
057   * List of compacted files inside this store that needs to be excluded in reads because further
058   * new reads will be using only the newly created files out of compaction. These compacted files
059   * will be deleted/cleared once all the existing readers on these compacted files are done.
060   */
061  private volatile ImmutableList<HStoreFile> compactedfiles = ImmutableList.of();
062
063  public DefaultStoreFileManager(CellComparator cellComparator,
064    Comparator<HStoreFile> storeFileComparator, Configuration conf,
065    CompactionConfiguration comConf) {
066    this.cellComparator = cellComparator;
067    this.storeFileComparator = storeFileComparator;
068    this.comConf = comConf;
069    this.blockingFileCount =
070      conf.getInt(HStore.BLOCKING_STOREFILES_KEY, HStore.DEFAULT_BLOCKING_STOREFILE_COUNT);
071  }
072
073  @Override
074  public void loadFiles(List<HStoreFile> storeFiles) {
075    this.storefiles = ImmutableList.sortedCopyOf(storeFileComparator, storeFiles);
076  }
077
078  @Override
079  public final Collection<HStoreFile> getStorefiles() {
080    return storefiles;
081  }
082
083  @Override
084  public Collection<HStoreFile> getCompactedfiles() {
085    return compactedfiles;
086  }
087
088  @Override
089  public void insertNewFiles(Collection<HStoreFile> sfs) {
090    this.storefiles =
091      ImmutableList.sortedCopyOf(storeFileComparator, Iterables.concat(this.storefiles, sfs));
092  }
093
094  @Override
095  public ImmutableCollection<HStoreFile> clearFiles() {
096    ImmutableList<HStoreFile> result = storefiles;
097    storefiles = ImmutableList.of();
098    return result;
099  }
100
101  @Override
102  public Collection<HStoreFile> clearCompactedFiles() {
103    List<HStoreFile> result = compactedfiles;
104    compactedfiles = ImmutableList.of();
105    return result;
106  }
107
108  @Override
109  public final int getStorefileCount() {
110    return storefiles.size();
111  }
112
113  @Override
114  public final int getCompactedFilesCount() {
115    return compactedfiles.size();
116  }
117
118  @Override
119  public void addCompactionResults(Collection<HStoreFile> newCompactedfiles,
120    Collection<HStoreFile> results) {
121    this.storefiles = ImmutableList.sortedCopyOf(storeFileComparator, Iterables
122      .concat(Iterables.filter(storefiles, sf -> !newCompactedfiles.contains(sf)), results));
123    // Mark the files as compactedAway once the storefiles and compactedfiles list is finalized
124    // Let a background thread close the actual reader on these compacted files and also
125    // ensure to evict the blocks from block cache so that they are no longer in
126    // cache
127    newCompactedfiles.forEach(HStoreFile::markCompactedAway);
128    this.compactedfiles = ImmutableList.sortedCopyOf(storeFileComparator,
129      Iterables.concat(this.compactedfiles, newCompactedfiles));
130  }
131
132  @Override
133  public void removeCompactedFiles(Collection<HStoreFile> removedCompactedfiles) {
134    this.compactedfiles =
135      this.compactedfiles.stream().filter(sf -> !removedCompactedfiles.contains(sf))
136        .sorted(storeFileComparator).collect(ImmutableList.toImmutableList());
137  }
138
139  @Override
140  public final Iterator<HStoreFile> getCandidateFilesForRowKeyBefore(KeyValue targetKey) {
141    return this.storefiles.reverse().iterator();
142  }
143
144  @Override
145  public Iterator<HStoreFile> updateCandidateFilesForRowKeyBefore(
146    Iterator<HStoreFile> candidateFiles, KeyValue targetKey, Cell candidate) {
147    // Default store has nothing useful to do here.
148    // TODO: move this comment when implementing Level:
149    // Level store can trim the list by range, removing all the files which cannot have
150    // any useful candidates less than "candidate".
151    return candidateFiles;
152  }
153
154  @Override
155  public final Optional<byte[]> getSplitPoint() throws IOException {
156    return StoreUtils.getSplitPoint(storefiles, cellComparator);
157  }
158
159  @Override
160  public final Collection<HStoreFile> getFilesForScan(byte[] startRow, boolean includeStartRow,
161    byte[] stopRow, boolean includeStopRow) {
162    // We cannot provide any useful input and already have the files sorted by seqNum.
163    return getStorefiles();
164  }
165
166  @Override
167  public int getStoreCompactionPriority() {
168    int priority = blockingFileCount - storefiles.size();
169    return (priority == HStore.PRIORITY_USER) ? priority + 1 : priority;
170  }
171
172  @Override
173  public Collection<HStoreFile> getUnneededFiles(long maxTs, List<HStoreFile> filesCompacting) {
174    ImmutableList<HStoreFile> files = storefiles;
175    // 1) We can never get rid of the last file which has the maximum seqid.
176    // 2) Files that are not the latest can't become one due to (1), so the rest are fair game.
177    return files.stream().limit(Math.max(0, files.size() - 1)).filter(sf -> {
178      long fileTs = sf.getReader().getMaxTimestamp();
179      if (fileTs < maxTs && !filesCompacting.contains(sf)) {
180        LOG.info("Found an expired store file {} whose maxTimestamp is {}, which is below {}",
181          sf.getPath(), fileTs, maxTs);
182        return true;
183      } else {
184        return false;
185      }
186    }).collect(Collectors.toList());
187  }
188
189  @Override
190  public double getCompactionPressure() {
191    int storefileCount = getStorefileCount();
192    int minFilesToCompact = comConf.getMinFilesToCompact();
193    if (storefileCount <= minFilesToCompact) {
194      return 0.0;
195    }
196    return (double) (storefileCount - minFilesToCompact) / (blockingFileCount - minFilesToCompact);
197  }
198
199  @Override
200  public Comparator<HStoreFile> getStoreFileComparator() {
201    return storeFileComparator;
202  }
203}