001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import java.io.IOException; 021import java.util.Collection; 022import java.util.Comparator; 023import java.util.Iterator; 024import java.util.List; 025import java.util.Optional; 026import java.util.stream.Collectors; 027import org.apache.hadoop.conf.Configuration; 028import org.apache.hadoop.hbase.Cell; 029import org.apache.hadoop.hbase.CellComparator; 030import org.apache.hadoop.hbase.KeyValue; 031import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration; 032import org.apache.yetus.audience.InterfaceAudience; 033import org.slf4j.Logger; 034import org.slf4j.LoggerFactory; 035 036import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableCollection; 037import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; 038import org.apache.hbase.thirdparty.com.google.common.collect.Iterables; 039 040/** 041 * Default implementation of StoreFileManager. Not thread-safe. 042 */ 043@InterfaceAudience.Private 044class DefaultStoreFileManager implements StoreFileManager { 045 private static final Logger LOG = LoggerFactory.getLogger(DefaultStoreFileManager.class); 046 047 private final CellComparator cellComparator; 048 private final CompactionConfiguration comConf; 049 private final int blockingFileCount; 050 private final Comparator<HStoreFile> storeFileComparator; 051 /** 052 * List of store files inside this store. This is an immutable list that is atomically replaced 053 * when its contents change. 054 */ 055 private volatile ImmutableList<HStoreFile> storefiles = ImmutableList.of(); 056 /** 057 * List of compacted files inside this store that needs to be excluded in reads because further 058 * new reads will be using only the newly created files out of compaction. These compacted files 059 * will be deleted/cleared once all the existing readers on these compacted files are done. 060 */ 061 private volatile ImmutableList<HStoreFile> compactedfiles = ImmutableList.of(); 062 063 public DefaultStoreFileManager(CellComparator cellComparator, 064 Comparator<HStoreFile> storeFileComparator, Configuration conf, 065 CompactionConfiguration comConf) { 066 this.cellComparator = cellComparator; 067 this.storeFileComparator = storeFileComparator; 068 this.comConf = comConf; 069 this.blockingFileCount = 070 conf.getInt(HStore.BLOCKING_STOREFILES_KEY, HStore.DEFAULT_BLOCKING_STOREFILE_COUNT); 071 } 072 073 @Override 074 public void loadFiles(List<HStoreFile> storeFiles) { 075 this.storefiles = ImmutableList.sortedCopyOf(storeFileComparator, storeFiles); 076 } 077 078 @Override 079 public final Collection<HStoreFile> getStorefiles() { 080 return storefiles; 081 } 082 083 @Override 084 public Collection<HStoreFile> getCompactedfiles() { 085 return compactedfiles; 086 } 087 088 @Override 089 public void insertNewFiles(Collection<HStoreFile> sfs) { 090 this.storefiles = 091 ImmutableList.sortedCopyOf(storeFileComparator, Iterables.concat(this.storefiles, sfs)); 092 } 093 094 @Override 095 public ImmutableCollection<HStoreFile> clearFiles() { 096 ImmutableList<HStoreFile> result = storefiles; 097 storefiles = ImmutableList.of(); 098 return result; 099 } 100 101 @Override 102 public Collection<HStoreFile> clearCompactedFiles() { 103 List<HStoreFile> result = compactedfiles; 104 compactedfiles = ImmutableList.of(); 105 return result; 106 } 107 108 @Override 109 public final int getStorefileCount() { 110 return storefiles.size(); 111 } 112 113 @Override 114 public final int getCompactedFilesCount() { 115 return compactedfiles.size(); 116 } 117 118 @Override 119 public void addCompactionResults(Collection<HStoreFile> newCompactedfiles, 120 Collection<HStoreFile> results) { 121 this.storefiles = ImmutableList.sortedCopyOf(storeFileComparator, Iterables 122 .concat(Iterables.filter(storefiles, sf -> !newCompactedfiles.contains(sf)), results)); 123 // Mark the files as compactedAway once the storefiles and compactedfiles list is finalized 124 // Let a background thread close the actual reader on these compacted files and also 125 // ensure to evict the blocks from block cache so that they are no longer in 126 // cache 127 newCompactedfiles.forEach(HStoreFile::markCompactedAway); 128 this.compactedfiles = ImmutableList.sortedCopyOf(storeFileComparator, 129 Iterables.concat(this.compactedfiles, newCompactedfiles)); 130 } 131 132 @Override 133 public void removeCompactedFiles(Collection<HStoreFile> removedCompactedfiles) { 134 this.compactedfiles = 135 this.compactedfiles.stream().filter(sf -> !removedCompactedfiles.contains(sf)) 136 .sorted(storeFileComparator).collect(ImmutableList.toImmutableList()); 137 } 138 139 @Override 140 public final Iterator<HStoreFile> getCandidateFilesForRowKeyBefore(KeyValue targetKey) { 141 return this.storefiles.reverse().iterator(); 142 } 143 144 @Override 145 public Iterator<HStoreFile> updateCandidateFilesForRowKeyBefore( 146 Iterator<HStoreFile> candidateFiles, KeyValue targetKey, Cell candidate) { 147 // Default store has nothing useful to do here. 148 // TODO: move this comment when implementing Level: 149 // Level store can trim the list by range, removing all the files which cannot have 150 // any useful candidates less than "candidate". 151 return candidateFiles; 152 } 153 154 @Override 155 public final Optional<byte[]> getSplitPoint() throws IOException { 156 return StoreUtils.getSplitPoint(storefiles, cellComparator); 157 } 158 159 @Override 160 public final Collection<HStoreFile> getFilesForScan(byte[] startRow, boolean includeStartRow, 161 byte[] stopRow, boolean includeStopRow) { 162 // We cannot provide any useful input and already have the files sorted by seqNum. 163 return getStorefiles(); 164 } 165 166 @Override 167 public int getStoreCompactionPriority() { 168 int priority = blockingFileCount - storefiles.size(); 169 return (priority == HStore.PRIORITY_USER) ? priority + 1 : priority; 170 } 171 172 @Override 173 public Collection<HStoreFile> getUnneededFiles(long maxTs, List<HStoreFile> filesCompacting) { 174 ImmutableList<HStoreFile> files = storefiles; 175 // 1) We can never get rid of the last file which has the maximum seqid. 176 // 2) Files that are not the latest can't become one due to (1), so the rest are fair game. 177 return files.stream().limit(Math.max(0, files.size() - 1)).filter(sf -> { 178 long fileTs = sf.getReader().getMaxTimestamp(); 179 if (fileTs < maxTs && !filesCompacting.contains(sf)) { 180 LOG.info("Found an expired store file {} whose maxTimestamp is {}, which is below {}", 181 sf.getPath(), fileTs, maxTs); 182 return true; 183 } else { 184 return false; 185 } 186 }).collect(Collectors.toList()); 187 } 188 189 @Override 190 public double getCompactionPressure() { 191 int storefileCount = getStorefileCount(); 192 int minFilesToCompact = comConf.getMinFilesToCompact(); 193 if (storefileCount <= minFilesToCompact) { 194 return 0.0; 195 } 196 return (double) (storefileCount - minFilesToCompact) / (blockingFileCount - minFilesToCompact); 197 } 198 199 @Override 200 public Comparator<HStoreFile> getStoreFileComparator() { 201 return storeFileComparator; 202 } 203}