001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver.storefiletracker; 019 020import java.io.IOException; 021import java.util.Collection; 022import java.util.List; 023import org.apache.hadoop.fs.FileStatus; 024import org.apache.hadoop.fs.Path; 025import org.apache.hadoop.hbase.TableName; 026import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 027import org.apache.hadoop.hbase.io.Reference; 028import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; 029import org.apache.hadoop.hbase.regionserver.HStoreFile; 030import org.apache.hadoop.hbase.regionserver.StoreFileInfo; 031import org.apache.hadoop.hbase.regionserver.StoreFileWriter; 032import org.apache.yetus.audience.InterfaceAudience; 033 034/** 035 * An interface to define how we track the store files for a give store. 036 * <p/> 037 * In the old time, we will write store to a tmp directory first, and then rename it to the actual 038 * data file. And once a store file is under data directory, we will consider it as 'committed'. And 039 * we need to do listing when loading store files. 040 * <p/> 041 * When cloud age is coming, now we want to store the store files on object storage, where rename 042 * and list are not as cheap as on HDFS, especially rename. Although introducing a metadata 043 * management layer for object storage could solve the problem, but we still want HBase to run on 044 * pure object storage, so here we introduce this interface to abstract how we track the store 045 * files. For the old implementation, we just persist nothing here, and do listing to load store 046 * files. When running on object storage, we could persist the store file list in a system region, 047 * or in a file on the object storage, to make it possible to write directly into the data directory 048 * to avoid renaming, and also avoid listing when loading store files. 049 * <p/> 050 * The implementation requires to be thread safe as flush and compaction may occur as the same time, 051 * and we could also do multiple compactions at the same time. As the implementation may choose to 052 * persist the store file list to external storage, which could be slow, it is the duty for the 053 * callers to not call it inside a lock which may block normal read/write requests. 054 */ 055@InterfaceAudience.Private 056public interface StoreFileTracker { 057 /** 058 * Load the store files list when opening a region. 059 */ 060 List<StoreFileInfo> load() throws IOException; 061 062 /** 063 * Add new store files. 064 * <p/> 065 * Used for flush and bulk load. 066 */ 067 void add(Collection<StoreFileInfo> newFiles) throws IOException; 068 069 /** 070 * Add new store files and remove compacted store files after compaction. 071 */ 072 void replace(Collection<StoreFileInfo> compactedFiles, Collection<StoreFileInfo> newFiles) 073 throws IOException; 074 075 /** 076 * Set the store files. 077 */ 078 void set(List<StoreFileInfo> files) throws IOException; 079 080 /** 081 * Create a writer for writing new store files. 082 * @return Writer for a new StoreFile 083 */ 084 StoreFileWriter createWriter(CreateStoreFileWriterParams params) throws IOException; 085 086 /** 087 * Adds StoreFileTracker implementations specific configurations into the table descriptor. 088 * <p/> 089 * This is used to avoid accidentally data loss when changing the cluster level store file tracker 090 * implementation, and also possible misconfiguration between master and region servers. 091 * <p/> 092 * See HBASE-26246 for more details. 093 * @param builder The table descriptor builder for the given table. 094 */ 095 TableDescriptorBuilder updateWithTrackerConfigs(TableDescriptorBuilder builder); 096 097 /** 098 * Whether the implementation of this tracker requires you to write to temp directory first, i.e, 099 * does not allow broken store files under the actual data directory. 100 */ 101 boolean requireWritingToTmpDirFirst(); 102 103 Reference createReference(Reference reference, Path path) throws IOException; 104 105 /** 106 * Reads the reference file from the given path. 107 * @param path the {@link Path} to the reference file in the file system. 108 * @return a {@link Reference} that points at top/bottom half of a an hfile 109 */ 110 Reference readReference(Path path) throws IOException; 111 112 /** 113 * Returns true if the specified family has reference files 114 * @return true if family contains reference files 115 */ 116 boolean hasReferences() throws IOException; 117 118 StoreFileInfo getStoreFileInfo(final FileStatus fileStatus, final Path initialPath, 119 final boolean primaryReplica) throws IOException; 120 121 StoreFileInfo getStoreFileInfo(final Path initialPath, final boolean primaryReplica) 122 throws IOException; 123 124 /** 125 * Create a new HFileLink 126 * <p> 127 * It also adds a back-reference to the hfile back-reference directory to simplify the 128 * reference-count and the cleaning process. 129 * @param hfileLinkName - HFileLink name (it contains hfile-region-table) 130 * @param createBackRef - Whether back reference should be created. Defaults to true. 131 * @return the file link name. 132 * @throws IOException on file or parent directory creation failure. 133 */ 134 String createHFileLink(final TableName linkedTable, final String linkedRegion, 135 final String hfileName, final boolean createBackRef) throws IOException; 136 137 /** 138 * Create a new HFileLink starting from a hfileLink name 139 * <p> 140 * It also adds a back-reference to the hfile back-reference directory to simplify the 141 * reference-count and the cleaning process. 142 * @param hfileLinkName - HFileLink name (it contains hfile-region-table) 143 * @param createBackRef - Whether back reference should be created. Defaults to true. 144 * @return the file link name. 145 * @throws IOException on file or parent directory creation failure. 146 */ 147 String createFromHFileLink(final String hfileName, final boolean createBackRef) 148 throws IOException; 149 150 /** 151 * Closes and archives the specified store files from the specified family. 152 * @param storeFiles set of store files to remove 153 * @throws IOException if the archiving fails 154 */ 155 void removeStoreFiles(List<HStoreFile> storeFiles) throws IOException; 156}