001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver.storefiletracker;
019
020import java.io.IOException;
021import java.util.Collection;
022import java.util.List;
023import org.apache.hadoop.fs.FileStatus;
024import org.apache.hadoop.fs.Path;
025import org.apache.hadoop.hbase.TableName;
026import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
027import org.apache.hadoop.hbase.io.HFileLink;
028import org.apache.hadoop.hbase.io.Reference;
029import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams;
030import org.apache.hadoop.hbase.regionserver.HStoreFile;
031import org.apache.hadoop.hbase.regionserver.StoreContext;
032import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
033import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
034import org.apache.yetus.audience.InterfaceAudience;
035
036/**
037 * An interface to define how we track the store files for a give store.
038 * <p/>
039 * In the old time, we will write store to a tmp directory first, and then rename it to the actual
040 * data file. And once a store file is under data directory, we will consider it as 'committed'. And
041 * we need to do listing when loading store files.
042 * <p/>
043 * When cloud age is coming, now we want to store the store files on object storage, where rename
044 * and list are not as cheap as on HDFS, especially rename. Although introducing a metadata
045 * management layer for object storage could solve the problem, but we still want HBase to run on
046 * pure object storage, so here we introduce this interface to abstract how we track the store
047 * files. For the old implementation, we just persist nothing here, and do listing to load store
048 * files. When running on object storage, we could persist the store file list in a system region,
049 * or in a file on the object storage, to make it possible to write directly into the data directory
050 * to avoid renaming, and also avoid listing when loading store files.
051 * <p/>
052 * The implementation requires to be thread safe as flush and compaction may occur as the same time,
053 * and we could also do multiple compactions at the same time. As the implementation may choose to
054 * persist the store file list to external storage, which could be slow, it is the duty for the
055 * callers to not call it inside a lock which may block normal read/write requests.
056 */
057@InterfaceAudience.Private
058public interface StoreFileTracker {
059  /**
060   * Load the store files list when opening a region.
061   */
062  List<StoreFileInfo> load() throws IOException;
063
064  /**
065   * Add new store files.
066   * <p/>
067   * Used for flush and bulk load.
068   */
069  void add(Collection<StoreFileInfo> newFiles) throws IOException;
070
071  /**
072   * Add new store files and remove compacted store files after compaction.
073   */
074  void replace(Collection<StoreFileInfo> compactedFiles, Collection<StoreFileInfo> newFiles)
075    throws IOException;
076
077  /**
078   * Set the store files.
079   */
080  void set(List<StoreFileInfo> files) throws IOException;
081
082  /**
083   * Create a writer for writing new store files.
084   * @return Writer for a new StoreFile
085   */
086  StoreFileWriter createWriter(CreateStoreFileWriterParams params) throws IOException;
087
088  /**
089   * Adds StoreFileTracker implementations specific configurations into the table descriptor.
090   * <p/>
091   * This is used to avoid accidentally data loss when changing the cluster level store file tracker
092   * implementation, and also possible misconfiguration between master and region servers.
093   * <p/>
094   * See HBASE-26246 for more details.
095   * @param builder The table descriptor builder for the given table.
096   */
097  TableDescriptorBuilder updateWithTrackerConfigs(TableDescriptorBuilder builder);
098
099  /**
100   * Whether the implementation of this tracker requires you to write to temp directory first, i.e,
101   * does not allow broken store files under the actual data directory.
102   */
103  boolean requireWritingToTmpDirFirst();
104
105  Reference createReference(Reference reference, Path path) throws IOException;
106
107  Reference createAndCommitReference(Reference reference, Path path) throws IOException;
108
109  /**
110   * Reads the reference file from the given path.
111   * @param path the {@link Path} to the reference file in the file system.
112   * @return a {@link Reference} that points at top/bottom half of a an hfile
113   */
114  Reference readReference(Path path) throws IOException;
115
116  /**
117   * Returns true if the specified family has reference files
118   * @return true if family contains reference files
119   */
120  boolean hasReferences() throws IOException;
121
122  StoreFileInfo getStoreFileInfo(final FileStatus fileStatus, final Path initialPath,
123    final boolean primaryReplica) throws IOException;
124
125  StoreFileInfo getStoreFileInfo(final Path initialPath, final boolean primaryReplica)
126    throws IOException;
127
128  /**
129   * Create a new HFileLink
130   * <p>
131   * It also adds a back-reference to the hfile back-reference directory to simplify the
132   * reference-count and the cleaning process.
133   * @param hfileLinkName - HFileLink name (it contains hfile-region-table)
134   * @param createBackRef - Whether back reference should be created. Defaults to true.
135   * @return the file link name.
136   * @throws IOException on file or parent directory creation failure.
137   */
138  HFileLink createHFileLink(final TableName linkedTable, final String linkedRegion,
139    final String hfileName, final boolean createBackRef) throws IOException;
140
141  /**
142   * Create a new HFileLink and add to SFT
143   * <p>
144   * It also adds a back-reference to the hfile back-reference directory to simplify the
145   * reference-count and the cleaning process.
146   * @param hfileLinkName - HFileLink name (it contains hfile-region-table)
147   * @param createBackRef - Whether back reference should be created. Defaults to true.
148   * @return the file link name.
149   * @throws IOException on file or parent directory creation failure.
150   */
151  HFileLink createAndCommitHFileLink(final TableName linkedTable, final String linkedRegion,
152    final String hfileName, final boolean createBackRef) throws IOException;
153
154  /**
155   * Create a new HFileLink starting from a hfileLink name
156   * <p>
157   * It also adds a back-reference to the hfile back-reference directory to simplify the
158   * reference-count and the cleaning process.
159   * @param hfileLinkName - HFileLink name (it contains hfile-region-table)
160   * @param createBackRef - Whether back reference should be created. Defaults to true.
161   * @return the file link name.
162   * @throws IOException on file or parent directory creation failure.
163   */
164  HFileLink createFromHFileLink(final String hfileName, final boolean createBackRef)
165    throws IOException;
166
167  /**
168   * Closes and archives the specified store files from the specified family.
169   * @param storeFiles set of store files to remove
170   * @throws IOException if the archiving fails
171   */
172  void removeStoreFiles(List<HStoreFile> storeFiles) throws IOException;
173
174  /**
175   * Get the store context. Get the store context.
176   * @return the store context.
177   */
178  StoreContext getStoreContext();
179}