001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.mob;
019
020import java.io.FileNotFoundException;
021import java.io.IOException;
022import java.util.ArrayList;
023import java.util.Date;
024import java.util.HashSet;
025import java.util.List;
026import java.util.Set;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.fs.FileSystem;
029import org.apache.hadoop.fs.LocatedFileStatus;
030import org.apache.hadoop.fs.Path;
031import org.apache.hadoop.fs.RemoteIterator;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.backup.HFileArchiver;
034import org.apache.hadoop.hbase.client.Admin;
035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
036import org.apache.hadoop.hbase.client.TableDescriptor;
037import org.apache.hadoop.hbase.io.hfile.CacheConfig;
038import org.apache.hadoop.hbase.regionserver.BloomType;
039import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
040import org.apache.hadoop.hbase.regionserver.HStoreFile;
041import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
042import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
043import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
044import org.apache.hadoop.hbase.util.Bytes;
045import org.apache.hadoop.hbase.util.CommonFSUtils;
046import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
047import org.apache.hadoop.hbase.util.FSUtils;
048import org.apache.yetus.audience.InterfaceAudience;
049import org.slf4j.Logger;
050import org.slf4j.LoggerFactory;
051
052import org.apache.hbase.thirdparty.com.google.common.collect.SetMultimap;
053
054@InterfaceAudience.Private
055public final class MobFileCleanupUtil {
056
057  private static final Logger LOG = LoggerFactory.getLogger(MobFileCleanupUtil.class);
058
059  private MobFileCleanupUtil() {
060  }
061
062  /**
063   * Performs housekeeping file cleaning (called by MOB Cleaner chore)
064   * @param conf  configuration
065   * @param table table name
066   * @throws IOException exception
067   */
068  public static void cleanupObsoleteMobFiles(Configuration conf, TableName table, Admin admin)
069    throws IOException {
070    long minAgeToArchive =
071      conf.getLong(MobConstants.MIN_AGE_TO_ARCHIVE_KEY, MobConstants.DEFAULT_MIN_AGE_TO_ARCHIVE);
072    // We check only those MOB files, which creation time is less
073    // than maxCreationTimeToArchive. This is a current time - 1h. 1 hour gap
074    // gives us full confidence that all corresponding store files will
075    // exist at the time cleaning procedure begins and will be examined.
076    // So, if MOB file creation time is greater than this maxTimeToArchive,
077    // this will be skipped and won't be archived.
078    long maxCreationTimeToArchive = EnvironmentEdgeManager.currentTime() - minAgeToArchive;
079    TableDescriptor htd = admin.getDescriptor(table);
080    List<ColumnFamilyDescriptor> list = MobUtils.getMobColumnFamilies(htd);
081    if (list.size() == 0) {
082      LOG.info("Skipping non-MOB table [{}]", table);
083      return;
084    } else {
085      LOG.info("Only MOB files whose creation time older than {} will be archived, table={}",
086        new Date(maxCreationTimeToArchive), table);
087    }
088
089    FileSystem fs = FileSystem.get(conf);
090    Set<String> regionNames = new HashSet<>();
091    Path rootDir = CommonFSUtils.getRootDir(conf);
092    Path tableDir = CommonFSUtils.getTableDir(rootDir, table);
093    List<Path> regionDirs = FSUtils.getRegionDirs(fs, tableDir);
094
095    Set<String> allActiveMobFileName = new HashSet<String>();
096    for (Path regionPath : regionDirs) {
097      regionNames.add(regionPath.getName());
098      HRegionFileSystem regionFS = HRegionFileSystem.create(conf, fs, tableDir,
099        HRegionFileSystem.loadRegionInfoFileContent(fs, regionPath));
100      for (ColumnFamilyDescriptor hcd : list) {
101        StoreFileTracker sft = StoreFileTrackerFactory.create(conf, htd, hcd, regionFS, false);
102        String family = hcd.getNameAsString();
103        Path storePath = new Path(regionPath, family);
104        boolean succeed = false;
105        Set<String> regionMobs = new HashSet<String>();
106
107        while (!succeed) {
108          if (!fs.exists(storePath)) {
109            String errMsg = String.format("Directory %s was deleted during MOB file cleaner chore"
110              + " execution, aborting MOB file cleaner chore.", storePath);
111            throw new IOException(errMsg);
112          }
113          List<StoreFileInfo> storeFileInfos = sft.load();
114          LOG.info("Found {} store files in: {}", storeFileInfos.size(), storePath);
115          Path currentPath = null;
116          try {
117            for (StoreFileInfo storeFileInfo : storeFileInfos) {
118              Path pp = storeFileInfo.getPath();
119              currentPath = pp;
120              LOG.trace("Store file: {}", pp);
121              HStoreFile sf = null;
122              byte[] mobRefData = null;
123              byte[] bulkloadMarkerData = null;
124              try {
125                sf = new HStoreFile(storeFileInfo, BloomType.NONE, CacheConfig.DISABLED);
126                sf.initReader();
127                mobRefData = sf.getMetadataValue(HStoreFile.MOB_FILE_REFS);
128                bulkloadMarkerData = sf.getMetadataValue(HStoreFile.BULKLOAD_TASK_KEY);
129                // close store file to avoid memory leaks
130                sf.closeStoreFile(true);
131              } catch (IOException ex) {
132                // When FileBased SFT is active the store dir can contain corrupted or incomplete
133                // files. So read errors are expected. We just skip these files.
134                if (ex instanceof FileNotFoundException) {
135                  throw ex;
136                }
137                LOG.debug("Failed to get mob data from file: {} due to error.", pp.toString(), ex);
138                continue;
139              }
140              if (mobRefData == null) {
141                if (bulkloadMarkerData == null) {
142                  LOG.warn("Found old store file with no MOB_FILE_REFS: {} - "
143                    + "can not proceed until all old files will be MOB-compacted.", pp);
144                  return;
145                } else {
146                  LOG.debug("Skipping file without MOB references (bulkloaded file):{}", pp);
147                  continue;
148                }
149              }
150              // file may or may not have MOB references, but was created by the distributed
151              // mob compaction code.
152              try {
153                SetMultimap<TableName, String> mobs =
154                  MobUtils.deserializeMobFileRefs(mobRefData).build();
155                LOG.debug("Found {} mob references for store={}", mobs.size(), sf);
156                LOG.trace("Specific mob references found for store={} : {}", sf, mobs);
157                regionMobs.addAll(mobs.values());
158              } catch (RuntimeException exception) {
159                throw new IOException("failure getting mob references for hfile " + sf, exception);
160              }
161            }
162          } catch (FileNotFoundException e) {
163            LOG.warn(
164              "Missing file:{} Starting MOB cleaning cycle from the beginning" + " due to error",
165              currentPath, e);
166            regionMobs.clear();
167            continue;
168          }
169          succeed = true;
170        }
171
172        // Add MOB references for current region/family
173        allActiveMobFileName.addAll(regionMobs);
174      } // END column families
175    } // END regions
176    // Check if number of MOB files too big (over 1M)
177    if (allActiveMobFileName.size() > 1000000) {
178      LOG.warn("Found too many active MOB files: {}, table={}, "
179        + "this may result in high memory pressure.", allActiveMobFileName.size(), table);
180    }
181    LOG.debug("Found: {} active mob refs for table={}", allActiveMobFileName.size(), table);
182    allActiveMobFileName.stream().forEach(LOG::trace);
183
184    // Now scan MOB directories and find MOB files with no references to them
185    for (ColumnFamilyDescriptor hcd : list) {
186      checkColumnFamilyDescriptor(conf, table, fs, admin, hcd, regionNames,
187        maxCreationTimeToArchive);
188    }
189  }
190
191  private static void checkColumnFamilyDescriptor(Configuration conf, TableName table,
192    FileSystem fs, Admin admin, ColumnFamilyDescriptor hcd, Set<String> regionNames,
193    long maxCreationTimeToArchive) throws IOException {
194    List<Path> toArchive = new ArrayList<Path>();
195    String family = hcd.getNameAsString();
196    Path dir = MobUtils.getMobFamilyPath(conf, table, family);
197    RemoteIterator<LocatedFileStatus> rit = fs.listLocatedStatus(dir);
198    while (rit.hasNext()) {
199      LocatedFileStatus lfs = rit.next();
200      Path p = lfs.getPath();
201      String[] mobParts = p.getName().split("_");
202      String regionName = mobParts[mobParts.length - 1];
203
204      if (!regionNames.contains(regionName)) {
205        // MOB belonged to a region no longer hosted
206        long creationTime = fs.getFileStatus(p).getModificationTime();
207        if (creationTime < maxCreationTimeToArchive) {
208          LOG.trace("Archiving MOB file {} creation time={}", p,
209            (fs.getFileStatus(p).getModificationTime()));
210          toArchive.add(p);
211        } else {
212          LOG.trace("Skipping fresh file: {}. Creation time={}", p,
213            fs.getFileStatus(p).getModificationTime());
214        }
215      } else {
216        LOG.trace("Keeping MOB file with existing region: {}", p);
217      }
218    }
219    LOG.info(" MOB Cleaner found {} files to archive for table={} family={}", toArchive.size(),
220      table, family);
221    archiveMobFiles(conf, table, admin, family.getBytes(), toArchive);
222    LOG.info(" MOB Cleaner archived {} files, table={} family={}", toArchive.size(), table, family);
223  }
224
225  /**
226   * Archives the mob files.
227   * @param conf       The current configuration.
228   * @param tableName  The table name.
229   * @param family     The name of the column family.
230   * @param storeFiles The files to be archived.
231   * @throws IOException exception
232   */
233  private static void archiveMobFiles(Configuration conf, TableName tableName, Admin admin,
234    byte[] family, List<Path> storeFiles) throws IOException {
235
236    if (storeFiles.size() == 0) {
237      // nothing to remove
238      LOG.debug("Skipping archiving old MOB files - no files found for table={} cf={}", tableName,
239        Bytes.toString(family));
240      return;
241    }
242    Path mobTableDir = CommonFSUtils.getTableDir(MobUtils.getMobHome(conf), tableName);
243    FileSystem fs = storeFiles.get(0).getFileSystem(conf);
244
245    for (Path p : storeFiles) {
246      LOG.debug("MOB Cleaner is archiving: {}", p);
247      HFileArchiver.archiveStoreFile(conf, fs, MobUtils.getMobRegionInfo(tableName), mobTableDir,
248        family, p);
249    }
250  }
251}