001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util.compaction;
019
020import java.io.IOException;
021import java.util.Collection;
022import java.util.List;
023import java.util.Optional;
024import java.util.Set;
025import org.apache.hadoop.conf.Configuration;
026import org.apache.hadoop.fs.FileStatus;
027import org.apache.hadoop.fs.FileSystem;
028import org.apache.hadoop.fs.Path;
029import org.apache.hadoop.hbase.client.Admin;
030import org.apache.hadoop.hbase.client.Connection;
031import org.apache.hadoop.hbase.client.ConnectionFactory;
032import org.apache.hadoop.hbase.client.RegionInfo;
033import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
034import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
035import org.apache.hadoop.hbase.util.FSUtils;
036import org.apache.yetus.audience.InterfaceAudience;
037import org.slf4j.Logger;
038import org.slf4j.LoggerFactory;
039import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
040import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
041
042@InterfaceAudience.Private
043class MajorCompactionRequest {
044
045  private static final Logger LOG = LoggerFactory.getLogger(MajorCompactionRequest.class);
046
047  protected final Configuration configuration;
048  protected final RegionInfo region;
049  private Set<String> stores;
050
051  MajorCompactionRequest(Configuration configuration, RegionInfo region) {
052    this.configuration = configuration;
053    this.region = region;
054  }
055
056  @VisibleForTesting
057  MajorCompactionRequest(Configuration configuration, RegionInfo region,
058      Set<String> stores) {
059    this(configuration, region);
060    this.stores = stores;
061  }
062
063  static Optional<MajorCompactionRequest> newRequest(Configuration configuration, RegionInfo info,
064      Set<String> stores, long timestamp) throws IOException {
065    MajorCompactionRequest request =
066        new MajorCompactionRequest(configuration, info, stores);
067    return request.createRequest(configuration, stores, timestamp);
068  }
069
070  RegionInfo getRegion() {
071    return region;
072  }
073
074  Set<String> getStores() {
075    return stores;
076  }
077
078  void setStores(Set<String> stores) {
079    this.stores = stores;
080  }
081
082  @VisibleForTesting
083  Optional<MajorCompactionRequest> createRequest(Configuration configuration,
084      Set<String> stores, long timestamp) throws IOException {
085    Set<String> familiesToCompact = getStoresRequiringCompaction(stores, timestamp);
086    MajorCompactionRequest request = null;
087    if (!familiesToCompact.isEmpty()) {
088      request = new MajorCompactionRequest(configuration, region, familiesToCompact);
089    }
090    return Optional.ofNullable(request);
091  }
092
093  Set<String> getStoresRequiringCompaction(Set<String> requestedStores, long timestamp)
094      throws IOException {
095    try(Connection connection = getConnection(configuration)) {
096      HRegionFileSystem fileSystem = getFileSystem(connection);
097      Set<String> familiesToCompact = Sets.newHashSet();
098      for (String family : requestedStores) {
099        if (shouldCFBeCompacted(fileSystem, family, timestamp)) {
100          familiesToCompact.add(family);
101        }
102      }
103      return familiesToCompact;
104    }
105  }
106
107  boolean shouldCFBeCompacted(HRegionFileSystem fileSystem, String family, long ts)
108      throws IOException {
109
110    // do we have any store files?
111    Collection<StoreFileInfo> storeFiles = fileSystem.getStoreFiles(family);
112    if (storeFiles == null) {
113      LOG.info("Excluding store: " + family + " for compaction for region:  " + fileSystem
114          .getRegionInfo().getEncodedName(), " has no store files");
115      return false;
116    }
117    // check for reference files
118    if (fileSystem.hasReferences(family) && familyHasReferenceFile(fileSystem, family, ts)) {
119      LOG.info("Including store: " + family + " with: " + storeFiles.size()
120          + " files for compaction for region: " + fileSystem.getRegionInfo().getEncodedName());
121      return true;
122    }
123    // check store file timestamps
124    boolean includeStore = this.shouldIncludeStore(fileSystem, family, storeFiles, ts);
125    if (!includeStore) {
126      LOG.info("Excluding store: " + family + " for compaction for region:  " + fileSystem
127          .getRegionInfo().getEncodedName() + " already compacted");
128    }
129    return includeStore;
130  }
131
132  protected boolean shouldIncludeStore(HRegionFileSystem fileSystem, String family,
133      Collection<StoreFileInfo> storeFiles, long ts) throws IOException {
134
135    for (StoreFileInfo storeFile : storeFiles) {
136      if (storeFile.getModificationTime() < ts) {
137        LOG.info("Including store: " + family + " with: " + storeFiles.size()
138            + " files for compaction for region: "
139            + fileSystem.getRegionInfo().getEncodedName());
140        return true;
141      }
142    }
143    return false;
144  }
145
146  @VisibleForTesting
147  Connection getConnection(Configuration configuration) throws IOException {
148    return ConnectionFactory.createConnection(configuration);
149  }
150
151  protected boolean familyHasReferenceFile(HRegionFileSystem fileSystem, String family, long ts)
152      throws IOException {
153    List<Path> referenceFiles =
154        getReferenceFilePaths(fileSystem.getFileSystem(), fileSystem.getStoreDir(family));
155    for (Path referenceFile : referenceFiles) {
156      FileStatus status = fileSystem.getFileSystem().getFileLinkStatus(referenceFile);
157      if (status.getModificationTime() < ts) {
158        LOG.info("Including store: " + family + " for compaction for region:  " + fileSystem
159            .getRegionInfo().getEncodedName() + " (reference store files)");
160        return true;
161      }
162    }
163    return false;
164
165  }
166
167  @VisibleForTesting
168  List<Path> getReferenceFilePaths(FileSystem fileSystem, Path familyDir)
169      throws IOException {
170    return FSUtils.getReferenceFilePaths(fileSystem, familyDir);
171  }
172
173  @VisibleForTesting
174  HRegionFileSystem getFileSystem(Connection connection) throws IOException {
175    Admin admin = connection.getAdmin();
176    return HRegionFileSystem.openRegionFromFileSystem(admin.getConfiguration(),
177        FSUtils.getCurrentFileSystem(admin.getConfiguration()),
178        FSUtils.getTableDir(FSUtils.getRootDir(admin.getConfiguration()), region.getTable()),
179        region, true);
180  }
181
182  @Override
183  public String toString() {
184    return "region: " + region.getEncodedName() + " store(s): " + stores;
185  }
186}