001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util.compaction;
019
020import java.io.IOException;
021import java.util.Collection;
022import java.util.List;
023import java.util.Optional;
024import java.util.Set;
025import org.apache.hadoop.conf.Configuration;
026import org.apache.hadoop.fs.FileStatus;
027import org.apache.hadoop.fs.FileSystem;
028import org.apache.hadoop.fs.Path;
029import org.apache.hadoop.hbase.client.Admin;
030import org.apache.hadoop.hbase.client.Connection;
031import org.apache.hadoop.hbase.client.ConnectionFactory;
032import org.apache.hadoop.hbase.client.RegionInfo;
033import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
034import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
035import org.apache.hadoop.hbase.util.FSUtils;
036import org.apache.yetus.audience.InterfaceAudience;
037import org.slf4j.Logger;
038import org.slf4j.LoggerFactory;
039import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
040import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
041
042@InterfaceAudience.Private
043class MajorCompactionRequest {
044
045  private static final Logger LOG = LoggerFactory.getLogger(MajorCompactionRequest.class);
046
047  private final Configuration configuration;
048  private final RegionInfo region;
049  private Set<String> stores;
050  private final long timestamp;
051
052  @VisibleForTesting
053  MajorCompactionRequest(Configuration configuration, RegionInfo region,
054      Set<String> stores, long timestamp) {
055    this.configuration = configuration;
056    this.region = region;
057    this.stores = stores;
058    this.timestamp = timestamp;
059  }
060
061  static Optional<MajorCompactionRequest> newRequest(Configuration configuration, RegionInfo info,
062      Set<String> stores, long timestamp) throws IOException {
063    MajorCompactionRequest request =
064        new MajorCompactionRequest(configuration, info, stores, timestamp);
065    return request.createRequest(configuration, stores);
066  }
067
068  RegionInfo getRegion() {
069    return region;
070  }
071
072  Set<String> getStores() {
073    return stores;
074  }
075
076  void setStores(Set<String> stores) {
077    this.stores = stores;
078  }
079
080  @VisibleForTesting
081  Optional<MajorCompactionRequest> createRequest(Configuration configuration,
082      Set<String> stores) throws IOException {
083    Set<String> familiesToCompact = getStoresRequiringCompaction(stores);
084    MajorCompactionRequest request = null;
085    if (!familiesToCompact.isEmpty()) {
086      request = new MajorCompactionRequest(configuration, region, familiesToCompact, timestamp);
087    }
088    return Optional.ofNullable(request);
089  }
090
091  Set<String> getStoresRequiringCompaction(Set<String> requestedStores) throws IOException {
092    try(Connection connection = getConnection(configuration)) {
093      HRegionFileSystem fileSystem = getFileSystem(connection);
094      Set<String> familiesToCompact = Sets.newHashSet();
095      for (String family : requestedStores) {
096        // do we have any store files?
097        Collection<StoreFileInfo> storeFiles = fileSystem.getStoreFiles(family);
098        if (storeFiles == null) {
099          LOG.info("Excluding store: " + family + " for compaction for region:  " + fileSystem
100              .getRegionInfo().getEncodedName(), " has no store files");
101          continue;
102        }
103        // check for reference files
104        if (fileSystem.hasReferences(family) && familyHasReferenceFile(fileSystem, family)) {
105          familiesToCompact.add(family);
106          LOG.info("Including store: " + family + " with: " + storeFiles.size()
107              + " files for compaction for region: " + fileSystem.getRegionInfo().getEncodedName());
108          continue;
109        }
110        // check store file timestamps
111        boolean includeStore = false;
112        for (StoreFileInfo storeFile : storeFiles) {
113          if (storeFile.getModificationTime() < timestamp) {
114            LOG.info("Including store: " + family + " with: " + storeFiles.size()
115                + " files for compaction for region: "
116                + fileSystem.getRegionInfo().getEncodedName());
117            familiesToCompact.add(family);
118            includeStore = true;
119            break;
120          }
121        }
122        if (!includeStore) {
123          LOG.info("Excluding store: " + family + " for compaction for region:  " + fileSystem
124              .getRegionInfo().getEncodedName(), " already compacted");
125        }
126      }
127      return familiesToCompact;
128    }
129  }
130
131  @VisibleForTesting
132  Connection getConnection(Configuration configuration) throws IOException {
133    return ConnectionFactory.createConnection(configuration);
134  }
135
136  private boolean familyHasReferenceFile(HRegionFileSystem fileSystem, String family)
137      throws IOException {
138    List<Path> referenceFiles =
139        getReferenceFilePaths(fileSystem.getFileSystem(), fileSystem.getStoreDir(family));
140    for (Path referenceFile : referenceFiles) {
141      FileStatus status = fileSystem.getFileSystem().getFileLinkStatus(referenceFile);
142      if (status.getModificationTime() < timestamp) {
143        LOG.info("Including store: " + family + " for compaction for region:  " + fileSystem
144            .getRegionInfo().getEncodedName() + " (reference store files)");
145        return true;
146      }
147    }
148    return false;
149
150  }
151
152  @VisibleForTesting
153  List<Path> getReferenceFilePaths(FileSystem fileSystem, Path familyDir)
154      throws IOException {
155    return FSUtils.getReferenceFilePaths(fileSystem, familyDir);
156  }
157
158  @VisibleForTesting
159  HRegionFileSystem getFileSystem(Connection connection) throws IOException {
160    Admin admin = connection.getAdmin();
161    return HRegionFileSystem.openRegionFromFileSystem(admin.getConfiguration(),
162        FSUtils.getCurrentFileSystem(admin.getConfiguration()),
163        FSUtils.getTableDir(FSUtils.getRootDir(admin.getConfiguration()), region.getTable()),
164        region, true);
165  }
166
167  @Override
168  public String toString() {
169    return "region: " + region.getEncodedName() + " store(s): " + stores;
170  }
171}