001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.hbase.util.compaction;
020
021import java.io.IOException;
022import java.util.Arrays;
023import java.util.Optional;
024import java.util.Set;
025import java.util.concurrent.Executors;
026
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.HBaseConfiguration;
029import org.apache.hadoop.hbase.HBaseInterfaceAudience;
030import org.apache.hadoop.hbase.HConstants;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
033import org.apache.hadoop.hbase.client.Connection;
034import org.apache.hadoop.hbase.client.ConnectionFactory;
035import org.apache.hadoop.hbase.client.RegionInfo;
036import org.apache.hadoop.hbase.client.TableDescriptor;
037import org.apache.hadoop.util.ToolRunner;
038import org.apache.yetus.audience.InterfaceAudience;
039import org.slf4j.Logger;
040import org.slf4j.LoggerFactory;
041import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
042import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
043import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
044import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLineParser;
045import org.apache.hbase.thirdparty.org.apache.commons.cli.DefaultParser;
046import org.apache.hbase.thirdparty.org.apache.commons.cli.Option;
047import org.apache.hbase.thirdparty.org.apache.commons.cli.Options;
048import org.apache.hbase.thirdparty.org.apache.commons.cli.ParseException;
049
050/**
051 * This tool compacts a table's regions that are beyond it's TTL. It helps to save disk space and
052 * regions become empty as a result of compaction.
053 */
054@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
055public class MajorCompactorTTL extends MajorCompactor {
056
057  private static final Logger LOG = LoggerFactory.getLogger(MajorCompactorTTL .class);
058
059  private TableDescriptor htd;
060
061  @VisibleForTesting
062  public MajorCompactorTTL(Configuration conf, TableDescriptor htd, int concurrency,
063      long sleepForMs) throws IOException {
064    this.connection = ConnectionFactory.createConnection(conf);
065    this.htd = htd;
066    this.tableName = htd.getTableName();
067    this.storesToCompact = Sets.newHashSet(); // Empty set so all stores will be compacted
068    this.sleepForMs = sleepForMs;
069    this.executor = Executors.newFixedThreadPool(concurrency);
070    this.clusterCompactionQueues = new ClusterCompactionQueues(concurrency);
071  }
072
073  protected MajorCompactorTTL() {
074    super();
075  }
076
077  @Override
078  protected Optional<MajorCompactionRequest> getMajorCompactionRequest(RegionInfo hri)
079      throws IOException {
080    return MajorCompactionTTLRequest.newRequest(connection.getConfiguration(), hri, htd);
081  }
082
083  @Override
084  protected Set<String> getStoresRequiringCompaction(MajorCompactionRequest request)
085      throws IOException {
086    return ((MajorCompactionTTLRequest)request).getStoresRequiringCompaction(htd).keySet();
087  }
088
089  public int compactRegionsTTLOnTable(Configuration conf, String table, int concurrency,
090      long sleep, int numServers, int numRegions, boolean dryRun, boolean skipWait)
091      throws Exception {
092
093    Connection conn = ConnectionFactory.createConnection(conf);
094    TableName tableName = TableName.valueOf(table);
095
096    TableDescriptor htd = conn.getAdmin().getDescriptor(tableName);
097    if (!doesAnyColFamilyHaveTTL(htd)) {
098      LOG.info("No TTL present for CF of table: " + tableName + ", skipping compaction");
099      return 0;
100    }
101
102    LOG.info("Major compacting table " + tableName + " based on TTL");
103    MajorCompactor compactor = new MajorCompactorTTL(conf, htd, concurrency, sleep);
104    compactor.setNumServers(numServers);
105    compactor.setNumRegions(numRegions);
106    compactor.setSkipWait(skipWait);
107
108    compactor.initializeWorkQueues();
109    if (!dryRun) {
110      compactor.compactAllRegions();
111    }
112    compactor.shutdown();
113    return ERRORS.size();
114  }
115
116  private boolean doesAnyColFamilyHaveTTL(TableDescriptor htd) {
117    for (ColumnFamilyDescriptor descriptor : htd.getColumnFamilies()) {
118      if (descriptor.getTimeToLive() != HConstants.FOREVER) {
119        return true;
120      }
121    }
122    return false;
123  }
124
125  private Options getOptions() {
126    Options options = getCommonOptions();
127
128    options.addOption(
129        Option.builder("table")
130            .required()
131            .desc("table name")
132            .hasArg()
133            .build()
134    );
135
136    return options;
137  }
138
139  @Override
140  public int run(String[] args) throws Exception {
141    Options options = getOptions();
142
143    final CommandLineParser cmdLineParser = new DefaultParser();
144    CommandLine commandLine;
145    try {
146      commandLine = cmdLineParser.parse(options, args);
147    } catch (ParseException parseException) {
148      System.out.println(
149          "ERROR: Unable to parse command-line arguments " + Arrays.toString(args) + " due to: "
150              + parseException);
151      printUsage(options);
152      return -1;
153    }
154    if (commandLine == null) {
155      System.out.println("ERROR: Failed parse, empty commandLine; " + Arrays.toString(args));
156      printUsage(options);
157      return -1;
158    }
159
160    String table = commandLine.getOptionValue("table");
161    int numServers = Integer.parseInt(commandLine.getOptionValue("numservers", "-1"));
162    int numRegions = Integer.parseInt(commandLine.getOptionValue("numregions", "-1"));
163    int concurrency = Integer.parseInt(commandLine.getOptionValue("servers", "1"));
164    long sleep = Long.parseLong(commandLine.getOptionValue("sleep", Long.toString(30000)));
165    boolean dryRun = commandLine.hasOption("dryRun");
166    boolean skipWait = commandLine.hasOption("skipWait");
167
168    return compactRegionsTTLOnTable(HBaseConfiguration.create(), table, concurrency, sleep,
169        numServers, numRegions, dryRun, skipWait);
170  }
171
172  public static void main(String[] args) throws Exception {
173    ToolRunner.run(HBaseConfiguration.create(), new MajorCompactorTTL(), args);
174  }
175}