001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util.compaction;
019
020import java.io.IOException;
021import java.util.Arrays;
022import java.util.Optional;
023import java.util.Set;
024import java.util.concurrent.Executors;
025import org.apache.hadoop.conf.Configuration;
026import org.apache.hadoop.hbase.HBaseConfiguration;
027import org.apache.hadoop.hbase.HBaseInterfaceAudience;
028import org.apache.hadoop.hbase.HConstants;
029import org.apache.hadoop.hbase.TableName;
030import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
031import org.apache.hadoop.hbase.client.Connection;
032import org.apache.hadoop.hbase.client.ConnectionFactory;
033import org.apache.hadoop.hbase.client.RegionInfo;
034import org.apache.hadoop.hbase.client.TableDescriptor;
035import org.apache.hadoop.util.ToolRunner;
036import org.apache.yetus.audience.InterfaceAudience;
037import org.slf4j.Logger;
038import org.slf4j.LoggerFactory;
039
040import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
041import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
042import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLineParser;
043import org.apache.hbase.thirdparty.org.apache.commons.cli.DefaultParser;
044import org.apache.hbase.thirdparty.org.apache.commons.cli.Option;
045import org.apache.hbase.thirdparty.org.apache.commons.cli.Options;
046import org.apache.hbase.thirdparty.org.apache.commons.cli.ParseException;
047
048/**
049 * This tool compacts a table's regions that are beyond it's TTL. It helps to save disk space and
050 * regions become empty as a result of compaction.
051 */
052@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
053public class MajorCompactorTTL extends MajorCompactor {
054
055  private static final Logger LOG = LoggerFactory.getLogger(MajorCompactorTTL.class);
056
057  private TableDescriptor htd;
058
059  @InterfaceAudience.Private
060  public MajorCompactorTTL(Configuration conf, TableDescriptor htd, int concurrency,
061    long sleepForMs) throws IOException {
062    this.connection = ConnectionFactory.createConnection(conf);
063    this.htd = htd;
064    this.tableName = htd.getTableName();
065    this.storesToCompact = Sets.newHashSet(); // Empty set so all stores will be compacted
066    this.sleepForMs = sleepForMs;
067    this.executor = Executors.newFixedThreadPool(concurrency);
068    this.clusterCompactionQueues = new ClusterCompactionQueues(concurrency);
069  }
070
071  protected MajorCompactorTTL() {
072    super();
073  }
074
075  @Override
076  protected Optional<MajorCompactionRequest> getMajorCompactionRequest(RegionInfo hri)
077    throws IOException {
078    return MajorCompactionTTLRequest.newRequest(connection, hri, htd);
079  }
080
081  @Override
082  protected Set<String> getStoresRequiringCompaction(MajorCompactionRequest request)
083    throws IOException {
084    return ((MajorCompactionTTLRequest) request).getStoresRequiringCompaction(htd).keySet();
085  }
086
087  public int compactRegionsTTLOnTable(Configuration conf, String table, int concurrency, long sleep,
088    int numServers, int numRegions, boolean dryRun, boolean skipWait) throws Exception {
089
090    Connection conn = ConnectionFactory.createConnection(conf);
091    TableName tableName = TableName.valueOf(table);
092
093    TableDescriptor htd = conn.getAdmin().getDescriptor(tableName);
094    if (!doesAnyColFamilyHaveTTL(htd)) {
095      LOG.info("No TTL present for CF of table: " + tableName + ", skipping compaction");
096      return 0;
097    }
098
099    LOG.info("Major compacting table " + tableName + " based on TTL");
100    MajorCompactor compactor = new MajorCompactorTTL(conf, htd, concurrency, sleep);
101    compactor.setNumServers(numServers);
102    compactor.setNumRegions(numRegions);
103    compactor.setSkipWait(skipWait);
104
105    compactor.initializeWorkQueues();
106    if (!dryRun) {
107      compactor.compactAllRegions();
108    }
109    compactor.shutdown();
110    return ERRORS.size();
111  }
112
113  private boolean doesAnyColFamilyHaveTTL(TableDescriptor htd) {
114    for (ColumnFamilyDescriptor descriptor : htd.getColumnFamilies()) {
115      if (descriptor.getTimeToLive() != HConstants.FOREVER) {
116        return true;
117      }
118    }
119    return false;
120  }
121
122  private Options getOptions() {
123    Options options = getCommonOptions();
124
125    options.addOption(Option.builder("table").required().desc("table name").hasArg().build());
126
127    return options;
128  }
129
130  @Override
131  public int run(String[] args) throws Exception {
132    Options options = getOptions();
133
134    final CommandLineParser cmdLineParser = new DefaultParser();
135    CommandLine commandLine;
136    try {
137      commandLine = cmdLineParser.parse(options, args);
138    } catch (ParseException parseException) {
139      System.out.println("ERROR: Unable to parse command-line arguments " + Arrays.toString(args)
140        + " due to: " + parseException);
141      printUsage(options);
142      return -1;
143    }
144    if (commandLine == null) {
145      System.out.println("ERROR: Failed parse, empty commandLine; " + Arrays.toString(args));
146      printUsage(options);
147      return -1;
148    }
149
150    String table = commandLine.getOptionValue("table");
151    int numServers = Integer.parseInt(commandLine.getOptionValue("numservers", "-1"));
152    int numRegions = Integer.parseInt(commandLine.getOptionValue("numregions", "-1"));
153    int concurrency = Integer.parseInt(commandLine.getOptionValue("servers", "1"));
154    long sleep = Long.parseLong(commandLine.getOptionValue("sleep", Long.toString(30000)));
155    boolean dryRun = commandLine.hasOption("dryRun");
156    boolean skipWait = commandLine.hasOption("skipWait");
157
158    return compactRegionsTTLOnTable(HBaseConfiguration.create(), table, concurrency, sleep,
159      numServers, numRegions, dryRun, skipWait);
160  }
161
162  public static void main(String[] args) throws Exception {
163    ToolRunner.run(HBaseConfiguration.create(), new MajorCompactorTTL(), args);
164  }
165}