001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.util.compaction; 019 020import java.io.IOException; 021import java.util.Arrays; 022import java.util.Optional; 023import java.util.Set; 024import java.util.concurrent.Executors; 025import org.apache.hadoop.conf.Configuration; 026import org.apache.hadoop.hbase.HBaseConfiguration; 027import org.apache.hadoop.hbase.HBaseInterfaceAudience; 028import org.apache.hadoop.hbase.HConstants; 029import org.apache.hadoop.hbase.TableName; 030import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 031import org.apache.hadoop.hbase.client.Connection; 032import org.apache.hadoop.hbase.client.ConnectionFactory; 033import org.apache.hadoop.hbase.client.RegionInfo; 034import org.apache.hadoop.hbase.client.TableDescriptor; 035import org.apache.hadoop.util.ToolRunner; 036import org.apache.yetus.audience.InterfaceAudience; 037import org.slf4j.Logger; 038import org.slf4j.LoggerFactory; 039 040import org.apache.hbase.thirdparty.com.google.common.collect.Sets; 041import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine; 042import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLineParser; 043import org.apache.hbase.thirdparty.org.apache.commons.cli.DefaultParser; 044import org.apache.hbase.thirdparty.org.apache.commons.cli.Option; 045import org.apache.hbase.thirdparty.org.apache.commons.cli.Options; 046import org.apache.hbase.thirdparty.org.apache.commons.cli.ParseException; 047 048/** 049 * This tool compacts a table's regions that are beyond it's TTL. It helps to save disk space and 050 * regions become empty as a result of compaction. 051 */ 052@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS) 053public class MajorCompactorTTL extends MajorCompactor { 054 055 private static final Logger LOG = LoggerFactory.getLogger(MajorCompactorTTL.class); 056 057 private TableDescriptor htd; 058 059 @InterfaceAudience.Private 060 public MajorCompactorTTL(Configuration conf, TableDescriptor htd, int concurrency, 061 long sleepForMs) throws IOException { 062 this.connection = ConnectionFactory.createConnection(conf); 063 this.htd = htd; 064 this.tableName = htd.getTableName(); 065 this.storesToCompact = Sets.newHashSet(); // Empty set so all stores will be compacted 066 this.sleepForMs = sleepForMs; 067 this.executor = Executors.newFixedThreadPool(concurrency); 068 this.clusterCompactionQueues = new ClusterCompactionQueues(concurrency); 069 } 070 071 protected MajorCompactorTTL() { 072 super(); 073 } 074 075 @Override 076 protected Optional<MajorCompactionRequest> getMajorCompactionRequest(RegionInfo hri) 077 throws IOException { 078 return MajorCompactionTTLRequest.newRequest(connection, hri, htd); 079 } 080 081 @Override 082 protected Set<String> getStoresRequiringCompaction(MajorCompactionRequest request) 083 throws IOException { 084 return ((MajorCompactionTTLRequest) request).getStoresRequiringCompaction(htd).keySet(); 085 } 086 087 public int compactRegionsTTLOnTable(Configuration conf, String table, int concurrency, long sleep, 088 int numServers, int numRegions, boolean dryRun, boolean skipWait) throws Exception { 089 090 Connection conn = ConnectionFactory.createConnection(conf); 091 TableName tableName = TableName.valueOf(table); 092 093 TableDescriptor htd = conn.getAdmin().getDescriptor(tableName); 094 if (!doesAnyColFamilyHaveTTL(htd)) { 095 LOG.info("No TTL present for CF of table: " + tableName + ", skipping compaction"); 096 return 0; 097 } 098 099 LOG.info("Major compacting table " + tableName + " based on TTL"); 100 MajorCompactor compactor = new MajorCompactorTTL(conf, htd, concurrency, sleep); 101 compactor.setNumServers(numServers); 102 compactor.setNumRegions(numRegions); 103 compactor.setSkipWait(skipWait); 104 105 compactor.initializeWorkQueues(); 106 if (!dryRun) { 107 compactor.compactAllRegions(); 108 } 109 compactor.shutdown(); 110 return ERRORS.size(); 111 } 112 113 private boolean doesAnyColFamilyHaveTTL(TableDescriptor htd) { 114 for (ColumnFamilyDescriptor descriptor : htd.getColumnFamilies()) { 115 if (descriptor.getTimeToLive() != HConstants.FOREVER) { 116 return true; 117 } 118 } 119 return false; 120 } 121 122 private Options getOptions() { 123 Options options = getCommonOptions(); 124 125 options.addOption(Option.builder("table").required().desc("table name").hasArg().build()); 126 127 return options; 128 } 129 130 @Override 131 public int run(String[] args) throws Exception { 132 Options options = getOptions(); 133 134 final CommandLineParser cmdLineParser = new DefaultParser(); 135 CommandLine commandLine; 136 try { 137 commandLine = cmdLineParser.parse(options, args); 138 } catch (ParseException parseException) { 139 System.out.println("ERROR: Unable to parse command-line arguments " + Arrays.toString(args) 140 + " due to: " + parseException); 141 printUsage(options); 142 return -1; 143 } 144 if (commandLine == null) { 145 System.out.println("ERROR: Failed parse, empty commandLine; " + Arrays.toString(args)); 146 printUsage(options); 147 return -1; 148 } 149 150 String table = commandLine.getOptionValue("table"); 151 int numServers = Integer.parseInt(commandLine.getOptionValue("numservers", "-1")); 152 int numRegions = Integer.parseInt(commandLine.getOptionValue("numregions", "-1")); 153 int concurrency = Integer.parseInt(commandLine.getOptionValue("servers", "1")); 154 long sleep = Long.parseLong(commandLine.getOptionValue("sleep", Long.toString(30000))); 155 boolean dryRun = commandLine.hasOption("dryRun"); 156 boolean skipWait = commandLine.hasOption("skipWait"); 157 158 return compactRegionsTTLOnTable(HBaseConfiguration.create(), table, concurrency, sleep, 159 numServers, numRegions, dryRun, skipWait); 160 } 161 162 public static void main(String[] args) throws Exception { 163 ToolRunner.run(HBaseConfiguration.create(), new MajorCompactorTTL(), args); 164 } 165}