001/** 002 * 003 * Licensed to the Apache Software Foundation (ASF) under one 004 * or more contributor license agreements. See the NOTICE file 005 * distributed with this work for additional information 006 * regarding copyright ownership. The ASF licenses this file 007 * to you under the Apache License, Version 2.0 (the 008 * "License"); you may not use this file except in compliance 009 * with the License. You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, software 014 * distributed under the License is distributed on an "AS IS" BASIS, 015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 016 * See the License for the specific language governing permissions and 017 * limitations under the License. 018 */ 019package org.apache.hadoop.hbase.mapreduce; 020 021import java.io.IOException; 022import java.util.Arrays; 023import java.util.List; 024 025import org.apache.hadoop.conf.Configuration; 026import org.apache.hadoop.fs.Path; 027import org.apache.hadoop.hbase.CompareOperator; 028import org.apache.hadoop.hbase.HConstants; 029import org.apache.hadoop.hbase.TableName; 030import org.apache.hadoop.hbase.client.Scan; 031import org.apache.hadoop.hbase.filter.Filter; 032import org.apache.hadoop.hbase.filter.IncompatibleFilterException; 033import org.apache.hadoop.hbase.filter.PrefixFilter; 034import org.apache.hadoop.hbase.filter.RegexStringComparator; 035import org.apache.hadoop.hbase.filter.RowFilter; 036import org.apache.hadoop.hbase.security.visibility.Authorizations; 037import org.apache.hadoop.hbase.util.Bytes; 038import org.apache.hadoop.hbase.util.Triple; 039import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 040import org.apache.yetus.audience.InterfaceAudience; 041import org.slf4j.Logger; 042import org.slf4j.LoggerFactory; 043 044/** 045 * Some helper methods are used by {@link org.apache.hadoop.hbase.mapreduce.Export} 046 * and org.apache.hadoop.hbase.coprocessor.Export (in hbase-endpooint). 047 */ 048@InterfaceAudience.Private 049public final class ExportUtils { 050 private static final Logger LOG = LoggerFactory.getLogger(ExportUtils.class); 051 public static final String RAW_SCAN = "hbase.mapreduce.include.deleted.rows"; 052 public static final String EXPORT_BATCHING = "hbase.export.scanner.batch"; 053 public static final String EXPORT_CACHING = "hbase.export.scanner.caching"; 054 public static final String EXPORT_VISIBILITY_LABELS = "hbase.export.visibility.labels"; 055 /** 056 * Common usage for other export tools. 057 * @param errorMsg Error message. Can be null. 058 */ 059 public static void usage(final String errorMsg) { 060 if (errorMsg != null && errorMsg.length() > 0) { 061 System.err.println("ERROR: " + errorMsg); 062 } 063 System.err.println("Usage: Export [-D <property=value>]* <tablename> <outputdir> [<versions> " + 064 "[<starttime> [<endtime>]] [^[regex pattern] or [Prefix] to filter]]\n"); 065 System.err.println(" Note: -D properties will be applied to the conf used. "); 066 System.err.println(" For example: "); 067 System.err.println(" -D " + FileOutputFormat.COMPRESS + "=true"); 068 System.err.println(" -D " + FileOutputFormat.COMPRESS_CODEC + "=org.apache.hadoop.io.compress.GzipCodec"); 069 System.err.println(" -D " + FileOutputFormat.COMPRESS_TYPE + "=BLOCK"); 070 System.err.println(" Additionally, the following SCAN properties can be specified"); 071 System.err.println(" to control/limit what is exported.."); 072 System.err.println(" -D " + TableInputFormat.SCAN_COLUMN_FAMILY + "=<family1>,<family2>, ..."); 073 System.err.println(" -D " + RAW_SCAN + "=true"); 074 System.err.println(" -D " + TableInputFormat.SCAN_ROW_START + "=<ROWSTART>"); 075 System.err.println(" -D " + TableInputFormat.SCAN_ROW_STOP + "=<ROWSTOP>"); 076 System.err.println(" -D " + HConstants.HBASE_CLIENT_SCANNER_CACHING + "=100"); 077 System.err.println(" -D " + EXPORT_VISIBILITY_LABELS + "=<labels>"); 078 System.err.println("For tables with very wide rows consider setting the batch size as below:\n" 079 + " -D " + EXPORT_BATCHING + "=10\n" 080 + " -D " + EXPORT_CACHING + "=100"); 081 } 082 083 private static Filter getExportFilter(String[] args) { 084 Filter exportFilter; 085 String filterCriteria = (args.length > 5) ? args[5]: null; 086 if (filterCriteria == null) return null; 087 if (filterCriteria.startsWith("^")) { 088 String regexPattern = filterCriteria.substring(1, filterCriteria.length()); 089 exportFilter = new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(regexPattern)); 090 } else { 091 exportFilter = new PrefixFilter(Bytes.toBytesBinary(filterCriteria)); 092 } 093 return exportFilter; 094 } 095 096 public static boolean isValidArguements(String[] args) { 097 return args != null && args.length >= 2; 098 } 099 100 public static Triple<TableName, Scan, Path> getArgumentsFromCommandLine( 101 Configuration conf, String[] args) throws IOException { 102 if (!isValidArguements(args)) { 103 return null; 104 } 105 return new Triple<>(TableName.valueOf(args[0]), getScanFromCommandLine(conf, args), new Path(args[1])); 106 } 107 108 static Scan getScanFromCommandLine(Configuration conf, String[] args) throws IOException { 109 Scan s = new Scan(); 110 // Optional arguments. 111 // Set Scan Versions 112 int versions = args.length > 2? Integer.parseInt(args[2]): 1; 113 s.setMaxVersions(versions); 114 // Set Scan Range 115 long startTime = args.length > 3? Long.parseLong(args[3]): 0L; 116 long endTime = args.length > 4? Long.parseLong(args[4]): Long.MAX_VALUE; 117 s.setTimeRange(startTime, endTime); 118 // Set cache blocks 119 s.setCacheBlocks(false); 120 // set Start and Stop row 121 if (conf.get(TableInputFormat.SCAN_ROW_START) != null) { 122 s.setStartRow(Bytes.toBytesBinary(conf.get(TableInputFormat.SCAN_ROW_START))); 123 } 124 if (conf.get(TableInputFormat.SCAN_ROW_STOP) != null) { 125 s.setStopRow(Bytes.toBytesBinary(conf.get(TableInputFormat.SCAN_ROW_STOP))); 126 } 127 // Set Scan Column Family 128 boolean raw = Boolean.parseBoolean(conf.get(RAW_SCAN)); 129 if (raw) { 130 s.setRaw(raw); 131 } 132 for (String columnFamily : conf.getTrimmedStrings(TableInputFormat.SCAN_COLUMN_FAMILY)) { 133 s.addFamily(Bytes.toBytes(columnFamily)); 134 } 135 // Set RowFilter or Prefix Filter if applicable. 136 Filter exportFilter = getExportFilter(args); 137 if (exportFilter!= null) { 138 LOG.info("Setting Scan Filter for Export."); 139 s.setFilter(exportFilter); 140 } 141 List<String> labels = null; 142 if (conf.get(EXPORT_VISIBILITY_LABELS) != null) { 143 labels = Arrays.asList(conf.getStrings(EXPORT_VISIBILITY_LABELS)); 144 if (!labels.isEmpty()) { 145 s.setAuthorizations(new Authorizations(labels)); 146 } 147 } 148 149 int batching = conf.getInt(EXPORT_BATCHING, -1); 150 if (batching != -1) { 151 try { 152 s.setBatch(batching); 153 } catch (IncompatibleFilterException e) { 154 LOG.error("Batching could not be set", e); 155 } 156 } 157 158 int caching = conf.getInt(EXPORT_CACHING, 100); 159 if (caching != -1) { 160 try { 161 s.setCaching(caching); 162 } catch (IncompatibleFilterException e) { 163 LOG.error("Caching could not be set", e); 164 } 165 } 166 LOG.info("versions=" + versions + ", starttime=" + startTime 167 + ", endtime=" + endTime + ", keepDeletedCells=" + raw 168 + ", visibility labels=" + labels); 169 return s; 170 } 171 172 private ExportUtils() { 173 } 174}