001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.mapreduce; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertNotNull; 022import static org.junit.Assert.assertNull; 023import static org.junit.Assert.assertTrue; 024import static org.junit.Assert.fail; 025 026import java.io.ByteArrayOutputStream; 027import java.io.PrintStream; 028import org.apache.hadoop.conf.Configuration; 029import org.apache.hadoop.hbase.CellUtil; 030import org.apache.hadoop.hbase.HBaseClassTestRule; 031import org.apache.hadoop.hbase.HBaseTestingUtility; 032import org.apache.hadoop.hbase.TableName; 033import org.apache.hadoop.hbase.client.Get; 034import org.apache.hadoop.hbase.client.Put; 035import org.apache.hadoop.hbase.client.Result; 036import org.apache.hadoop.hbase.client.Table; 037import org.apache.hadoop.hbase.testclassification.LargeTests; 038import org.apache.hadoop.hbase.testclassification.MapReduceTests; 039import org.apache.hadoop.hbase.util.Bytes; 040import org.apache.hadoop.hbase.util.LauncherSecurityManager; 041import org.apache.hadoop.util.ToolRunner; 042import org.junit.AfterClass; 043import org.junit.BeforeClass; 044import org.junit.ClassRule; 045import org.junit.Rule; 046import org.junit.Test; 047import org.junit.experimental.categories.Category; 048import org.junit.rules.TestName; 049 050/** 051 * Basic test for the CopyTable M/R tool 052 */ 053@Category({MapReduceTests.class, LargeTests.class}) 054public class TestCopyTable { 055 056 @ClassRule 057 public static final HBaseClassTestRule CLASS_RULE = 058 HBaseClassTestRule.forClass(TestCopyTable.class); 059 060 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 061 private static final byte[] ROW1 = Bytes.toBytes("row1"); 062 private static final byte[] ROW2 = Bytes.toBytes("row2"); 063 private static final String FAMILY_A_STRING = "a"; 064 private static final String FAMILY_B_STRING = "b"; 065 private static final byte[] FAMILY_A = Bytes.toBytes(FAMILY_A_STRING); 066 private static final byte[] FAMILY_B = Bytes.toBytes(FAMILY_B_STRING); 067 private static final byte[] QUALIFIER = Bytes.toBytes("q"); 068 069 @Rule 070 public TestName name = new TestName(); 071 072 @BeforeClass 073 public static void beforeClass() throws Exception { 074 TEST_UTIL.startMiniCluster(3); 075 } 076 077 @AfterClass 078 public static void afterClass() throws Exception { 079 TEST_UTIL.shutdownMiniCluster(); 080 } 081 082 private void doCopyTableTest(boolean bulkload) throws Exception { 083 final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1"); 084 final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2"); 085 final byte[] FAMILY = Bytes.toBytes("family"); 086 final byte[] COLUMN1 = Bytes.toBytes("c1"); 087 088 try (Table t1 = TEST_UTIL.createTable(tableName1, FAMILY); 089 Table t2 = TEST_UTIL.createTable(tableName2, FAMILY);) { 090 // put rows into the first table 091 for (int i = 0; i < 10; i++) { 092 Put p = new Put(Bytes.toBytes("row" + i)); 093 p.addColumn(FAMILY, COLUMN1, COLUMN1); 094 t1.put(p); 095 } 096 097 CopyTable copy = new CopyTable(); 098 099 int code; 100 if (bulkload) { 101 code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), 102 copy, new String[] { "--new.name=" + tableName2.getNameAsString(), 103 "--bulkload", tableName1.getNameAsString() }); 104 } else { 105 code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), 106 copy, new String[] { "--new.name=" + tableName2.getNameAsString(), 107 tableName1.getNameAsString() }); 108 } 109 assertEquals("copy job failed", 0, code); 110 111 // verify the data was copied into table 2 112 for (int i = 0; i < 10; i++) { 113 Get g = new Get(Bytes.toBytes("row" + i)); 114 Result r = t2.get(g); 115 assertEquals(1, r.size()); 116 assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1)); 117 } 118 } finally { 119 TEST_UTIL.deleteTable(tableName1); 120 TEST_UTIL.deleteTable(tableName2); 121 } 122 } 123 124 /** 125 * Simple end-to-end test 126 * @throws Exception 127 */ 128 @Test 129 public void testCopyTable() throws Exception { 130 doCopyTableTest(false); 131 } 132 133 /** 134 * Simple end-to-end test with bulkload. 135 */ 136 @Test 137 public void testCopyTableWithBulkload() throws Exception { 138 doCopyTableTest(true); 139 } 140 141 @Test 142 public void testStartStopRow() throws Exception { 143 final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1"); 144 final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2"); 145 final byte[] FAMILY = Bytes.toBytes("family"); 146 final byte[] COLUMN1 = Bytes.toBytes("c1"); 147 final byte[] ROW0 = Bytes.toBytesBinary("\\x01row0"); 148 final byte[] ROW1 = Bytes.toBytesBinary("\\x01row1"); 149 final byte[] ROW2 = Bytes.toBytesBinary("\\x01row2"); 150 151 Table t1 = TEST_UTIL.createTable(tableName1, FAMILY); 152 Table t2 = TEST_UTIL.createTable(tableName2, FAMILY); 153 154 // put rows into the first table 155 Put p = new Put(ROW0); 156 p.addColumn(FAMILY, COLUMN1, COLUMN1); 157 t1.put(p); 158 p = new Put(ROW1); 159 p.addColumn(FAMILY, COLUMN1, COLUMN1); 160 t1.put(p); 161 p = new Put(ROW2); 162 p.addColumn(FAMILY, COLUMN1, COLUMN1); 163 t1.put(p); 164 165 CopyTable copy = new CopyTable(); 166 assertEquals( 167 0, 168 ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), 169 copy, new String[] { "--new.name=" + tableName2, "--startrow=\\x01row1", 170 "--stoprow=\\x01row2", tableName1.getNameAsString() })); 171 172 // verify the data was copied into table 2 173 // row1 exist, row0, row2 do not exist 174 Get g = new Get(ROW1); 175 Result r = t2.get(g); 176 assertEquals(1, r.size()); 177 assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1)); 178 179 g = new Get(ROW0); 180 r = t2.get(g); 181 assertEquals(0, r.size()); 182 183 g = new Get(ROW2); 184 r = t2.get(g); 185 assertEquals(0, r.size()); 186 187 t1.close(); 188 t2.close(); 189 TEST_UTIL.deleteTable(tableName1); 190 TEST_UTIL.deleteTable(tableName2); 191 } 192 193 /** 194 * Test copy of table from sourceTable to targetTable all rows from family a 195 */ 196 @Test 197 public void testRenameFamily() throws Exception { 198 final TableName sourceTable = TableName.valueOf(name.getMethodName() + "source"); 199 final TableName targetTable = TableName.valueOf(name.getMethodName() + "-target"); 200 201 byte[][] families = { FAMILY_A, FAMILY_B }; 202 203 Table t = TEST_UTIL.createTable(sourceTable, families); 204 Table t2 = TEST_UTIL.createTable(targetTable, families); 205 Put p = new Put(ROW1); 206 p.addColumn(FAMILY_A, QUALIFIER, Bytes.toBytes("Data11")); 207 p.addColumn(FAMILY_B, QUALIFIER, Bytes.toBytes("Data12")); 208 p.addColumn(FAMILY_A, QUALIFIER, Bytes.toBytes("Data13")); 209 t.put(p); 210 p = new Put(ROW2); 211 p.addColumn(FAMILY_B, QUALIFIER, Bytes.toBytes("Dat21")); 212 p.addColumn(FAMILY_A, QUALIFIER, Bytes.toBytes("Data22")); 213 p.addColumn(FAMILY_B, QUALIFIER, Bytes.toBytes("Data23")); 214 t.put(p); 215 216 long currentTime = System.currentTimeMillis(); 217 String[] args = new String[] { "--new.name=" + targetTable, "--families=a:b", "--all.cells", 218 "--starttime=" + (currentTime - 100000), "--endtime=" + (currentTime + 100000), 219 "--versions=1", sourceTable.getNameAsString() }; 220 assertNull(t2.get(new Get(ROW1)).getRow()); 221 222 assertTrue(runCopy(args)); 223 224 assertNotNull(t2.get(new Get(ROW1)).getRow()); 225 Result res = t2.get(new Get(ROW1)); 226 byte[] b1 = res.getValue(FAMILY_B, QUALIFIER); 227 assertEquals("Data13", new String(b1)); 228 assertNotNull(t2.get(new Get(ROW2)).getRow()); 229 res = t2.get(new Get(ROW2)); 230 b1 = res.getValue(FAMILY_A, QUALIFIER); 231 // Data from the family of B is not copied 232 assertNull(b1); 233 234 } 235 236 /** 237 * Test main method of CopyTable. 238 */ 239 @Test 240 public void testMainMethod() throws Exception { 241 String[] emptyArgs = { "-h" }; 242 PrintStream oldWriter = System.err; 243 ByteArrayOutputStream data = new ByteArrayOutputStream(); 244 PrintStream writer = new PrintStream(data); 245 System.setErr(writer); 246 SecurityManager SECURITY_MANAGER = System.getSecurityManager(); 247 LauncherSecurityManager newSecurityManager= new LauncherSecurityManager(); 248 System.setSecurityManager(newSecurityManager); 249 try { 250 CopyTable.main(emptyArgs); 251 fail("should be exit"); 252 } catch (SecurityException e) { 253 assertEquals(1, newSecurityManager.getExitCode()); 254 } finally { 255 System.setErr(oldWriter); 256 System.setSecurityManager(SECURITY_MANAGER); 257 } 258 assertTrue(data.toString().contains("rs.class")); 259 // should print usage information 260 assertTrue(data.toString().contains("Usage:")); 261 } 262 263 private boolean runCopy(String[] args) throws Exception { 264 int status = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), new CopyTable(), 265 args); 266 return status == 0; 267 } 268}