001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.mapreduce; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertNotNull; 022import static org.junit.Assert.assertNull; 023import static org.junit.Assert.assertTrue; 024import static org.junit.Assert.fail; 025 026import java.io.ByteArrayOutputStream; 027import java.io.IOException; 028import java.io.PrintStream; 029import org.apache.hadoop.conf.Configuration; 030import org.apache.hadoop.hbase.Cell; 031import org.apache.hadoop.hbase.CellUtil; 032import org.apache.hadoop.hbase.HBaseClassTestRule; 033import org.apache.hadoop.hbase.HBaseTestingUtil; 034import org.apache.hadoop.hbase.TableName; 035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 036import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 037import org.apache.hadoop.hbase.client.Get; 038import org.apache.hadoop.hbase.client.Put; 039import org.apache.hadoop.hbase.client.Result; 040import org.apache.hadoop.hbase.client.Table; 041import org.apache.hadoop.hbase.client.TableDescriptor; 042import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 043import org.apache.hadoop.hbase.mob.MobTestUtil; 044import org.apache.hadoop.hbase.testclassification.LargeTests; 045import org.apache.hadoop.hbase.testclassification.MapReduceTests; 046import org.apache.hadoop.hbase.util.Bytes; 047import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 048import org.apache.hadoop.hbase.util.LauncherSecurityManager; 049import org.apache.hadoop.util.ToolRunner; 050import org.junit.AfterClass; 051import org.junit.Assert; 052import org.junit.BeforeClass; 053import org.junit.ClassRule; 054import org.junit.Rule; 055import org.junit.Test; 056import org.junit.experimental.categories.Category; 057import org.junit.rules.TestName; 058 059/** 060 * Basic test for the CopyTable M/R tool 061 */ 062@Category({ MapReduceTests.class, LargeTests.class }) 063public class TestCopyTable { 064 065 @ClassRule 066 public static final HBaseClassTestRule CLASS_RULE = 067 HBaseClassTestRule.forClass(TestCopyTable.class); 068 069 private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); 070 private static final byte[] ROW1 = Bytes.toBytes("row1"); 071 private static final byte[] ROW2 = Bytes.toBytes("row2"); 072 private static final String FAMILY_A_STRING = "a"; 073 private static final String FAMILY_B_STRING = "b"; 074 private static final byte[] FAMILY_A = Bytes.toBytes(FAMILY_A_STRING); 075 private static final byte[] FAMILY_B = Bytes.toBytes(FAMILY_B_STRING); 076 private static final byte[] QUALIFIER = Bytes.toBytes("q"); 077 078 @Rule 079 public TestName name = new TestName(); 080 081 @BeforeClass 082 public static void beforeClass() throws Exception { 083 TEST_UTIL.startMiniCluster(3); 084 } 085 086 @AfterClass 087 public static void afterClass() throws Exception { 088 TEST_UTIL.shutdownMiniCluster(); 089 } 090 091 private void doCopyTableTest(boolean bulkload) throws Exception { 092 final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1"); 093 final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2"); 094 final byte[] FAMILY = Bytes.toBytes("family"); 095 final byte[] COLUMN1 = Bytes.toBytes("c1"); 096 097 try (Table t1 = TEST_UTIL.createTable(tableName1, FAMILY); 098 Table t2 = TEST_UTIL.createTable(tableName2, FAMILY)) { 099 // put rows into the first table 100 loadData(t1, FAMILY, COLUMN1); 101 102 CopyTable copy = new CopyTable(); 103 int code; 104 if (bulkload) { 105 code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), copy, 106 new String[] { "--new.name=" + tableName2.getNameAsString(), "--bulkload", 107 tableName1.getNameAsString() }); 108 } else { 109 code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), copy, new String[] { 110 "--new.name=" + tableName2.getNameAsString(), tableName1.getNameAsString() }); 111 } 112 assertEquals("copy job failed", 0, code); 113 114 // verify the data was copied into table 2 115 verifyRows(t2, FAMILY, COLUMN1); 116 } finally { 117 TEST_UTIL.deleteTable(tableName1); 118 TEST_UTIL.deleteTable(tableName2); 119 } 120 } 121 122 private void doCopyTableTestWithMob(boolean bulkload) throws Exception { 123 final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1"); 124 final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2"); 125 final byte[] FAMILY = Bytes.toBytes("mob"); 126 final byte[] COLUMN1 = Bytes.toBytes("c1"); 127 128 ColumnFamilyDescriptorBuilder cfd = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY); 129 130 cfd.setMobEnabled(true); 131 cfd.setMobThreshold(5); 132 TableDescriptor desc1 = 133 TableDescriptorBuilder.newBuilder(tableName1).setColumnFamily(cfd.build()).build(); 134 TableDescriptor desc2 = 135 TableDescriptorBuilder.newBuilder(tableName2).setColumnFamily(cfd.build()).build(); 136 137 try (Table t1 = TEST_UTIL.createTable(desc1, null); 138 Table t2 = TEST_UTIL.createTable(desc2, null);) { 139 140 // put rows into the first table 141 for (int i = 0; i < 10; i++) { 142 Put p = new Put(Bytes.toBytes("row" + i)); 143 p.addColumn(FAMILY, COLUMN1, COLUMN1); 144 t1.put(p); 145 } 146 147 CopyTable copy = new CopyTable(); 148 149 int code; 150 if (bulkload) { 151 code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), copy, 152 new String[] { "--new.name=" + tableName2.getNameAsString(), "--bulkload", 153 tableName1.getNameAsString() }); 154 } else { 155 code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), copy, new String[] { 156 "--new.name=" + tableName2.getNameAsString(), tableName1.getNameAsString() }); 157 } 158 assertEquals("copy job failed", 0, code); 159 160 // verify the data was copied into table 2 161 for (int i = 0; i < 10; i++) { 162 Get g = new Get(Bytes.toBytes("row" + i)); 163 Result r = t2.get(g); 164 assertEquals(1, r.size()); 165 assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1)); 166 assertEquals("compare row values between two tables", 167 t1.getDescriptor().getValue("row" + i), t2.getDescriptor().getValue("row" + i)); 168 } 169 170 assertEquals("compare count of mob rows after table copy", MobTestUtil.countMobRows(t1), 171 MobTestUtil.countMobRows(t2)); 172 assertEquals("compare count of mob row values between two tables", 173 t1.getDescriptor().getValues().size(), t2.getDescriptor().getValues().size()); 174 assertTrue("The mob row count is 0 but should be > 0", MobTestUtil.countMobRows(t2) > 0); 175 } finally { 176 TEST_UTIL.deleteTable(tableName1); 177 TEST_UTIL.deleteTable(tableName2); 178 } 179 } 180 181 /** 182 * Simple end-to-end test 183 */ 184 @Test 185 public void testCopyTable() throws Exception { 186 doCopyTableTest(false); 187 } 188 189 /** 190 * Simple end-to-end test with bulkload. 191 */ 192 @Test 193 public void testCopyTableWithBulkload() throws Exception { 194 doCopyTableTest(true); 195 } 196 197 /** 198 * Simple end-to-end test on table with MOB 199 */ 200 @Test 201 public void testCopyTableWithMob() throws Exception { 202 doCopyTableTestWithMob(false); 203 } 204 205 /** 206 * Simple end-to-end test with bulkload on table with MOB. 207 */ 208 @Test 209 public void testCopyTableWithBulkloadWithMob() throws Exception { 210 doCopyTableTestWithMob(true); 211 } 212 213 @Test 214 public void testStartStopRow() throws Exception { 215 final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1"); 216 final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2"); 217 final byte[] FAMILY = Bytes.toBytes("family"); 218 final byte[] COLUMN1 = Bytes.toBytes("c1"); 219 final byte[] row0 = Bytes.toBytesBinary("\\x01row0"); 220 final byte[] row1 = Bytes.toBytesBinary("\\x01row1"); 221 final byte[] row2 = Bytes.toBytesBinary("\\x01row2"); 222 223 try (Table t1 = TEST_UTIL.createTable(tableName1, FAMILY); 224 Table t2 = TEST_UTIL.createTable(tableName2, FAMILY)) { 225 226 // put rows into the first table 227 Put p = new Put(row0); 228 p.addColumn(FAMILY, COLUMN1, COLUMN1); 229 t1.put(p); 230 p = new Put(row1); 231 p.addColumn(FAMILY, COLUMN1, COLUMN1); 232 t1.put(p); 233 p = new Put(row2); 234 p.addColumn(FAMILY, COLUMN1, COLUMN1); 235 t1.put(p); 236 237 CopyTable copy = new CopyTable(); 238 assertEquals(0, 239 ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), copy, 240 new String[] { "--new.name=" + tableName2, "--startrow=\\x01row1", "--stoprow=\\x01row2", 241 tableName1.getNameAsString() })); 242 243 // verify the data was copied into table 2 244 // row1 exist, row0, row2 do not exist 245 Get g = new Get(row1); 246 Result r = t2.get(g); 247 assertEquals(1, r.size()); 248 assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1)); 249 250 g = new Get(row0); 251 r = t2.get(g); 252 assertEquals(0, r.size()); 253 254 g = new Get(row2); 255 r = t2.get(g); 256 assertEquals(0, r.size()); 257 258 } finally { 259 TEST_UTIL.deleteTable(tableName1); 260 TEST_UTIL.deleteTable(tableName2); 261 } 262 } 263 264 /** 265 * Test copy of table from sourceTable to targetTable all rows from family a 266 */ 267 @Test 268 public void testRenameFamily() throws Exception { 269 final TableName sourceTable = TableName.valueOf(name.getMethodName() + "source"); 270 final TableName targetTable = TableName.valueOf(name.getMethodName() + "-target"); 271 272 byte[][] families = { FAMILY_A, FAMILY_B }; 273 274 Table t = TEST_UTIL.createTable(sourceTable, families); 275 Table t2 = TEST_UTIL.createTable(targetTable, families); 276 Put p = new Put(ROW1); 277 p.addColumn(FAMILY_A, QUALIFIER, Bytes.toBytes("Data11")); 278 p.addColumn(FAMILY_B, QUALIFIER, Bytes.toBytes("Data12")); 279 p.addColumn(FAMILY_A, QUALIFIER, Bytes.toBytes("Data13")); 280 t.put(p); 281 p = new Put(ROW2); 282 p.addColumn(FAMILY_B, QUALIFIER, Bytes.toBytes("Dat21")); 283 p.addColumn(FAMILY_A, QUALIFIER, Bytes.toBytes("Data22")); 284 p.addColumn(FAMILY_B, QUALIFIER, Bytes.toBytes("Data23")); 285 t.put(p); 286 287 long currentTime = EnvironmentEdgeManager.currentTime(); 288 String[] args = new String[] { "--new.name=" + targetTable, "--families=a:b", "--all.cells", 289 "--starttime=" + (currentTime - 100000), "--endtime=" + (currentTime + 100000), 290 "--versions=1", sourceTable.getNameAsString() }; 291 assertNull(t2.get(new Get(ROW1)).getRow()); 292 293 assertTrue(runCopy(args)); 294 295 assertNotNull(t2.get(new Get(ROW1)).getRow()); 296 Result res = t2.get(new Get(ROW1)); 297 byte[] b1 = res.getValue(FAMILY_B, QUALIFIER); 298 assertEquals("Data13", Bytes.toString(b1)); 299 assertNotNull(t2.get(new Get(ROW2)).getRow()); 300 res = t2.get(new Get(ROW2)); 301 b1 = res.getValue(FAMILY_A, QUALIFIER); 302 // Data from the family of B is not copied 303 assertNull(b1); 304 305 } 306 307 /** 308 * Test main method of CopyTable. 309 */ 310 @Test 311 public void testMainMethod() throws Exception { 312 String[] emptyArgs = { "-h" }; 313 PrintStream oldWriter = System.err; 314 ByteArrayOutputStream data = new ByteArrayOutputStream(); 315 PrintStream writer = new PrintStream(data); 316 System.setErr(writer); 317 SecurityManager SECURITY_MANAGER = System.getSecurityManager(); 318 LauncherSecurityManager newSecurityManager = new LauncherSecurityManager(); 319 System.setSecurityManager(newSecurityManager); 320 try { 321 CopyTable.main(emptyArgs); 322 fail("should be exit"); 323 } catch (SecurityException e) { 324 assertEquals(1, newSecurityManager.getExitCode()); 325 } finally { 326 System.setErr(oldWriter); 327 System.setSecurityManager(SECURITY_MANAGER); 328 } 329 assertTrue(data.toString().contains("rs.class")); 330 // should print usage information 331 assertTrue(data.toString().contains("Usage:")); 332 } 333 334 private boolean runCopy(String[] args) throws Exception { 335 int status = 336 ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), new CopyTable(), args); 337 return status == 0; 338 } 339 340 private void loadData(Table t, byte[] family, byte[] column) throws IOException { 341 for (int i = 0; i < 10; i++) { 342 byte[] row = Bytes.toBytes("row" + i); 343 Put p = new Put(row); 344 p.addColumn(family, column, row); 345 t.put(p); 346 } 347 } 348 349 private void verifyRows(Table t, byte[] family, byte[] column) throws IOException { 350 for (int i = 0; i < 10; i++) { 351 byte[] row = Bytes.toBytes("row" + i); 352 Get g = new Get(row).addFamily(family); 353 Result r = t.get(g); 354 Assert.assertNotNull(r); 355 Assert.assertEquals(1, r.size()); 356 Cell cell = r.rawCells()[0]; 357 Assert.assertTrue(CellUtil.matchingQualifier(cell, column)); 358 Assert.assertEquals(Bytes.compareTo(cell.getValueArray(), cell.getValueOffset(), 359 cell.getValueLength(), row, 0, row.length), 0); 360 } 361 } 362 363 private Table createTable(TableName tableName, byte[] family, boolean isMob) throws IOException { 364 if (isMob) { 365 ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(family) 366 .setMobEnabled(true).setMobThreshold(1).build(); 367 TableDescriptor desc = 368 TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(cfd).build(); 369 return TEST_UTIL.createTable(desc, null); 370 } else { 371 return TEST_UTIL.createTable(tableName, family); 372 } 373 } 374 375 private void testCopyTableBySnapshot(String tablePrefix, boolean bulkLoad, boolean isMob) 376 throws Exception { 377 TableName table1 = TableName.valueOf(tablePrefix + 1); 378 TableName table2 = TableName.valueOf(tablePrefix + 2); 379 Table t1 = createTable(table1, FAMILY_A, isMob); 380 Table t2 = createTable(table2, FAMILY_A, isMob); 381 loadData(t1, FAMILY_A, Bytes.toBytes("qualifier")); 382 String snapshot = tablePrefix + "_snapshot"; 383 TEST_UTIL.getAdmin().snapshot(snapshot, table1); 384 boolean success; 385 if (bulkLoad) { 386 success = 387 runCopy(new String[] { "--snapshot", "--new.name=" + table2, "--bulkload", snapshot }); 388 } else { 389 success = runCopy(new String[] { "--snapshot", "--new.name=" + table2, snapshot }); 390 } 391 Assert.assertTrue(success); 392 verifyRows(t2, FAMILY_A, Bytes.toBytes("qualifier")); 393 } 394 395 @Test 396 public void testLoadingSnapshotToTable() throws Exception { 397 testCopyTableBySnapshot("testLoadingSnapshotToTable", false, false); 398 } 399 400 @Test 401 public void tsetLoadingSnapshotToMobTable() throws Exception { 402 testCopyTableBySnapshot("testLoadingSnapshotToMobTable", false, true); 403 } 404 405 @Test 406 public void testLoadingSnapshotAndBulkLoadToTable() throws Exception { 407 testCopyTableBySnapshot("testLoadingSnapshotAndBulkLoadToTable", true, false); 408 } 409 410 @Test 411 public void testLoadingSnapshotAndBulkLoadToMobTable() throws Exception { 412 testCopyTableBySnapshot("testLoadingSnapshotAndBulkLoadToMobTable", true, true); 413 } 414 415 @Test 416 public void testLoadingSnapshotToRemoteCluster() throws Exception { 417 Assert.assertFalse(runCopy( 418 new String[] { "--snapshot", "--peerAdr=hbase://remoteHBase", "sourceSnapshotName" })); 419 } 420 421 @Test 422 public void testLoadingSnapshotWithoutSnapshotName() throws Exception { 423 Assert.assertFalse(runCopy(new String[] { "--snapshot", "--peerAdr=hbase://remoteHBase" })); 424 } 425 426 @Test 427 public void testLoadingSnapshotWithoutDestTable() throws Exception { 428 Assert.assertFalse(runCopy(new String[] { "--snapshot", "sourceSnapshotName" })); 429 } 430}