Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.mapreduce;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertNull;
023import static org.junit.Assert.assertTrue;
024import static org.junit.Assert.fail;
025
026import java.io.ByteArrayOutputStream;
027import java.io.IOException;
028import java.io.PrintStream;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.hbase.Cell;
031import org.apache.hadoop.hbase.CellUtil;
032import org.apache.hadoop.hbase.HBaseClassTestRule;
033import org.apache.hadoop.hbase.HBaseTestingUtility;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
036import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
037import org.apache.hadoop.hbase.client.Get;
038import org.apache.hadoop.hbase.client.Put;
039import org.apache.hadoop.hbase.client.Result;
040import org.apache.hadoop.hbase.client.Table;
041import org.apache.hadoop.hbase.client.TableDescriptor;
042import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
043import org.apache.hadoop.hbase.mob.MobTestUtil;
044import org.apache.hadoop.hbase.testclassification.LargeTests;
045import org.apache.hadoop.hbase.testclassification.MapReduceTests;
046import org.apache.hadoop.hbase.util.Bytes;
047import org.apache.hadoop.hbase.util.LauncherSecurityManager;
048import org.apache.hadoop.util.ToolRunner;
049import org.junit.AfterClass;
050import org.junit.Assert;
051import org.junit.BeforeClass;
052import org.junit.ClassRule;
053import org.junit.Rule;
054import org.junit.Test;
055import org.junit.experimental.categories.Category;
056import org.junit.rules.TestName;
057
058/**
059 * Basic test for the CopyTable M/R tool
060 */
061@Category({MapReduceTests.class, LargeTests.class})
062public class TestCopyTable {
063
064  @ClassRule
065  public static final HBaseClassTestRule CLASS_RULE =
066      HBaseClassTestRule.forClass(TestCopyTable.class);
067
068  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
069  private static final byte[] ROW1 = Bytes.toBytes("row1");
070  private static final byte[] ROW2 = Bytes.toBytes("row2");
071  private static final String FAMILY_A_STRING = "a";
072  private static final String FAMILY_B_STRING = "b";
073  private static final byte[] FAMILY_A = Bytes.toBytes(FAMILY_A_STRING);
074  private static final byte[] FAMILY_B = Bytes.toBytes(FAMILY_B_STRING);
075  private static final byte[] QUALIFIER = Bytes.toBytes("q");
076
077  @Rule
078  public TestName name = new TestName();
079
080  @BeforeClass
081  public static void beforeClass() throws Exception {
082    TEST_UTIL.startMiniCluster(3);
083  }
084
085  @AfterClass
086  public static void afterClass() throws Exception {
087    TEST_UTIL.shutdownMiniCluster();
088  }
089
090  private void doCopyTableTest(boolean bulkload) throws Exception {
091    final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1");
092    final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2");
093    final byte[] FAMILY = Bytes.toBytes("family");
094    final byte[] COLUMN1 = Bytes.toBytes("c1");
095
096    try (Table t1 = TEST_UTIL.createTable(tableName1, FAMILY);
097         Table t2 = TEST_UTIL.createTable(tableName2, FAMILY);) {
098      // put rows into the first table
099      loadData(t1, FAMILY, COLUMN1);
100
101      CopyTable copy = new CopyTable();
102      int code;
103      if (bulkload) {
104        code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()),
105            copy, new String[] { "--new.name=" + tableName2.getNameAsString(),
106            "--bulkload", tableName1.getNameAsString() });
107      } else {
108        code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()),
109            copy, new String[] { "--new.name=" + tableName2.getNameAsString(),
110            tableName1.getNameAsString() });
111      }
112      assertEquals("copy job failed", 0, code);
113
114      // verify the data was copied into table 2
115      verifyRows(t2, FAMILY, COLUMN1);
116    } finally {
117      TEST_UTIL.deleteTable(tableName1);
118      TEST_UTIL.deleteTable(tableName2);
119    }
120  }
121
122  private void doCopyTableTestWithMob(boolean bulkload) throws Exception {
123    final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1");
124    final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2");
125    final byte[] FAMILY = Bytes.toBytes("mob");
126    final byte[] COLUMN1 = Bytes.toBytes("c1");
127
128    ColumnFamilyDescriptorBuilder cfd = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY);
129
130    cfd.setMobEnabled(true);
131    cfd.setMobThreshold(5);
132    TableDescriptor desc1 = TableDescriptorBuilder.newBuilder(tableName1)
133            .setColumnFamily(cfd.build())
134            .build();
135    TableDescriptor desc2 = TableDescriptorBuilder.newBuilder(tableName2)
136            .setColumnFamily(cfd.build())
137            .build();
138
139    try (Table t1 = TEST_UTIL.createTable(desc1, null);
140         Table t2 = TEST_UTIL.createTable(desc2, null);) {
141
142      // put rows into the first table
143      for (int i = 0; i < 10; i++) {
144        Put p = new Put(Bytes.toBytes("row" + i));
145        p.addColumn(FAMILY, COLUMN1, COLUMN1);
146        t1.put(p);
147      }
148
149      CopyTable copy = new CopyTable();
150
151      int code;
152      if (bulkload) {
153        code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()),
154            copy, new String[] { "--new.name=" + tableName2.getNameAsString(),
155              "--bulkload", tableName1.getNameAsString() });
156      } else {
157        code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()),
158            copy, new String[] { "--new.name=" + tableName2.getNameAsString(),
159            tableName1.getNameAsString() });
160      }
161      assertEquals("copy job failed", 0, code);
162
163      // verify the data was copied into table 2
164      for (int i = 0; i < 10; i++) {
165        Get g = new Get(Bytes.toBytes("row" + i));
166        Result r = t2.get(g);
167        assertEquals(1, r.size());
168        assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
169        assertEquals("compare row values between two tables",
170              t1.getDescriptor().getValue("row" + i),
171              t2.getDescriptor().getValue("row" + i));
172      }
173
174      assertEquals("compare count of mob rows after table copy", MobTestUtil.countMobRows(TEST_UTIL, t1),
175              MobTestUtil.countMobRows(TEST_UTIL, t2));
176      assertEquals("compare count of mob row values between two tables",
177              t1.getDescriptor().getValues().size(),
178              t2.getDescriptor().getValues().size());
179      assertTrue("The mob row count is 0 but should be > 0",
180            MobTestUtil.countMobRows(TEST_UTIL, t2) > 0);
181    } finally {
182      TEST_UTIL.deleteTable(tableName1);
183      TEST_UTIL.deleteTable(tableName2);
184    }
185  }
186
187  /**
188   * Simple end-to-end test
189   * @throws Exception
190   */
191  @Test
192  public void testCopyTable() throws Exception {
193    doCopyTableTest(false);
194  }
195
196  /**
197   * Simple end-to-end test with bulkload.
198   */
199  @Test
200  public void testCopyTableWithBulkload() throws Exception {
201    doCopyTableTest(true);
202  }
203
204  /**
205   * Simple end-to-end test on table with MOB
206   */
207  @Test
208  public void testCopyTableWithMob() throws Exception {
209    doCopyTableTestWithMob(false);
210  }
211
212  /**
213   * Simple end-to-end test with bulkload on table with MOB.
214   */
215  @Test
216  public void testCopyTableWithBulkloadWithMob() throws Exception {
217    doCopyTableTestWithMob(true);
218  }
219
220  @Test
221  public void testStartStopRow() throws Exception {
222    final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1");
223    final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2");
224    final byte[] FAMILY = Bytes.toBytes("family");
225    final byte[] COLUMN1 = Bytes.toBytes("c1");
226    final byte[] ROW0 = Bytes.toBytesBinary("\\x01row0");
227    final byte[] ROW1 = Bytes.toBytesBinary("\\x01row1");
228    final byte[] ROW2 = Bytes.toBytesBinary("\\x01row2");
229
230    Table t1 = TEST_UTIL.createTable(tableName1, FAMILY);
231    Table t2 = TEST_UTIL.createTable(tableName2, FAMILY);
232
233    // put rows into the first table
234    Put p = new Put(ROW0);
235    p.addColumn(FAMILY, COLUMN1, COLUMN1);
236    t1.put(p);
237    p = new Put(ROW1);
238    p.addColumn(FAMILY, COLUMN1, COLUMN1);
239    t1.put(p);
240    p = new Put(ROW2);
241    p.addColumn(FAMILY, COLUMN1, COLUMN1);
242    t1.put(p);
243
244    CopyTable copy = new CopyTable();
245    assertEquals(
246      0,
247      ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()),
248        copy, new String[] { "--new.name=" + tableName2, "--startrow=\\x01row1",
249            "--stoprow=\\x01row2", tableName1.getNameAsString() }));
250
251    // verify the data was copied into table 2
252    // row1 exist, row0, row2 do not exist
253    Get g = new Get(ROW1);
254    Result r = t2.get(g);
255    assertEquals(1, r.size());
256    assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
257
258    g = new Get(ROW0);
259    r = t2.get(g);
260    assertEquals(0, r.size());
261
262    g = new Get(ROW2);
263    r = t2.get(g);
264    assertEquals(0, r.size());
265
266    t1.close();
267    t2.close();
268    TEST_UTIL.deleteTable(tableName1);
269    TEST_UTIL.deleteTable(tableName2);
270  }
271
272  /**
273   * Test copy of table from sourceTable to targetTable all rows from family a
274   */
275  @Test
276  public void testRenameFamily() throws Exception {
277    final TableName sourceTable = TableName.valueOf(name.getMethodName() + "source");
278    final TableName targetTable = TableName.valueOf(name.getMethodName() + "-target");
279
280    byte[][] families = { FAMILY_A, FAMILY_B };
281
282    Table t = TEST_UTIL.createTable(sourceTable, families);
283    Table t2 = TEST_UTIL.createTable(targetTable, families);
284    Put p = new Put(ROW1);
285    p.addColumn(FAMILY_A, QUALIFIER, Bytes.toBytes("Data11"));
286    p.addColumn(FAMILY_B, QUALIFIER, Bytes.toBytes("Data12"));
287    p.addColumn(FAMILY_A, QUALIFIER, Bytes.toBytes("Data13"));
288    t.put(p);
289    p = new Put(ROW2);
290    p.addColumn(FAMILY_B, QUALIFIER, Bytes.toBytes("Dat21"));
291    p.addColumn(FAMILY_A, QUALIFIER, Bytes.toBytes("Data22"));
292    p.addColumn(FAMILY_B, QUALIFIER, Bytes.toBytes("Data23"));
293    t.put(p);
294
295    long currentTime = System.currentTimeMillis();
296    String[] args = new String[] { "--new.name=" + targetTable, "--families=a:b", "--all.cells",
297        "--starttime=" + (currentTime - 100000), "--endtime=" + (currentTime + 100000),
298        "--versions=1", sourceTable.getNameAsString() };
299    assertNull(t2.get(new Get(ROW1)).getRow());
300
301    assertTrue(runCopy(args));
302
303    assertNotNull(t2.get(new Get(ROW1)).getRow());
304    Result res = t2.get(new Get(ROW1));
305    byte[] b1 = res.getValue(FAMILY_B, QUALIFIER);
306    assertEquals("Data13", new String(b1));
307    assertNotNull(t2.get(new Get(ROW2)).getRow());
308    res = t2.get(new Get(ROW2));
309    b1 = res.getValue(FAMILY_A, QUALIFIER);
310    // Data from the family of B is not copied
311    assertNull(b1);
312
313  }
314
315  /**
316   * Test main method of CopyTable.
317   */
318  @Test
319  public void testMainMethod() throws Exception {
320    String[] emptyArgs = { "-h" };
321    PrintStream oldWriter = System.err;
322    ByteArrayOutputStream data = new ByteArrayOutputStream();
323    PrintStream writer = new PrintStream(data);
324    System.setErr(writer);
325    SecurityManager SECURITY_MANAGER = System.getSecurityManager();
326    LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
327    System.setSecurityManager(newSecurityManager);
328    try {
329      CopyTable.main(emptyArgs);
330      fail("should be exit");
331    } catch (SecurityException e) {
332      assertEquals(1, newSecurityManager.getExitCode());
333    } finally {
334      System.setErr(oldWriter);
335      System.setSecurityManager(SECURITY_MANAGER);
336    }
337    assertTrue(data.toString().contains("rs.class"));
338    // should print usage information
339    assertTrue(data.toString().contains("Usage:"));
340  }
341
342  private boolean runCopy(String[] args) throws Exception {
343    int status = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), new CopyTable(),
344        args);
345    return status == 0;
346  }
347
348  private void loadData(Table t, byte[] family, byte[] column) throws IOException {
349    for (int i = 0; i < 10; i++) {
350      byte[] row = Bytes.toBytes("row" + i);
351      Put p = new Put(row);
352      p.addColumn(family, column, row);
353      t.put(p);
354    }
355  }
356
357  private void verifyRows(Table t, byte[] family, byte[] column) throws IOException {
358    for (int i = 0; i < 10; i++) {
359      byte[] row = Bytes.toBytes("row" + i);
360      Get g = new Get(row).addFamily(family);
361      Result r = t.get(g);
362      Assert.assertNotNull(r);
363      Assert.assertEquals(1, r.size());
364      Cell cell = r.rawCells()[0];
365      Assert.assertTrue(CellUtil.matchingQualifier(cell, column));
366      Assert.assertEquals(Bytes.compareTo(cell.getValueArray(), cell.getValueOffset(),
367        cell.getValueLength(), row, 0, row.length), 0);
368    }
369  }
370
371  private Table createTable(TableName tableName, byte[] family, boolean isMob) throws IOException {
372    if (isMob) {
373      ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(family)
374          .setMobEnabled(true).setMobThreshold(1).build();
375      TableDescriptor desc =
376          TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(cfd).build();
377      return TEST_UTIL.createTable(desc, null);
378    } else {
379      return TEST_UTIL.createTable(tableName, family);
380    }
381  }
382
383  private void testCopyTableBySnapshot(String tablePrefix, boolean bulkLoad, boolean isMob)
384      throws Exception {
385    TableName table1 = TableName.valueOf(tablePrefix + 1);
386    TableName table2 = TableName.valueOf(tablePrefix + 2);
387    Table t1 = createTable(table1, FAMILY_A, isMob);
388    Table t2 = createTable(table2, FAMILY_A, isMob);
389    loadData(t1, FAMILY_A, Bytes.toBytes("qualifier"));
390    String snapshot = tablePrefix + "_snapshot";
391    TEST_UTIL.getAdmin().snapshot(snapshot, table1);
392    boolean success;
393    if (bulkLoad) {
394      success =
395          runCopy(new String[] { "--snapshot", "--new.name=" + table2, "--bulkload", snapshot });
396    } else {
397      success = runCopy(new String[] { "--snapshot", "--new.name=" + table2, snapshot });
398    }
399    Assert.assertTrue(success);
400    verifyRows(t2, FAMILY_A, Bytes.toBytes("qualifier"));
401  }
402
403  @Test
404  public void testLoadingSnapshotToTable() throws Exception {
405    testCopyTableBySnapshot("testLoadingSnapshotToTable", false, false);
406  }
407
408  @Test
409  public void tsetLoadingSnapshotToMobTable() throws Exception {
410    testCopyTableBySnapshot("testLoadingSnapshotToMobTable", false, true);
411  }
412
413  @Test
414  public void testLoadingSnapshotAndBulkLoadToTable() throws Exception {
415    testCopyTableBySnapshot("testLoadingSnapshotAndBulkLoadToTable", true, false);
416  }
417
418  @Test
419  public void testLoadingSnapshotAndBulkLoadToMobTable() throws Exception {
420    testCopyTableBySnapshot("testLoadingSnapshotAndBulkLoadToMobTable", true, true);
421  }
422
423  @Test
424  public void testLoadingSnapshotToRemoteCluster() throws Exception {
425    Assert.assertFalse(runCopy(
426      new String[] { "--snapshot", "--peerAdr=hbase://remoteHBase", "sourceSnapshotName" }));
427  }
428
429  @Test
430  public void testLoadingSnapshotWithoutSnapshotName() throws Exception {
431    Assert.assertFalse(runCopy(new String[] { "--snapshot", "--peerAdr=hbase://remoteHBase" }));
432  }
433
434  @Test
435  public void testLoadingSnapshotWithoutDestTable() throws Exception {
436    Assert.assertFalse(runCopy(new String[] { "--snapshot", "sourceSnapshotName" }));
437  }
438}