001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.mapreduce;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertNull;
023import static org.junit.Assert.assertTrue;
024import static org.junit.Assert.fail;
025
026import java.io.ByteArrayOutputStream;
027import java.io.PrintStream;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.hbase.CellUtil;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.HBaseTestingUtility;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.client.Get;
034import org.apache.hadoop.hbase.client.Put;
035import org.apache.hadoop.hbase.client.Result;
036import org.apache.hadoop.hbase.client.Table;
037import org.apache.hadoop.hbase.testclassification.LargeTests;
038import org.apache.hadoop.hbase.testclassification.MapReduceTests;
039import org.apache.hadoop.hbase.util.Bytes;
040import org.apache.hadoop.hbase.util.LauncherSecurityManager;
041import org.apache.hadoop.util.ToolRunner;
042import org.junit.AfterClass;
043import org.junit.BeforeClass;
044import org.junit.ClassRule;
045import org.junit.Rule;
046import org.junit.Test;
047import org.junit.experimental.categories.Category;
048import org.junit.rules.TestName;
049
050/**
051 * Basic test for the CopyTable M/R tool
052 */
053@Category({MapReduceTests.class, LargeTests.class})
054public class TestCopyTable {
055
056  @ClassRule
057  public static final HBaseClassTestRule CLASS_RULE =
058      HBaseClassTestRule.forClass(TestCopyTable.class);
059
060  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
061  private static final byte[] ROW1 = Bytes.toBytes("row1");
062  private static final byte[] ROW2 = Bytes.toBytes("row2");
063  private static final String FAMILY_A_STRING = "a";
064  private static final String FAMILY_B_STRING = "b";
065  private static final byte[] FAMILY_A = Bytes.toBytes(FAMILY_A_STRING);
066  private static final byte[] FAMILY_B = Bytes.toBytes(FAMILY_B_STRING);
067  private static final byte[] QUALIFIER = Bytes.toBytes("q");
068
069  @Rule
070  public TestName name = new TestName();
071
072  @BeforeClass
073  public static void beforeClass() throws Exception {
074    TEST_UTIL.startMiniCluster(3);
075  }
076
077  @AfterClass
078  public static void afterClass() throws Exception {
079    TEST_UTIL.shutdownMiniCluster();
080  }
081
082  private void doCopyTableTest(boolean bulkload) throws Exception {
083    final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1");
084    final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2");
085    final byte[] FAMILY = Bytes.toBytes("family");
086    final byte[] COLUMN1 = Bytes.toBytes("c1");
087
088    try (Table t1 = TEST_UTIL.createTable(tableName1, FAMILY);
089         Table t2 = TEST_UTIL.createTable(tableName2, FAMILY);) {
090      // put rows into the first table
091      for (int i = 0; i < 10; i++) {
092        Put p = new Put(Bytes.toBytes("row" + i));
093        p.addColumn(FAMILY, COLUMN1, COLUMN1);
094        t1.put(p);
095      }
096
097      CopyTable copy = new CopyTable();
098
099      int code;
100      if (bulkload) {
101        code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()),
102            copy, new String[] { "--new.name=" + tableName2.getNameAsString(),
103            "--bulkload", tableName1.getNameAsString() });
104      } else {
105        code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()),
106            copy, new String[] { "--new.name=" + tableName2.getNameAsString(),
107            tableName1.getNameAsString() });
108      }
109      assertEquals("copy job failed", 0, code);
110
111      // verify the data was copied into table 2
112      for (int i = 0; i < 10; i++) {
113        Get g = new Get(Bytes.toBytes("row" + i));
114        Result r = t2.get(g);
115        assertEquals(1, r.size());
116        assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
117      }
118    } finally {
119      TEST_UTIL.deleteTable(tableName1);
120      TEST_UTIL.deleteTable(tableName2);
121    }
122  }
123
124  /**
125   * Simple end-to-end test
126   * @throws Exception
127   */
128  @Test
129  public void testCopyTable() throws Exception {
130    doCopyTableTest(false);
131  }
132
133  /**
134   * Simple end-to-end test with bulkload.
135   */
136  @Test
137  public void testCopyTableWithBulkload() throws Exception {
138    doCopyTableTest(true);
139  }
140
141  @Test
142  public void testStartStopRow() throws Exception {
143    final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1");
144    final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2");
145    final byte[] FAMILY = Bytes.toBytes("family");
146    final byte[] COLUMN1 = Bytes.toBytes("c1");
147    final byte[] ROW0 = Bytes.toBytesBinary("\\x01row0");
148    final byte[] ROW1 = Bytes.toBytesBinary("\\x01row1");
149    final byte[] ROW2 = Bytes.toBytesBinary("\\x01row2");
150
151    Table t1 = TEST_UTIL.createTable(tableName1, FAMILY);
152    Table t2 = TEST_UTIL.createTable(tableName2, FAMILY);
153
154    // put rows into the first table
155    Put p = new Put(ROW0);
156    p.addColumn(FAMILY, COLUMN1, COLUMN1);
157    t1.put(p);
158    p = new Put(ROW1);
159    p.addColumn(FAMILY, COLUMN1, COLUMN1);
160    t1.put(p);
161    p = new Put(ROW2);
162    p.addColumn(FAMILY, COLUMN1, COLUMN1);
163    t1.put(p);
164
165    CopyTable copy = new CopyTable();
166    assertEquals(
167      0,
168      ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()),
169        copy, new String[] { "--new.name=" + tableName2, "--startrow=\\x01row1",
170            "--stoprow=\\x01row2", tableName1.getNameAsString() }));
171
172    // verify the data was copied into table 2
173    // row1 exist, row0, row2 do not exist
174    Get g = new Get(ROW1);
175    Result r = t2.get(g);
176    assertEquals(1, r.size());
177    assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
178
179    g = new Get(ROW0);
180    r = t2.get(g);
181    assertEquals(0, r.size());
182
183    g = new Get(ROW2);
184    r = t2.get(g);
185    assertEquals(0, r.size());
186
187    t1.close();
188    t2.close();
189    TEST_UTIL.deleteTable(tableName1);
190    TEST_UTIL.deleteTable(tableName2);
191  }
192
193  /**
194   * Test copy of table from sourceTable to targetTable all rows from family a
195   */
196  @Test
197  public void testRenameFamily() throws Exception {
198    final TableName sourceTable = TableName.valueOf(name.getMethodName() + "source");
199    final TableName targetTable = TableName.valueOf(name.getMethodName() + "-target");
200
201    byte[][] families = { FAMILY_A, FAMILY_B };
202
203    Table t = TEST_UTIL.createTable(sourceTable, families);
204    Table t2 = TEST_UTIL.createTable(targetTable, families);
205    Put p = new Put(ROW1);
206    p.addColumn(FAMILY_A, QUALIFIER, Bytes.toBytes("Data11"));
207    p.addColumn(FAMILY_B, QUALIFIER, Bytes.toBytes("Data12"));
208    p.addColumn(FAMILY_A, QUALIFIER, Bytes.toBytes("Data13"));
209    t.put(p);
210    p = new Put(ROW2);
211    p.addColumn(FAMILY_B, QUALIFIER, Bytes.toBytes("Dat21"));
212    p.addColumn(FAMILY_A, QUALIFIER, Bytes.toBytes("Data22"));
213    p.addColumn(FAMILY_B, QUALIFIER, Bytes.toBytes("Data23"));
214    t.put(p);
215
216    long currentTime = System.currentTimeMillis();
217    String[] args = new String[] { "--new.name=" + targetTable, "--families=a:b", "--all.cells",
218        "--starttime=" + (currentTime - 100000), "--endtime=" + (currentTime + 100000),
219        "--versions=1", sourceTable.getNameAsString() };
220    assertNull(t2.get(new Get(ROW1)).getRow());
221
222    assertTrue(runCopy(args));
223
224    assertNotNull(t2.get(new Get(ROW1)).getRow());
225    Result res = t2.get(new Get(ROW1));
226    byte[] b1 = res.getValue(FAMILY_B, QUALIFIER);
227    assertEquals("Data13", new String(b1));
228    assertNotNull(t2.get(new Get(ROW2)).getRow());
229    res = t2.get(new Get(ROW2));
230    b1 = res.getValue(FAMILY_A, QUALIFIER);
231    // Data from the family of B is not copied
232    assertNull(b1);
233
234  }
235
236  /**
237   * Test main method of CopyTable.
238   */
239  @Test
240  public void testMainMethod() throws Exception {
241    String[] emptyArgs = { "-h" };
242    PrintStream oldWriter = System.err;
243    ByteArrayOutputStream data = new ByteArrayOutputStream();
244    PrintStream writer = new PrintStream(data);
245    System.setErr(writer);
246    SecurityManager SECURITY_MANAGER = System.getSecurityManager();
247    LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
248    System.setSecurityManager(newSecurityManager);
249    try {
250      CopyTable.main(emptyArgs);
251      fail("should be exit");
252    } catch (SecurityException e) {
253      assertEquals(1, newSecurityManager.getExitCode());
254    } finally {
255      System.setErr(oldWriter);
256      System.setSecurityManager(SECURITY_MANAGER);
257    }
258    assertTrue(data.toString().contains("rs.class"));
259    // should print usage information
260    assertTrue(data.toString().contains("Usage:"));
261  }
262
263  private boolean runCopy(String[] args) throws Exception {
264    int status = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), new CopyTable(),
265        args);
266    return status == 0;
267  }
268}