Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.mapreduce;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022import static org.junit.jupiter.api.Assertions.assertThrows;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024import static org.junit.jupiter.api.Assertions.fail;
025
026import java.io.ByteArrayOutputStream;
027import java.io.IOException;
028import java.io.PrintStream;
029import java.util.HashMap;
030import java.util.Map;
031import org.apache.hadoop.hbase.HBaseTestingUtil;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
035import org.apache.hadoop.hbase.client.SnapshotDescription;
036import org.apache.hadoop.hbase.client.SnapshotType;
037import org.apache.hadoop.hbase.client.Table;
038import org.apache.hadoop.hbase.client.TableDescriptor;
039import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
040import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
041import org.apache.hadoop.hbase.snapshot.SnapshotTTLExpiredException;
042import org.apache.hadoop.hbase.testclassification.LargeTests;
043import org.apache.hadoop.hbase.testclassification.MapReduceTests;
044import org.apache.hadoop.hbase.util.Bytes;
045import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
046import org.apache.hadoop.hbase.util.LauncherSecurityManager;
047import org.junit.jupiter.api.AfterAll;
048import org.junit.jupiter.api.BeforeAll;
049import org.junit.jupiter.api.Tag;
050import org.junit.jupiter.api.Test;
051import org.junit.jupiter.api.TestInfo;
052
053/**
054 * Basic test for the CopyTable M/R tool
055 */
056@Tag(MapReduceTests.TAG)
057@Tag(LargeTests.TAG)
058public class TestCopyTable extends CopyTableTestBase {
059
060  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
061
062  @BeforeAll
063  public static void beforeClass() throws Exception {
064    TEST_UTIL.startMiniCluster(3);
065  }
066
067  @AfterAll
068  public static void afterClass() throws Exception {
069    TEST_UTIL.shutdownMiniCluster();
070  }
071
072  @Override
073  protected Table createSourceTable(TableDescriptor desc) throws Exception {
074    return TEST_UTIL.createTable(desc, null);
075  }
076
077  @Override
078  protected Table createTargetTable(TableDescriptor desc) throws Exception {
079    return TEST_UTIL.createTable(desc, null);
080  }
081
082  @Override
083  protected void dropSourceTable(TableName tableName) throws Exception {
084    TEST_UTIL.deleteTable(tableName);
085  }
086
087  @Override
088  protected void dropTargetTable(TableName tableName) throws Exception {
089    TEST_UTIL.deleteTable(tableName);
090  }
091
092  @Override
093  protected String[] getPeerClusterOptions() throws Exception {
094    return new String[0];
095  }
096
097  /**
098   * Simple end-to-end test
099   */
100  @Test
101  public void testCopyTable(TestInfo testInfo) throws Exception {
102    doCopyTableTest(TEST_UTIL.getConfiguration(), false, testInfo);
103  }
104
105  /**
106   * Simple end-to-end test with bulkload.
107   */
108  @Test
109  public void testCopyTableWithBulkload(TestInfo testInfo) throws Exception {
110    doCopyTableTest(TEST_UTIL.getConfiguration(), true, testInfo);
111  }
112
113  /**
114   * Simple end-to-end test on table with MOB
115   */
116  @Test
117  public void testCopyTableWithMob(TestInfo testInfo) throws Exception {
118    doCopyTableTestWithMob(TEST_UTIL.getConfiguration(), false, testInfo);
119  }
120
121  /**
122   * Simple end-to-end test with bulkload on table with MOB.
123   */
124  @Test
125  public void testCopyTableWithBulkloadWithMob(TestInfo testInfo) throws Exception {
126    doCopyTableTestWithMob(TEST_UTIL.getConfiguration(), true, testInfo);
127  }
128
129  @Test
130  public void testStartStopRow(TestInfo testInfo) throws Exception {
131    testStartStopRow(TEST_UTIL.getConfiguration(), testInfo);
132  }
133
134  /**
135   * Test copy of table from sourceTable to targetTable all rows from family a
136   */
137  @Test
138  public void testRenameFamily(TestInfo testInfo) throws Exception {
139    testRenameFamily(TEST_UTIL.getConfiguration(), testInfo);
140  }
141
142  /**
143   * Test main method of CopyTable.
144   */
145  @Test
146  public void testMainMethod() throws Exception {
147    String[] emptyArgs = { "-h" };
148    PrintStream oldWriter = System.err;
149    ByteArrayOutputStream data = new ByteArrayOutputStream();
150    PrintStream writer = new PrintStream(data);
151    System.setErr(writer);
152    SecurityManager SECURITY_MANAGER = System.getSecurityManager();
153    LauncherSecurityManager newSecurityManager = new LauncherSecurityManager();
154    System.setSecurityManager(newSecurityManager);
155    try {
156      CopyTable.main(emptyArgs);
157      fail("should be exit");
158    } catch (SecurityException e) {
159      assertEquals(1, newSecurityManager.getExitCode());
160    } finally {
161      System.setErr(oldWriter);
162      System.setSecurityManager(SECURITY_MANAGER);
163    }
164    assertTrue(data.toString().contains("rs.class"));
165    // should print usage information
166    assertTrue(data.toString().contains("Usage:"));
167  }
168
169  private Table createTable(TableName tableName, byte[] family, boolean isMob) throws IOException {
170    if (isMob) {
171      ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(family)
172        .setMobEnabled(true).setMobThreshold(1).build();
173      TableDescriptor desc =
174        TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(cfd).build();
175      return TEST_UTIL.createTable(desc, null);
176    } else {
177      return TEST_UTIL.createTable(tableName, family);
178    }
179  }
180
181  private void testCopyTableBySnapshot(String tablePrefix, boolean bulkLoad, boolean isMob)
182    throws Exception {
183    TableName table1 = TableName.valueOf(tablePrefix + 1);
184    TableName table2 = TableName.valueOf(tablePrefix + 2);
185    String snapshot = tablePrefix + "_snapshot";
186    try (Table t1 = createTable(table1, FAMILY_A, isMob);
187      Table t2 = createTable(table2, FAMILY_A, isMob)) {
188      loadData(t1, FAMILY_A, Bytes.toBytes("qualifier"));
189      TEST_UTIL.getAdmin().snapshot(snapshot, table1);
190      boolean success;
191      if (bulkLoad) {
192        success = runCopy(TEST_UTIL.getConfiguration(),
193          new String[] { "--snapshot", "--new.name=" + table2, "--bulkload", snapshot });
194      } else {
195        success = runCopy(TEST_UTIL.getConfiguration(),
196          new String[] { "--snapshot", "--new.name=" + table2, snapshot });
197      }
198      assertTrue(success);
199      verifyRows(t2, FAMILY_A, Bytes.toBytes("qualifier"));
200    } finally {
201      TEST_UTIL.getAdmin().deleteSnapshot(snapshot);
202      TEST_UTIL.deleteTable(table1);
203      TEST_UTIL.deleteTable(table2);
204    }
205  }
206
207  @Test
208  public void testLoadingSnapshotToTable() throws Exception {
209    testCopyTableBySnapshot("testLoadingSnapshotToTable", false, false);
210  }
211
212  @Test
213  public void testLoadingTtlExpiredSnapshotToTable() throws Exception {
214    String tablePrefix = "testLoadingExpiredSnapshotToTable";
215    TableName table1 = TableName.valueOf(tablePrefix + 1);
216    TableName table2 = TableName.valueOf(tablePrefix + 2);
217    Table t1 = createTable(table1, FAMILY_A, false);
218    createTable(table2, FAMILY_A, false);
219    loadData(t1, FAMILY_A, Bytes.toBytes("qualifier"));
220    String snapshot = tablePrefix + "_snapshot";
221    Map<String, Object> properties = new HashMap<>();
222    properties.put("TTL", 10);
223    SnapshotDescription snapshotDescription = new SnapshotDescription(snapshot, table1,
224      SnapshotType.FLUSH, null, EnvironmentEdgeManager.currentTime(), -1, properties);
225    TEST_UTIL.getAdmin().snapshot(snapshotDescription);
226    boolean isExist =
227      TEST_UTIL.getAdmin().listSnapshots().stream().anyMatch(ele -> snapshot.equals(ele.getName()));
228    assertTrue(isExist);
229    int retry = 6;
230    while (
231      !SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDescription.getTtl(),
232        snapshotDescription.getCreationTime(), EnvironmentEdgeManager.currentTime()) && retry > 0
233    ) {
234      retry--;
235      Thread.sleep(10 * 1000);
236    }
237    boolean isExpiredSnapshot =
238      SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDescription.getTtl(),
239        snapshotDescription.getCreationTime(), EnvironmentEdgeManager.currentTime());
240    assertTrue(isExpiredSnapshot);
241    String[] args = new String[] { "--snapshot", "--new.name=" + table2, "--bulkload", snapshot };
242    assertThrows(SnapshotTTLExpiredException.class,
243      () -> runCopy(TEST_UTIL.getConfiguration(), args));
244  }
245
246  @Test
247  public void tsetLoadingSnapshotToMobTable() throws Exception {
248    testCopyTableBySnapshot("testLoadingSnapshotToMobTable", false, true);
249  }
250
251  @Test
252  public void testLoadingSnapshotAndBulkLoadToTable() throws Exception {
253    testCopyTableBySnapshot("testLoadingSnapshotAndBulkLoadToTable", true, false);
254  }
255
256  @Test
257  public void testLoadingSnapshotAndBulkLoadToMobTable() throws Exception {
258    testCopyTableBySnapshot("testLoadingSnapshotAndBulkLoadToMobTable", true, true);
259  }
260
261  @Test
262  public void testLoadingSnapshotWithoutSnapshotName() throws Exception {
263    assertFalse(runCopy(TEST_UTIL.getConfiguration(), new String[] { "--snapshot" }));
264  }
265
266  @Test
267  public void testLoadingSnapshotWithoutDestTable() throws Exception {
268    assertFalse(
269      runCopy(TEST_UTIL.getConfiguration(), new String[] { "--snapshot", "sourceSnapshotName" }));
270  }
271
272}