001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.mapreduce;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertNotNull;
022import static org.junit.jupiter.api.Assertions.assertNull;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024
025import java.io.IOException;
026import org.apache.commons.lang3.ArrayUtils;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.Cell;
029import org.apache.hadoop.hbase.CellUtil;
030import org.apache.hadoop.hbase.TableName;
031import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
032import org.apache.hadoop.hbase.client.Get;
033import org.apache.hadoop.hbase.client.Put;
034import org.apache.hadoop.hbase.client.Result;
035import org.apache.hadoop.hbase.client.Table;
036import org.apache.hadoop.hbase.client.TableDescriptor;
037import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
038import org.apache.hadoop.hbase.mob.MobTestUtil;
039import org.apache.hadoop.hbase.util.Bytes;
040import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
041import org.apache.hadoop.util.ToolRunner;
042import org.junit.jupiter.api.TestInfo;
043
044/**
045 * Base class for testing CopyTable MR tool.
046 */
047public abstract class CopyTableTestBase {
048
049  protected static final byte[] ROW1 = Bytes.toBytes("row1");
050  protected static final byte[] ROW2 = Bytes.toBytes("row2");
051  protected static final String FAMILY_A_STRING = "a";
052  protected static final String FAMILY_B_STRING = "b";
053  protected static final byte[] FAMILY_A = Bytes.toBytes(FAMILY_A_STRING);
054  protected static final byte[] FAMILY_B = Bytes.toBytes(FAMILY_B_STRING);
055  protected static final byte[] QUALIFIER = Bytes.toBytes("q");
056
057  protected abstract Table createSourceTable(TableDescriptor desc) throws Exception;
058
059  protected abstract Table createTargetTable(TableDescriptor desc) throws Exception;
060
061  protected abstract void dropSourceTable(TableName tableName) throws Exception;
062
063  protected abstract void dropTargetTable(TableName tableName) throws Exception;
064
065  protected abstract String[] getPeerClusterOptions() throws Exception;
066
067  protected final void loadData(Table t, byte[] family, byte[] column) throws IOException {
068    for (int i = 0; i < 10; i++) {
069      byte[] row = Bytes.toBytes("row" + i);
070      Put p = new Put(row);
071      p.addColumn(family, column, row);
072      t.put(p);
073    }
074  }
075
076  protected final void verifyRows(Table t, byte[] family, byte[] column) throws IOException {
077    for (int i = 0; i < 10; i++) {
078      byte[] row = Bytes.toBytes("row" + i);
079      Get g = new Get(row).addFamily(family);
080      Result r = t.get(g);
081      assertNotNull(r);
082      assertEquals(1, r.size());
083      Cell cell = r.rawCells()[0];
084      assertTrue(CellUtil.matchingQualifier(cell, column));
085      assertEquals(Bytes.compareTo(cell.getValueArray(), cell.getValueOffset(),
086        cell.getValueLength(), row, 0, row.length), 0);
087    }
088  }
089
090  protected final void doCopyTableTest(Configuration conf, boolean bulkload, TestInfo testInfo)
091    throws Exception {
092    TableName tableName1 = TableName.valueOf(testInfo.getTestMethod().get().getName() + "1");
093    TableName tableName2 = TableName.valueOf(testInfo.getTestMethod().get().getName() + "2");
094    byte[] family = Bytes.toBytes("family");
095    byte[] column = Bytes.toBytes("c1");
096    TableDescriptor desc1 = TableDescriptorBuilder.newBuilder(tableName1)
097      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build();
098    TableDescriptor desc2 = TableDescriptorBuilder.newBuilder(tableName2)
099      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build();
100
101    try (Table t1 = createSourceTable(desc1); Table t2 = createTargetTable(desc2)) {
102      // put rows into the first table
103      loadData(t1, family, column);
104
105      String[] peerClusterOptions = getPeerClusterOptions();
106      if (bulkload) {
107        assertTrue(runCopy(conf,
108          ArrayUtils.addAll(peerClusterOptions, "--new.name=" + tableName2.getNameAsString(),
109            "--bulkload", tableName1.getNameAsString())));
110      } else {
111        assertTrue(runCopy(conf, ArrayUtils.addAll(peerClusterOptions,
112          "--new.name=" + tableName2.getNameAsString(), tableName1.getNameAsString())));
113      }
114
115      // verify the data was copied into table 2
116      verifyRows(t2, family, column);
117    } finally {
118      dropSourceTable(tableName1);
119      dropTargetTable(tableName2);
120    }
121  }
122
123  protected final void doCopyTableTestWithMob(Configuration conf, boolean bulkload,
124    TestInfo testInfo) throws Exception {
125    TableName tableName1 = TableName.valueOf(testInfo.getTestMethod().get().getName() + "1");
126    TableName tableName2 = TableName.valueOf(testInfo.getTestMethod().get().getName() + "2");
127    byte[] family = Bytes.toBytes("mob");
128    byte[] column = Bytes.toBytes("c1");
129
130    ColumnFamilyDescriptorBuilder cfd = ColumnFamilyDescriptorBuilder.newBuilder(family);
131
132    cfd.setMobEnabled(true);
133    cfd.setMobThreshold(5);
134    TableDescriptor desc1 =
135      TableDescriptorBuilder.newBuilder(tableName1).setColumnFamily(cfd.build()).build();
136    TableDescriptor desc2 =
137      TableDescriptorBuilder.newBuilder(tableName2).setColumnFamily(cfd.build()).build();
138
139    try (Table t1 = createSourceTable(desc1); Table t2 = createTargetTable(desc2)) {
140      // put rows into the first table
141      for (int i = 0; i < 10; i++) {
142        Put p = new Put(Bytes.toBytes("row" + i));
143        p.addColumn(family, column, column);
144        t1.put(p);
145      }
146
147      String[] peerClusterOptions = getPeerClusterOptions();
148      if (bulkload) {
149        assertTrue(runCopy(conf,
150          ArrayUtils.addAll(peerClusterOptions, "--new.name=" + tableName2.getNameAsString(),
151            "--bulkload", tableName1.getNameAsString())));
152      } else {
153        assertTrue(runCopy(conf, ArrayUtils.addAll(peerClusterOptions,
154          "--new.name=" + tableName2.getNameAsString(), tableName1.getNameAsString())));
155      }
156
157      // verify the data was copied into table 2
158      for (int i = 0; i < 10; i++) {
159        Get g = new Get(Bytes.toBytes("row" + i));
160        Result r = t2.get(g);
161        assertEquals(1, r.size());
162        assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], column));
163        assertEquals(t1.getDescriptor().getValue("row" + i), t2.getDescriptor().getValue("row" + i),
164          "compare row values between two tables");
165      }
166
167      assertEquals(MobTestUtil.countMobRows(t1), MobTestUtil.countMobRows(t2),
168        "compare count of mob rows after table copy");
169      assertEquals(t1.getDescriptor().getValues().size(), t2.getDescriptor().getValues().size(),
170        "compare count of mob row values between two tables");
171      assertTrue(MobTestUtil.countMobRows(t2) > 0, "The mob row count is 0 but should be > 0");
172    } finally {
173      dropSourceTable(tableName1);
174      dropTargetTable(tableName2);
175    }
176  }
177
178  protected final boolean runCopy(Configuration conf, String[] args) throws Exception {
179    int status = ToolRunner.run(conf, new CopyTable(), args);
180    return status == 0;
181  }
182
183  protected final void testStartStopRow(Configuration conf, TestInfo testInfo) throws Exception {
184    final TableName tableName1 = TableName.valueOf(testInfo.getTestMethod().get().getName() + "1");
185    final TableName tableName2 = TableName.valueOf(testInfo.getTestMethod().get().getName() + "2");
186    final byte[] family = Bytes.toBytes("family");
187    final byte[] column = Bytes.toBytes("c1");
188    final byte[] row0 = Bytes.toBytesBinary("\\x01row0");
189    final byte[] row1 = Bytes.toBytesBinary("\\x01row1");
190    final byte[] row2 = Bytes.toBytesBinary("\\x01row2");
191    TableDescriptor desc1 = TableDescriptorBuilder.newBuilder(tableName1)
192      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build();
193    TableDescriptor desc2 = TableDescriptorBuilder.newBuilder(tableName2)
194      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build();
195    try (Table t1 = createSourceTable(desc1); Table t2 = createTargetTable(desc2)) {
196      // put rows into the first table
197      Put p = new Put(row0);
198      p.addColumn(family, column, column);
199      t1.put(p);
200      p = new Put(row1);
201      p.addColumn(family, column, column);
202      t1.put(p);
203      p = new Put(row2);
204      p.addColumn(family, column, column);
205      t1.put(p);
206
207      String[] peerClusterOptions = getPeerClusterOptions();
208      assertTrue(runCopy(conf, ArrayUtils.addAll(peerClusterOptions, "--new.name=" + tableName2,
209        "--startrow=\\x01row1", "--stoprow=\\x01row2", tableName1.getNameAsString())));
210
211      // verify the data was copied into table 2
212      // row1 exist, row0, row2 do not exist
213      Get g = new Get(row1);
214      Result r = t2.get(g);
215      assertEquals(1, r.size());
216      assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], column));
217
218      g = new Get(row0);
219      r = t2.get(g);
220      assertEquals(0, r.size());
221
222      g = new Get(row2);
223      r = t2.get(g);
224      assertEquals(0, r.size());
225    } finally {
226      dropSourceTable(tableName1);
227      dropTargetTable(tableName2);
228    }
229  }
230
231  protected final void testRenameFamily(Configuration conf, TestInfo testInfo) throws Exception {
232    TableName sourceTable = TableName.valueOf(testInfo.getTestMethod().get().getName() + "-source");
233    TableName targetTable = TableName.valueOf(testInfo.getTestMethod().get().getName() + "-target");
234
235    TableDescriptor desc1 = TableDescriptorBuilder.newBuilder(sourceTable)
236      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_A))
237      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_B)).build();
238    TableDescriptor desc2 = TableDescriptorBuilder.newBuilder(targetTable)
239      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_A))
240      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_B)).build();
241
242    try (Table t = createSourceTable(desc1); Table t2 = createTargetTable(desc2)) {
243      Put p = new Put(ROW1);
244      p.addColumn(FAMILY_A, QUALIFIER, Bytes.toBytes("Data11"));
245      p.addColumn(FAMILY_B, QUALIFIER, Bytes.toBytes("Data12"));
246      p.addColumn(FAMILY_A, QUALIFIER, Bytes.toBytes("Data13"));
247      t.put(p);
248      p = new Put(ROW2);
249      p.addColumn(FAMILY_B, QUALIFIER, Bytes.toBytes("Dat21"));
250      p.addColumn(FAMILY_A, QUALIFIER, Bytes.toBytes("Data22"));
251      p.addColumn(FAMILY_B, QUALIFIER, Bytes.toBytes("Data23"));
252      t.put(p);
253
254      long currentTime = EnvironmentEdgeManager.currentTime();
255      String[] args = ArrayUtils.addAll(getPeerClusterOptions(), "--new.name=" + targetTable,
256        "--families=a:b", "--all.cells", "--starttime=" + (currentTime - 100000),
257        "--endtime=" + (currentTime + 100000), "--versions=1", sourceTable.getNameAsString());
258      assertNull(t2.get(new Get(ROW1)).getRow());
259
260      assertTrue(runCopy(conf, args));
261
262      assertNotNull(t2.get(new Get(ROW1)).getRow());
263      Result res = t2.get(new Get(ROW1));
264      byte[] b1 = res.getValue(FAMILY_B, QUALIFIER);
265      assertEquals("Data13", Bytes.toString(b1));
266      assertNotNull(t2.get(new Get(ROW2)).getRow());
267      res = t2.get(new Get(ROW2));
268      b1 = res.getValue(FAMILY_A, QUALIFIER);
269      // Data from the family of B is not copied
270      assertNull(b1);
271    } finally {
272      dropSourceTable(sourceTable);
273      dropTargetTable(targetTable);
274    }
275  }
276}