001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.mapreduce;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertTrue;
022import static org.junit.jupiter.api.Assertions.fail;
023
024import java.io.ByteArrayOutputStream;
025import java.io.File;
026import java.io.FileInputStream;
027import java.io.PrintStream;
028import org.apache.commons.io.IOUtils;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.FileUtil;
031import org.apache.hadoop.fs.LocalFileSystem;
032import org.apache.hadoop.fs.Path;
033import org.apache.hadoop.hbase.HBaseConfiguration;
034import org.apache.hadoop.hbase.HBaseTestingUtil;
035import org.apache.hadoop.hbase.TableName;
036import org.apache.hadoop.hbase.client.Put;
037import org.apache.hadoop.hbase.client.Table;
038import org.apache.hadoop.hbase.testclassification.LargeTests;
039import org.apache.hadoop.hbase.testclassification.MapReduceTests;
040import org.apache.hadoop.hbase.util.Bytes;
041import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
042import org.apache.hadoop.hbase.util.LauncherSecurityManager;
043import org.apache.hadoop.util.ToolRunner;
044import org.junit.jupiter.api.AfterAll;
045import org.junit.jupiter.api.BeforeAll;
046import org.junit.jupiter.api.Tag;
047import org.junit.jupiter.api.Test;
048import org.junit.jupiter.api.TestInfo;
049
050@Tag(MapReduceTests.TAG)
051@Tag(LargeTests.TAG)
052public class TestCellCounter {
053
054  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
055  private static final byte[] ROW1 = Bytes.toBytesBinary("\\x01row1");
056  private static final byte[] ROW2 = Bytes.toBytesBinary("\\x01row2");
057  private static final String FAMILY_A_STRING = "a";
058  private static final String FAMILY_B_STRING = "b";
059  private static final byte[] FAMILY_A = Bytes.toBytes(FAMILY_A_STRING);
060  private static final byte[] FAMILY_B = Bytes.toBytes(FAMILY_B_STRING);
061  private static final byte[] QUALIFIER = Bytes.toBytes("q");
062
063  private static Path FQ_OUTPUT_DIR;
064  private static final String OUTPUT_DIR =
065    "target" + File.separator + "test-data" + File.separator + "output";
066  private static long now = EnvironmentEdgeManager.currentTime();
067
068  @BeforeAll
069  public static void beforeClass() throws Exception {
070    UTIL.startMiniCluster();
071    FQ_OUTPUT_DIR = new Path(OUTPUT_DIR).makeQualified(new LocalFileSystem());
072    FileUtil.fullyDelete(new File(OUTPUT_DIR));
073  }
074
075  @AfterAll
076  public static void afterClass() throws Exception {
077    UTIL.shutdownMiniCluster();
078  }
079
080  /**
081   * Test CellCounter all data should print to output
082   */
083  @Test
084  public void testCellCounter(TestInfo testInfo) throws Exception {
085    final TableName sourceTable = TableName.valueOf(testInfo.getTestMethod().get().getName());
086    byte[][] families = { FAMILY_A, FAMILY_B };
087    try (Table t = UTIL.createTable(sourceTable, families)) {
088      Put p = new Put(ROW1);
089      p.addColumn(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
090      p.addColumn(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
091      p.addColumn(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
092      t.put(p);
093      p = new Put(ROW2);
094      p.addColumn(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
095      p.addColumn(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
096      p.addColumn(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
097      t.put(p);
098      String[] args = { sourceTable.getNameAsString(), FQ_OUTPUT_DIR.toString(), ";", "^row1" };
099      runCount(args);
100      FileInputStream inputStream =
101        new FileInputStream(OUTPUT_DIR + File.separator + "part-r-00000");
102      String data = IOUtils.toString(inputStream);
103      inputStream.close();
104      assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
105      assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
106      assertTrue(data.contains("Total ROWS" + "\t" + "1"));
107      assertTrue(data.contains("b;q" + "\t" + "1"));
108      assertTrue(data.contains("a;q" + "\t" + "1"));
109      assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
110      assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
111    } finally {
112      FileUtil.fullyDelete(new File(OUTPUT_DIR));
113    }
114  }
115
116  /**
117   * Test CellCounter all data should print to output
118   */
119  @Test
120  public void testCellCounterPrefix(TestInfo testInfo) throws Exception {
121    final TableName sourceTable = TableName.valueOf(testInfo.getTestMethod().get().getName());
122    byte[][] families = { FAMILY_A, FAMILY_B };
123    try (Table t = UTIL.createTable(sourceTable, families)) {
124      Put p = new Put(ROW1);
125      p.addColumn(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
126      p.addColumn(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
127      p.addColumn(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
128      t.put(p);
129      p = new Put(ROW2);
130      p.addColumn(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
131      p.addColumn(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
132      p.addColumn(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
133      t.put(p);
134      String[] args = { sourceTable.getNameAsString(), FQ_OUTPUT_DIR.toString(), ";", "\\x01row1" };
135      runCount(args);
136      FileInputStream inputStream =
137        new FileInputStream(OUTPUT_DIR + File.separator + "part-r-00000");
138      String data = IOUtils.toString(inputStream);
139      inputStream.close();
140      assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
141      assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
142      assertTrue(data.contains("Total ROWS" + "\t" + "1"));
143      assertTrue(data.contains("b;q" + "\t" + "1"));
144      assertTrue(data.contains("a;q" + "\t" + "1"));
145      assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
146      assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
147    } finally {
148      FileUtil.fullyDelete(new File(OUTPUT_DIR));
149    }
150  }
151
152  /**
153   * Test CellCounter with time range all data should print to output
154   */
155  @Test
156  public void testCellCounterStartTimeRange(TestInfo testInfo) throws Exception {
157    final TableName sourceTable = TableName.valueOf(testInfo.getTestMethod().get().getName());
158    byte[][] families = { FAMILY_A, FAMILY_B };
159    try (Table t = UTIL.createTable(sourceTable, families)) {
160      Put p = new Put(ROW1);
161      p.addColumn(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
162      p.addColumn(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
163      p.addColumn(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
164      t.put(p);
165      p = new Put(ROW2);
166      p.addColumn(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
167      p.addColumn(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
168      p.addColumn(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
169      t.put(p);
170      String[] args = { sourceTable.getNameAsString(), FQ_OUTPUT_DIR.toString(), ";", "^row1",
171        "--starttime=" + now, "--endtime=" + now + 2 };
172      runCount(args);
173      FileInputStream inputStream =
174        new FileInputStream(OUTPUT_DIR + File.separator + "part-r-00000");
175      String data = IOUtils.toString(inputStream);
176      inputStream.close();
177      assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
178      assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
179      assertTrue(data.contains("Total ROWS" + "\t" + "1"));
180      assertTrue(data.contains("b;q" + "\t" + "1"));
181      assertTrue(data.contains("a;q" + "\t" + "1"));
182      assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
183      assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
184    } finally {
185      FileUtil.fullyDelete(new File(OUTPUT_DIR));
186    }
187  }
188
189  /**
190   * Test CellCounter with time range all data should print to output
191   */
192  @Test
193  public void testCellCounteEndTimeRange(TestInfo testInfo) throws Exception {
194    final TableName sourceTable = TableName.valueOf(testInfo.getTestMethod().get().getName());
195    byte[][] families = { FAMILY_A, FAMILY_B };
196    try (Table t = UTIL.createTable(sourceTable, families)) {
197      Put p = new Put(ROW1);
198      p.addColumn(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
199      p.addColumn(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
200      p.addColumn(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
201      t.put(p);
202      p = new Put(ROW2);
203      p.addColumn(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
204      p.addColumn(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
205      p.addColumn(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
206      t.put(p);
207      String[] args = { sourceTable.getNameAsString(), FQ_OUTPUT_DIR.toString(), ";", "^row1",
208        "--endtime=" + now + 1 };
209      runCount(args);
210      FileInputStream inputStream =
211        new FileInputStream(OUTPUT_DIR + File.separator + "part-r-00000");
212      String data = IOUtils.toString(inputStream);
213      inputStream.close();
214      assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
215      assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
216      assertTrue(data.contains("Total ROWS" + "\t" + "1"));
217      assertTrue(data.contains("b;q" + "\t" + "1"));
218      assertTrue(data.contains("a;q" + "\t" + "1"));
219      assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
220      assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
221    } finally {
222      FileUtil.fullyDelete(new File(OUTPUT_DIR));
223    }
224  }
225
226  /**
227   * Test CellCounter with time range all data should print to output
228   */
229  @Test
230  public void testCellCounteOutOfTimeRange(TestInfo testInfo) throws Exception {
231    final TableName sourceTable = TableName.valueOf(testInfo.getTestMethod().get().getName());
232    byte[][] families = { FAMILY_A, FAMILY_B };
233    try (Table t = UTIL.createTable(sourceTable, families)) {
234      Put p = new Put(ROW1);
235      p.addColumn(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
236      p.addColumn(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
237      p.addColumn(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
238      t.put(p);
239      p = new Put(ROW2);
240      p.addColumn(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
241      p.addColumn(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
242      p.addColumn(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
243      t.put(p);
244      String[] args = { sourceTable.getNameAsString(), FQ_OUTPUT_DIR.toString(), ";",
245        "--starttime=" + now + 1, "--endtime=" + now + 2 };
246
247      runCount(args);
248      FileInputStream inputStream =
249        new FileInputStream(OUTPUT_DIR + File.separator + "part-r-00000");
250      String data = IOUtils.toString(inputStream);
251      inputStream.close();
252      // nothing should hace been emitted to the reducer
253      assertTrue(data.isEmpty());
254    } finally {
255      FileUtil.fullyDelete(new File(OUTPUT_DIR));
256    }
257  }
258
259  private boolean runCount(String[] args) throws Exception {
260    // need to make a copy of the configuration because to make sure
261    // different temp dirs are used.
262    int status =
263      ToolRunner.run(new Configuration(UTIL.getConfiguration()), new CellCounter(), args);
264    return status == 0;
265  }
266
267  /**
268   * Test main method of CellCounter
269   */
270  @Test
271  public void testCellCounterMain() throws Exception {
272    PrintStream oldPrintStream = System.err;
273    SecurityManager SECURITY_MANAGER = System.getSecurityManager();
274    LauncherSecurityManager newSecurityManager = new LauncherSecurityManager();
275    System.setSecurityManager(newSecurityManager);
276    ByteArrayOutputStream data = new ByteArrayOutputStream();
277    String[] args = {};
278    System.setErr(new PrintStream(data));
279    try {
280      System.setErr(new PrintStream(data));
281
282      try {
283        CellCounter.main(args);
284        fail("should be SecurityException");
285      } catch (SecurityException e) {
286        assertEquals(-1, newSecurityManager.getExitCode());
287        assertTrue(data.toString().contains("ERROR: Wrong number of parameters:"));
288        // should be information about usage
289        assertTrue(data.toString().contains("Usage:"));
290      }
291
292    } finally {
293      System.setErr(oldPrintStream);
294      System.setSecurityManager(SECURITY_MANAGER);
295    }
296  }
297
298  /**
299   * Test CellCounter for complete table all data should print to output
300   */
301  @Test
302  public void testCellCounterForCompleteTable(TestInfo testInfo) throws Exception {
303    final TableName sourceTable = TableName.valueOf(testInfo.getTestMethod().get().getName());
304    String outputPath = OUTPUT_DIR + sourceTable;
305    LocalFileSystem localFileSystem = new LocalFileSystem();
306    Path outputDir = new Path(outputPath).makeQualified(localFileSystem.getUri(),
307      localFileSystem.getWorkingDirectory());
308    byte[][] families = { FAMILY_A, FAMILY_B };
309    Table t = UTIL.createTable(sourceTable, families);
310    try {
311      Put p = new Put(ROW1);
312      p.addColumn(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
313      p.addColumn(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
314      p.addColumn(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
315      t.put(p);
316      p = new Put(ROW2);
317      p.addColumn(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
318      p.addColumn(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
319      p.addColumn(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
320      t.put(p);
321      String[] args = { sourceTable.getNameAsString(), outputDir.toString(), ";" };
322      runCount(args);
323      FileInputStream inputStream =
324        new FileInputStream(outputPath + File.separator + "part-r-00000");
325      String data = IOUtils.toString(inputStream);
326      inputStream.close();
327      assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
328      assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "4"));
329      assertTrue(data.contains("Total ROWS" + "\t" + "2"));
330      assertTrue(data.contains("b;q" + "\t" + "2"));
331      assertTrue(data.contains("a;q" + "\t" + "2"));
332      assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
333      assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
334      assertTrue(data.contains("row2;a;q_Versions" + "\t" + "1"));
335      assertTrue(data.contains("row2;b;q_Versions" + "\t" + "1"));
336
337      FileUtil.fullyDelete(new File(outputPath));
338      args = new String[] { "-D " + TableInputFormat.SCAN_COLUMN_FAMILY + "=a, b",
339        sourceTable.getNameAsString(), outputDir.toString(), ";" };
340      runCount(args);
341      inputStream = new FileInputStream(outputPath + File.separator + "part-r-00000");
342      String data2 = IOUtils.toString(inputStream);
343      inputStream.close();
344      assertEquals(data, data2);
345    } finally {
346      t.close();
347      localFileSystem.close();
348      FileUtil.fullyDelete(new File(outputPath));
349    }
350  }
351
352  @Test
353  public void TestCellCounterWithoutOutputDir() throws Exception {
354    String[] args = new String[] { "tableName" };
355    assertEquals(-1, ToolRunner.run(HBaseConfiguration.create(), new CellCounter(), args),
356      "CellCounter should exit with -1 as output directory is not specified.");
357  }
358}