001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.mapreduce;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertTrue;
022import static org.junit.jupiter.api.Assertions.fail;
023
024import java.io.ByteArrayOutputStream;
025import java.io.IOException;
026import java.io.PrintStream;
027import java.util.ArrayList;
028import java.util.Arrays;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.hbase.HBaseTestingUtil;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.client.Delete;
033import org.apache.hadoop.hbase.client.Put;
034import org.apache.hadoop.hbase.client.Table;
035import org.apache.hadoop.hbase.testclassification.LargeTests;
036import org.apache.hadoop.hbase.testclassification.MapReduceTests;
037import org.apache.hadoop.hbase.util.Bytes;
038import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
039import org.apache.hadoop.hbase.util.LauncherSecurityManager;
040import org.apache.hadoop.mapreduce.Counter;
041import org.apache.hadoop.mapreduce.Counters;
042import org.apache.hadoop.mapreduce.Job;
043import org.junit.jupiter.api.AfterAll;
044import org.junit.jupiter.api.BeforeAll;
045import org.junit.jupiter.api.Tag;
046import org.junit.jupiter.api.Test;
047import org.slf4j.Logger;
048import org.slf4j.LoggerFactory;
049
050/**
051 * Test the rowcounter map reduce job.
052 */
053@Tag(MapReduceTests.TAG)
054@Tag(LargeTests.TAG)
055public class TestRowCounter {
056
057  private static final Logger LOG = LoggerFactory.getLogger(TestRowCounter.class);
058  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
059  private final static String TABLE_NAME = "testRowCounter";
060  private final static String TABLE_NAME_TS_RANGE = "testRowCounter_ts_range";
061  private final static String COL_FAM = "col_fam";
062  private final static String COL1 = "c1";
063  private final static String COL2 = "c2";
064  private final static String COMPOSITE_COLUMN = "C:A:A";
065  private final static int TOTAL_ROWS = 10;
066  private final static int ROWS_WITH_ONE_COL = 2;
067
068  /**
069   * @throws java.lang.Exception
070   */
071  @BeforeAll
072  public static void setUpBeforeClass() throws Exception {
073    TEST_UTIL.startMiniCluster();
074    Table table = TEST_UTIL.createTable(TableName.valueOf(TABLE_NAME), Bytes.toBytes(COL_FAM));
075    writeRows(table, TOTAL_ROWS, ROWS_WITH_ONE_COL);
076    table.close();
077  }
078
079  /**
080   * @throws java.lang.Exception
081   */
082  @AfterAll
083  public static void tearDownAfterClass() throws Exception {
084    TEST_UTIL.shutdownMiniCluster();
085  }
086
087  /**
088   * Test a case when no column was specified in command line arguments.
089   */
090  @Test
091  public void testRowCounterNoColumn() throws Exception {
092    String[] args = new String[] { TABLE_NAME };
093    runRowCount(args, 10);
094  }
095
096  /**
097   * Test a case when the column specified in command line arguments is exclusive for few rows.
098   */
099  @Test
100  public void testRowCounterExclusiveColumn() throws Exception {
101    String[] args = new String[] { TABLE_NAME, COL_FAM + ":" + COL1 };
102    runRowCount(args, 8);
103  }
104
105  /**
106   * Test a case when the column specified in command line arguments is one for which the qualifier
107   * contains colons.
108   */
109  @Test
110  public void testRowCounterColumnWithColonInQualifier() throws Exception {
111    String[] args = new String[] { TABLE_NAME, COL_FAM + ":" + COMPOSITE_COLUMN };
112    runRowCount(args, 8);
113  }
114
115  /**
116   * Test a case when the column specified in command line arguments is not part of first KV for a
117   * row.
118   */
119  @Test
120  public void testRowCounterHiddenColumn() throws Exception {
121    String[] args = new String[] { TABLE_NAME, COL_FAM + ":" + COL2 };
122    runRowCount(args, 10);
123  }
124
125  /**
126   * Test a case when the column specified in command line arguments is exclusive for few rows and
127   * also a row range filter is specified
128   */
129  @Test
130  public void testRowCounterColumnAndRowRange() throws Exception {
131    String[] args = new String[] { TABLE_NAME, "--range=\\x00rov,\\x00rox", COL_FAM + ":" + COL1 };
132    runRowCount(args, 8);
133  }
134
135  /**
136   * Test a case when a range is specified with single range of start-end keys
137   */
138  @Test
139  public void testRowCounterRowSingleRange() throws Exception {
140    String[] args = new String[] { TABLE_NAME, "--range=\\x00row1,\\x00row3" };
141    runRowCount(args, 2);
142  }
143
144  /**
145   * Test a case when a range is specified with single range with end key only
146   */
147  @Test
148  public void testRowCounterRowSingleRangeUpperBound() throws Exception {
149    String[] args = new String[] { TABLE_NAME, "--range=,\\x00row3" };
150    runRowCount(args, 3);
151  }
152
153  /**
154   * Test a case when a range is specified with two ranges where one range is with end key only
155   */
156  @Test
157  public void testRowCounterRowMultiRangeUpperBound() throws Exception {
158    String[] args = new String[] { TABLE_NAME, "--range=,\\x00row3;\\x00row5,\\x00row7" };
159    runRowCount(args, 5);
160  }
161
162  /**
163   * Test a case when a range is specified with multiple ranges of start-end keys
164   */
165  @Test
166  public void testRowCounterRowMultiRange() throws Exception {
167    String[] args = new String[] { TABLE_NAME, "--range=\\x00row1,\\x00row3;\\x00row5,\\x00row8" };
168    runRowCount(args, 5);
169  }
170
171  /**
172   * Test a case when a range is specified with multiple ranges of start-end keys; one range is
173   * filled, another two are not
174   */
175  @Test
176  public void testRowCounterRowMultiEmptyRange() throws Exception {
177    String[] args = new String[] { TABLE_NAME, "--range=\\x00row1,\\x00row3;;" };
178    runRowCount(args, 2);
179  }
180
181  @Test
182  public void testRowCounter10kRowRange() throws Exception {
183    String tableName = TABLE_NAME + "10k";
184
185    try (
186      Table table = TEST_UTIL.createTable(TableName.valueOf(tableName), Bytes.toBytes(COL_FAM))) {
187      writeRows(table, 10000, 0);
188    }
189    String[] args = new String[] { tableName, "--range=\\x00row9872,\\x00row9875" };
190    runRowCount(args, 3);
191  }
192
193  /**
194   * Test a case when the timerange is specified with --starttime and --endtime options
195   */
196  @Test
197  public void testRowCounterTimeRange() throws Exception {
198    final byte[] family = Bytes.toBytes(COL_FAM);
199    final byte[] col1 = Bytes.toBytes(COL1);
200    Put put1 = new Put(Bytes.toBytes("row_timerange_" + 1));
201    Put put2 = new Put(Bytes.toBytes("row_timerange_" + 2));
202    Put put3 = new Put(Bytes.toBytes("row_timerange_" + 3));
203
204    long ts;
205
206    // clean up content of TABLE_NAME
207    Table table =
208      TEST_UTIL.createTable(TableName.valueOf(TABLE_NAME_TS_RANGE), Bytes.toBytes(COL_FAM));
209
210    ts = EnvironmentEdgeManager.currentTime();
211    put1.addColumn(family, col1, ts, Bytes.toBytes("val1"));
212    table.put(put1);
213    Thread.sleep(100);
214
215    ts = EnvironmentEdgeManager.currentTime();
216    put2.addColumn(family, col1, ts, Bytes.toBytes("val2"));
217    put3.addColumn(family, col1, ts, Bytes.toBytes("val3"));
218    table.put(put2);
219    table.put(put3);
220    table.close();
221
222    String[] args = new String[] { TABLE_NAME_TS_RANGE, COL_FAM + ":" + COL1, "--starttime=" + 0,
223      "--endtime=" + ts };
224    runRowCount(args, 1);
225
226    args = new String[] { TABLE_NAME_TS_RANGE, COL_FAM + ":" + COL1, "--starttime=" + 0,
227      "--endtime=" + (ts - 10) };
228    runRowCount(args, 1);
229
230    args = new String[] { TABLE_NAME_TS_RANGE, COL_FAM + ":" + COL1, "--starttime=" + ts,
231      "--endtime=" + (ts + 1000) };
232    runRowCount(args, 2);
233
234    args = new String[] { TABLE_NAME_TS_RANGE, COL_FAM + ":" + COL1,
235      "--starttime=" + (ts - 30 * 1000), "--endtime=" + (ts + 30 * 1000), };
236    runRowCount(args, 3);
237  }
238
239  /**
240   * Run the RowCounter map reduce job and verify the row count.
241   * @param args          the command line arguments to be used for rowcounter job.
242   * @param expectedCount the expected row count (result of map reduce job).
243   */
244  private void runRowCount(String[] args, int expectedCount) throws Exception {
245    RowCounter rowCounter = new RowCounter();
246    rowCounter.setConf(TEST_UTIL.getConfiguration());
247    args = Arrays.copyOf(args, args.length + 1);
248    args[args.length - 1] = "--expectedCount=" + expectedCount;
249    long start = EnvironmentEdgeManager.currentTime();
250    int result = rowCounter.run(args);
251    long duration = EnvironmentEdgeManager.currentTime() - start;
252    LOG.debug("row count duration (ms): " + duration);
253    assertTrue(result == 0);
254  }
255
256  /**
257   * Run the RowCounter map reduce job and verify the row count.
258   * @param args          the command line arguments to be used for rowcounter job.
259   * @param expectedCount the expected row count (result of map reduce job).
260   * @throws Exception in case of any unexpected error.
261   */
262  private void runCreateSubmittableJobWithArgs(String[] args, int expectedCount) throws Exception {
263    Job job = RowCounter.createSubmittableJob(TEST_UTIL.getConfiguration(), args);
264    long start = EnvironmentEdgeManager.currentTime();
265    job.waitForCompletion(true);
266    long duration = EnvironmentEdgeManager.currentTime() - start;
267    LOG.debug("row count duration (ms): " + duration);
268    assertTrue(job.isSuccessful());
269    Counter counter = job.getCounters().findCounter(RowCounter.RowCounterMapper.Counters.ROWS);
270    assertEquals(expectedCount, counter.getValue());
271  }
272
273  @Test
274  public void testCreateSubmittableJobWithArgsNoColumn() throws Exception {
275    String[] args = new String[] { TABLE_NAME };
276    runCreateSubmittableJobWithArgs(args, 10);
277  }
278
279  /**
280   * Test a case when the column specified in command line arguments is exclusive for few rows.
281   * @throws Exception in case of any unexpected error.
282   */
283  @Test
284  public void testCreateSubmittableJobWithArgsExclusiveColumn() throws Exception {
285    String[] args = new String[] { TABLE_NAME, COL_FAM + ":" + COL1 };
286    runCreateSubmittableJobWithArgs(args, 8);
287  }
288
289  /**
290   * Test a case when the column specified in command line arguments is one for which the qualifier
291   * contains colons.
292   * @throws Exception in case of any unexpected error.
293   */
294  @Test
295  public void testCreateSubmittableJobWithArgsColumnWithColonInQualifier() throws Exception {
296    String[] args = new String[] { TABLE_NAME, COL_FAM + ":" + COMPOSITE_COLUMN };
297    runCreateSubmittableJobWithArgs(args, 8);
298  }
299
300  /**
301   * Test a case when the column specified in command line arguments is not part of first KV for a
302   * row.
303   * @throws Exception in case of any unexpected error.
304   */
305  @Test
306  public void testCreateSubmittableJobWithArgsHiddenColumn() throws Exception {
307    String[] args = new String[] { TABLE_NAME, COL_FAM + ":" + COL2 };
308    runCreateSubmittableJobWithArgs(args, 10);
309  }
310
311  /**
312   * Test a case when the column specified in command line arguments is exclusive for few rows and
313   * also a row range filter is specified
314   * @throws Exception in case of any unexpected error.
315   */
316  @Test
317  public void testCreateSubmittableJobWithArgsColumnAndRowRange() throws Exception {
318    String[] args = new String[] { TABLE_NAME, "--range=\\x00rov,\\x00rox", COL_FAM + ":" + COL1 };
319    runCreateSubmittableJobWithArgs(args, 8);
320  }
321
322  /**
323   * Test a case when a range is specified with single range of start-end keys
324   * @throws Exception in case of any unexpected error.
325   */
326  @Test
327  public void testCreateSubmittableJobWithArgsRowSingleRange() throws Exception {
328    String[] args = new String[] { TABLE_NAME, "--range=\\x00row1,\\x00row3" };
329    runCreateSubmittableJobWithArgs(args, 2);
330  }
331
332  /**
333   * Test a case when a range is specified with single range with end key only
334   * @throws Exception in case of any unexpected error.
335   */
336  @Test
337  public void testCreateSubmittableJobWithArgsRowSingleRangeUpperBound() throws Exception {
338    String[] args = new String[] { TABLE_NAME, "--range=,\\x00row3" };
339    runCreateSubmittableJobWithArgs(args, 3);
340  }
341
342  /**
343   * Test a case when a range is specified with two ranges where one range is with end key only
344   * @throws Exception in case of any unexpected error.
345   */
346  @Test
347  public void testCreateSubmittableJobWithArgsRowMultiRangeUpperBound() throws Exception {
348    String[] args = new String[] { TABLE_NAME, "--range=,\\x00row3;\\x00row5,\\x00row7" };
349    runCreateSubmittableJobWithArgs(args, 5);
350  }
351
352  /**
353   * Test a case when a range is specified with multiple ranges of start-end keys
354   * @throws Exception in case of any unexpected error.
355   */
356  @Test
357  public void testCreateSubmittableJobWithArgsRowMultiRange() throws Exception {
358    String[] args = new String[] { TABLE_NAME, "--range=\\x00row1,\\x00row3;\\x00row5,\\x00row8" };
359    runCreateSubmittableJobWithArgs(args, 5);
360  }
361
362  /**
363   * Test a case when a range is specified with multiple ranges of start-end keys; one range is
364   * filled, another two are not
365   * @throws Exception in case of any unexpected error.
366   */
367  @Test
368  public void testCreateSubmittableJobWithArgsRowMultiEmptyRange() throws Exception {
369    String[] args = new String[] { TABLE_NAME, "--range=\\x00row1,\\x00row3;;" };
370    runCreateSubmittableJobWithArgs(args, 2);
371  }
372
373  @Test
374  public void testCreateSubmittableJobWithArgs10kRowRange() throws Exception {
375    String tableName = TABLE_NAME + "CreateSubmittableJobWithArgs10kRowRange";
376
377    try (
378      Table table = TEST_UTIL.createTable(TableName.valueOf(tableName), Bytes.toBytes(COL_FAM))) {
379      writeRows(table, 10000, 0);
380    }
381    String[] args = new String[] { tableName, "--range=\\x00row9872,\\x00row9875" };
382    runCreateSubmittableJobWithArgs(args, 3);
383  }
384
385  /**
386   * Test a case when the timerange is specified with --starttime and --endtime options
387   * @throws Exception in case of any unexpected error.
388   */
389  @Test
390  public void testCreateSubmittableJobWithArgsTimeRange() throws Exception {
391    final byte[] family = Bytes.toBytes(COL_FAM);
392    final byte[] col1 = Bytes.toBytes(COL1);
393    Put put1 = new Put(Bytes.toBytes("row_timerange_" + 1));
394    Put put2 = new Put(Bytes.toBytes("row_timerange_" + 2));
395    Put put3 = new Put(Bytes.toBytes("row_timerange_" + 3));
396
397    long ts;
398
399    String tableName = TABLE_NAME_TS_RANGE + "CreateSubmittableJobWithArgs";
400    // clean up content of TABLE_NAME
401    Table table = TEST_UTIL.createTable(TableName.valueOf(tableName), Bytes.toBytes(COL_FAM));
402
403    ts = EnvironmentEdgeManager.currentTime();
404    put1.addColumn(family, col1, ts, Bytes.toBytes("val1"));
405    table.put(put1);
406    Thread.sleep(100);
407
408    ts = EnvironmentEdgeManager.currentTime();
409    put2.addColumn(family, col1, ts, Bytes.toBytes("val2"));
410    put3.addColumn(family, col1, ts, Bytes.toBytes("val3"));
411    table.put(put2);
412    table.put(put3);
413    table.close();
414
415    String[] args =
416      new String[] { tableName, COL_FAM + ":" + COL1, "--starttime=" + 0, "--endtime=" + ts };
417    runCreateSubmittableJobWithArgs(args, 1);
418
419    args = new String[] { tableName, COL_FAM + ":" + COL1, "--starttime=" + 0,
420      "--endtime=" + (ts - 10) };
421    runCreateSubmittableJobWithArgs(args, 1);
422
423    args = new String[] { tableName, COL_FAM + ":" + COL1, "--starttime=" + ts,
424      "--endtime=" + (ts + 1000) };
425    runCreateSubmittableJobWithArgs(args, 2);
426
427    args = new String[] { tableName, COL_FAM + ":" + COL1, "--starttime=" + (ts - 30 * 1000),
428      "--endtime=" + (ts + 30 * 1000), };
429    runCreateSubmittableJobWithArgs(args, 3);
430  }
431
432  /**
433   * Writes TOTAL_ROWS number of distinct rows in to the table. Few rows have two columns, Few have
434   * one.
435   */
436  private static void writeRows(Table table, int totalRows, int rowsWithOneCol) throws IOException {
437    final byte[] family = Bytes.toBytes(COL_FAM);
438    final byte[] value = Bytes.toBytes("abcd");
439    final byte[] col1 = Bytes.toBytes(COL1);
440    final byte[] col2 = Bytes.toBytes(COL2);
441    final byte[] col3 = Bytes.toBytes(COMPOSITE_COLUMN);
442    ArrayList<Put> rowsUpdate = new ArrayList<>();
443    // write few rows with two columns
444    int i = 0;
445    for (; i < totalRows - rowsWithOneCol; i++) {
446      // Use binary rows values to test for HBASE-15287.
447      byte[] row = Bytes.toBytesBinary("\\x00row" + i);
448      Put put = new Put(row);
449      put.addColumn(family, col1, value);
450      put.addColumn(family, col2, value);
451      put.addColumn(family, col3, value);
452      rowsUpdate.add(put);
453    }
454
455    // write few rows with only one column
456    for (; i < totalRows; i++) {
457      byte[] row = Bytes.toBytes("row" + i);
458      Put put = new Put(row);
459      put.addColumn(family, col2, value);
460      rowsUpdate.add(put);
461    }
462    table.put(rowsUpdate);
463  }
464
465  /**
466   * test main method. Import should print help and call System.exit
467   */
468  @Test
469  public void testImportMain() throws Exception {
470    SecurityManager SECURITY_MANAGER = System.getSecurityManager();
471    LauncherSecurityManager newSecurityManager = new LauncherSecurityManager();
472    System.setSecurityManager(newSecurityManager);
473    String[] args = {};
474    try {
475      try {
476        RowCounter.main(args);
477        fail("should be SecurityException");
478      } catch (SecurityException e) {
479        assertEquals(RowCounter.EXIT_FAILURE, newSecurityManager.getExitCode());
480      }
481      try {
482        args = new String[2];
483        args[0] = "table";
484        args[1] = "--range=1";
485        RowCounter.main(args);
486        fail("should be SecurityException");
487      } catch (SecurityException e) {
488        assertEquals(RowCounter.EXIT_FAILURE, newSecurityManager.getExitCode());
489      }
490
491    } finally {
492      System.setSecurityManager(SECURITY_MANAGER);
493    }
494  }
495
496  @Test
497  public void testHelp() throws Exception {
498    PrintStream oldPrintStream = System.out;
499    try {
500      ByteArrayOutputStream data = new ByteArrayOutputStream();
501      PrintStream stream = new PrintStream(data);
502      System.setOut(stream);
503      String[] args = { "-h" };
504      runRowCount(args, 0);
505      assertUsageContent(data.toString());
506      args = new String[] { "--help" };
507      runRowCount(args, 0);
508      assertUsageContent(data.toString());
509    } finally {
510      System.setOut(oldPrintStream);
511    }
512  }
513
514  @Test
515  public void testInvalidTable() throws Exception {
516    try {
517      String[] args = { "invalid" };
518      runRowCount(args, 0);
519      fail("RowCounter should had failed with invalid table.");
520    } catch (Throwable e) {
521      assertTrue(e instanceof AssertionError);
522    }
523  }
524
525  /**
526   * Step 1: Add 10 rows(row1, row2, row3, row4, row5, row6, row7, row8, row9, row10) to a table.
527   * Each row contains 1 column family and 4 columns and values for two different timestamps - 5 &
528   * 10.
529   * <p>
530   * Step 2: Delete the latest version of column A for row1. --> 1 X Delete
531   * <p>
532   * Step 3: Delete the cell for timestamp 5 of column B for row1. --> 1 X Delete
533   * <p>
534   * Step 4: Delete a column family for row2 and row4. --> 2 X DeleteFamily
535   * <p>
536   * Step 5: Delete all versions of a specific column for row3, row5 and row6. --> 3 X DeleteColumn
537   * <p>
538   * Step 6: Delete all columns for timestamp 5 for row 7. --> 1 X DeleteFamilyVersion
539   * <p>
540   * Case 1: Run row counter without countDeleteMarkers and validate counter values.
541   * <p>
542   * Case 2: Run row counter with countDeleteMarkers flag and validate counter values.
543   * <p>
544   * Case 3: Run row counter with countDeleteMarkers flag for a row range and validate counter
545   * values.
546   */
547  @Test
548  public void testRowCounterWithCountDeleteMarkersOption() throws Exception {
549    // Test Setup
550
551    final TableName tableName =
552      TableName.valueOf(TABLE_NAME + "_" + "withCountDeleteMarkersOption");
553    // Row keys are represented in this way because of HBASE-15287
554    final byte[][] rowKeys = { Bytes.toBytesBinary("\\x00row1"), Bytes.toBytesBinary("\\x00row2"),
555      Bytes.toBytesBinary("\\x00row3"), Bytes.toBytesBinary("\\x00row4"),
556      Bytes.toBytesBinary("\\x00row5"), Bytes.toBytesBinary("\\x00row6"),
557      Bytes.toBytesBinary("\\x00row7"), Bytes.toBytesBinary("\\x00row8"),
558      Bytes.toBytesBinary("\\x00row9"), Bytes.toBytesBinary("\\x00row10") };
559    final byte[] columnFamily = Bytes.toBytes("cf");
560    final byte[][] columns =
561      { Bytes.toBytes("A"), Bytes.toBytes("B"), Bytes.toBytes("C"), Bytes.toBytes("D") };
562    final byte[][] values = { Bytes.toBytes("a"), Bytes.toBytes("b") };
563
564    try (Table table = TEST_UTIL.createTable(tableName, columnFamily)) {
565      // Step 1: Insert rows with columns
566      for (byte[] rowKey : rowKeys) {
567        Put put = new Put(rowKey);
568        for (byte[] col : columns) {
569          long timestamp = 5L;
570          for (byte[] value : values) {
571            put.addColumn(columnFamily, col, timestamp, value);
572            timestamp += 5L;
573          }
574        }
575        table.put(put);
576      }
577      TEST_UTIL.getAdmin().flush(tableName);
578
579      // Steps 2-6
580      Delete deleteA = new Delete(rowKeys[0]).addColumn(columnFamily, columns[0]);
581      Delete deleteB = new Delete(rowKeys[0]).addColumn(columnFamily, columns[1], 5L);
582      Delete deleteC = new Delete(rowKeys[1]).addFamily(columnFamily);
583      Delete deleteD = new Delete(rowKeys[2]).addColumns(columnFamily, columns[0]);
584      Delete deleteE = new Delete(rowKeys[3]).addFamily(columnFamily);
585      Delete deleteF = new Delete(rowKeys[4]).addColumns(columnFamily, columns[0]);
586      Delete deleteG = new Delete(rowKeys[5]).addColumns(columnFamily, columns[0]);
587      Delete deleteH = new Delete(rowKeys[6]).addFamilyVersion(columnFamily, 5L);
588
589      table.delete(deleteA);
590      table.delete(deleteB);
591      table.delete(deleteC);
592      table.delete(deleteD);
593      table.delete(deleteE);
594      table.delete(deleteF);
595      table.delete(deleteG);
596      table.delete(deleteH);
597      TEST_UTIL.getAdmin().flush(tableName);
598    }
599
600    RowCounter rowCounterWithoutCountDeleteMarkers = new RowCounter();
601    RowCounter rowCounterWithCountDeleteMarkers = new RowCounter();
602    RowCounter rowCounterForRangeWithCountDeleteMarkers = new RowCounter();
603    rowCounterWithoutCountDeleteMarkers.setConf(new Configuration(TEST_UTIL.getConfiguration()));
604    rowCounterWithCountDeleteMarkers.setConf(new Configuration(TEST_UTIL.getConfiguration()));
605    rowCounterForRangeWithCountDeleteMarkers
606      .setConf(new Configuration(TEST_UTIL.getConfiguration()));
607
608    // Invocation
609
610    rowCounterWithoutCountDeleteMarkers.run(new String[] { tableName.getNameAsString() });
611    rowCounterWithCountDeleteMarkers
612      .run(new String[] { tableName.getNameAsString(), "--countDeleteMarkers" });
613    rowCounterForRangeWithCountDeleteMarkers.run(new String[] { tableName.getNameAsString(),
614      "--countDeleteMarkers", "--range=\\x00row8,\\x00row9" });
615
616    // Validation
617
618    // Case 1:
619    validateCounterCounts(rowCounterWithoutCountDeleteMarkers.getMapReduceJob().getCounters(), 8, 0,
620      0, 0, 0, 0);
621
622    // Case 2:
623    validateCounterCounts(rowCounterWithCountDeleteMarkers.getMapReduceJob().getCounters(), 10, 7,
624      2, 3, 2, 1);
625
626    // Case 3:
627    validateCounterCounts(rowCounterForRangeWithCountDeleteMarkers.getMapReduceJob().getCounters(),
628      1, 0, 0, 0, 0, 0);
629  }
630
631  private void validateCounterCounts(Counters counters, long rowCount,
632    long rowsWithDeleteMarkersCount, long deleteCount, long deleteColumnCount,
633    long deleteFamilyCount, long deleteFamilyVersionCount) {
634
635    long actualRowCount =
636      counters.findCounter(RowCounter.RowCounterMapper.Counters.ROWS).getValue();
637    long actualRowsWithDeleteMarkersCount =
638      counters.findCounter(RowCounter.RowCounterMapper.Counters.ROWS_WITH_DELETE_MARKER).getValue();
639    long actualDeleteCount =
640      counters.findCounter(RowCounter.RowCounterMapper.Counters.DELETE).getValue();
641    long actualDeleteColumnCount =
642      counters.findCounter(RowCounter.RowCounterMapper.Counters.DELETE_COLUMN).getValue();
643    long actualDeleteFamilyCount =
644      counters.findCounter(RowCounter.RowCounterMapper.Counters.DELETE_FAMILY).getValue();
645    long actualDeleteFamilyVersionCount =
646      counters.findCounter(RowCounter.RowCounterMapper.Counters.DELETE_FAMILY_VERSION).getValue();
647
648    assertEquals(rowCount, actualRowCount);
649    assertEquals(rowsWithDeleteMarkersCount, actualRowsWithDeleteMarkersCount);
650    assertEquals(deleteCount, actualDeleteCount);
651    assertEquals(deleteColumnCount, actualDeleteColumnCount);
652    assertEquals(deleteFamilyCount, actualDeleteFamilyCount);
653    assertEquals(deleteFamilyVersionCount, actualDeleteFamilyVersionCount);
654  }
655
656  private void assertUsageContent(String usage) {
657    assertTrue(usage
658      .contains("usage: hbase rowcounter " + "<tablename> [options] [<column1> <column2>...]"));
659    assertTrue(usage.contains("Options:\n"));
660    assertTrue(usage.contains(
661      "--starttime=<arg>       " + "starting time filter to start counting rows from.\n"));
662    assertTrue(usage.contains("--endtime=<arg>         "
663      + "end time filter limit, to only count rows up to this timestamp.\n"));
664    assertTrue(usage
665      .contains("--range=<arg>           " + "[startKey],[endKey][;[startKey],[endKey]...]]\n"));
666    assertTrue(usage.contains("--expectedCount=<arg>   expected number of rows to be count.\n"));
667    assertTrue(
668      usage.contains("For performance, " + "consider the following configuration properties:\n"));
669    assertTrue(usage.contains("-Dhbase.client.scanner.caching=100\n"));
670    assertTrue(usage.contains("-Dmapreduce.map.speculative=false\n"));
671  }
672
673}