001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.mapreduce; 019 020import static org.junit.jupiter.api.Assertions.assertEquals; 021import static org.junit.jupiter.api.Assertions.assertTrue; 022import static org.junit.jupiter.api.Assertions.fail; 023 024import java.io.ByteArrayOutputStream; 025import java.io.IOException; 026import java.io.PrintStream; 027import java.util.ArrayList; 028import java.util.Arrays; 029import org.apache.hadoop.conf.Configuration; 030import org.apache.hadoop.hbase.HBaseTestingUtil; 031import org.apache.hadoop.hbase.TableName; 032import org.apache.hadoop.hbase.client.Delete; 033import org.apache.hadoop.hbase.client.Put; 034import org.apache.hadoop.hbase.client.Table; 035import org.apache.hadoop.hbase.testclassification.LargeTests; 036import org.apache.hadoop.hbase.testclassification.MapReduceTests; 037import org.apache.hadoop.hbase.util.Bytes; 038import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 039import org.apache.hadoop.hbase.util.LauncherSecurityManager; 040import org.apache.hadoop.mapreduce.Counter; 041import org.apache.hadoop.mapreduce.Counters; 042import org.apache.hadoop.mapreduce.Job; 043import org.junit.jupiter.api.AfterAll; 044import org.junit.jupiter.api.BeforeAll; 045import org.junit.jupiter.api.Tag; 046import org.junit.jupiter.api.Test; 047import org.slf4j.Logger; 048import org.slf4j.LoggerFactory; 049 050/** 051 * Test the rowcounter map reduce job. 052 */ 053@Tag(MapReduceTests.TAG) 054@Tag(LargeTests.TAG) 055public class TestRowCounter { 056 057 private static final Logger LOG = LoggerFactory.getLogger(TestRowCounter.class); 058 private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); 059 private final static String TABLE_NAME = "testRowCounter"; 060 private final static String TABLE_NAME_TS_RANGE = "testRowCounter_ts_range"; 061 private final static String COL_FAM = "col_fam"; 062 private final static String COL1 = "c1"; 063 private final static String COL2 = "c2"; 064 private final static String COMPOSITE_COLUMN = "C:A:A"; 065 private final static int TOTAL_ROWS = 10; 066 private final static int ROWS_WITH_ONE_COL = 2; 067 068 /** 069 * @throws java.lang.Exception 070 */ 071 @BeforeAll 072 public static void setUpBeforeClass() throws Exception { 073 TEST_UTIL.startMiniCluster(); 074 Table table = TEST_UTIL.createTable(TableName.valueOf(TABLE_NAME), Bytes.toBytes(COL_FAM)); 075 writeRows(table, TOTAL_ROWS, ROWS_WITH_ONE_COL); 076 table.close(); 077 } 078 079 /** 080 * @throws java.lang.Exception 081 */ 082 @AfterAll 083 public static void tearDownAfterClass() throws Exception { 084 TEST_UTIL.shutdownMiniCluster(); 085 } 086 087 /** 088 * Test a case when no column was specified in command line arguments. 089 */ 090 @Test 091 public void testRowCounterNoColumn() throws Exception { 092 String[] args = new String[] { TABLE_NAME }; 093 runRowCount(args, 10); 094 } 095 096 /** 097 * Test a case when the column specified in command line arguments is exclusive for few rows. 098 */ 099 @Test 100 public void testRowCounterExclusiveColumn() throws Exception { 101 String[] args = new String[] { TABLE_NAME, COL_FAM + ":" + COL1 }; 102 runRowCount(args, 8); 103 } 104 105 /** 106 * Test a case when the column specified in command line arguments is one for which the qualifier 107 * contains colons. 108 */ 109 @Test 110 public void testRowCounterColumnWithColonInQualifier() throws Exception { 111 String[] args = new String[] { TABLE_NAME, COL_FAM + ":" + COMPOSITE_COLUMN }; 112 runRowCount(args, 8); 113 } 114 115 /** 116 * Test a case when the column specified in command line arguments is not part of first KV for a 117 * row. 118 */ 119 @Test 120 public void testRowCounterHiddenColumn() throws Exception { 121 String[] args = new String[] { TABLE_NAME, COL_FAM + ":" + COL2 }; 122 runRowCount(args, 10); 123 } 124 125 /** 126 * Test a case when the column specified in command line arguments is exclusive for few rows and 127 * also a row range filter is specified 128 */ 129 @Test 130 public void testRowCounterColumnAndRowRange() throws Exception { 131 String[] args = new String[] { TABLE_NAME, "--range=\\x00rov,\\x00rox", COL_FAM + ":" + COL1 }; 132 runRowCount(args, 8); 133 } 134 135 /** 136 * Test a case when a range is specified with single range of start-end keys 137 */ 138 @Test 139 public void testRowCounterRowSingleRange() throws Exception { 140 String[] args = new String[] { TABLE_NAME, "--range=\\x00row1,\\x00row3" }; 141 runRowCount(args, 2); 142 } 143 144 /** 145 * Test a case when a range is specified with single range with end key only 146 */ 147 @Test 148 public void testRowCounterRowSingleRangeUpperBound() throws Exception { 149 String[] args = new String[] { TABLE_NAME, "--range=,\\x00row3" }; 150 runRowCount(args, 3); 151 } 152 153 /** 154 * Test a case when a range is specified with two ranges where one range is with end key only 155 */ 156 @Test 157 public void testRowCounterRowMultiRangeUpperBound() throws Exception { 158 String[] args = new String[] { TABLE_NAME, "--range=,\\x00row3;\\x00row5,\\x00row7" }; 159 runRowCount(args, 5); 160 } 161 162 /** 163 * Test a case when a range is specified with multiple ranges of start-end keys 164 */ 165 @Test 166 public void testRowCounterRowMultiRange() throws Exception { 167 String[] args = new String[] { TABLE_NAME, "--range=\\x00row1,\\x00row3;\\x00row5,\\x00row8" }; 168 runRowCount(args, 5); 169 } 170 171 /** 172 * Test a case when a range is specified with multiple ranges of start-end keys; one range is 173 * filled, another two are not 174 */ 175 @Test 176 public void testRowCounterRowMultiEmptyRange() throws Exception { 177 String[] args = new String[] { TABLE_NAME, "--range=\\x00row1,\\x00row3;;" }; 178 runRowCount(args, 2); 179 } 180 181 @Test 182 public void testRowCounter10kRowRange() throws Exception { 183 String tableName = TABLE_NAME + "10k"; 184 185 try ( 186 Table table = TEST_UTIL.createTable(TableName.valueOf(tableName), Bytes.toBytes(COL_FAM))) { 187 writeRows(table, 10000, 0); 188 } 189 String[] args = new String[] { tableName, "--range=\\x00row9872,\\x00row9875" }; 190 runRowCount(args, 3); 191 } 192 193 /** 194 * Test a case when the timerange is specified with --starttime and --endtime options 195 */ 196 @Test 197 public void testRowCounterTimeRange() throws Exception { 198 final byte[] family = Bytes.toBytes(COL_FAM); 199 final byte[] col1 = Bytes.toBytes(COL1); 200 Put put1 = new Put(Bytes.toBytes("row_timerange_" + 1)); 201 Put put2 = new Put(Bytes.toBytes("row_timerange_" + 2)); 202 Put put3 = new Put(Bytes.toBytes("row_timerange_" + 3)); 203 204 long ts; 205 206 // clean up content of TABLE_NAME 207 Table table = 208 TEST_UTIL.createTable(TableName.valueOf(TABLE_NAME_TS_RANGE), Bytes.toBytes(COL_FAM)); 209 210 ts = EnvironmentEdgeManager.currentTime(); 211 put1.addColumn(family, col1, ts, Bytes.toBytes("val1")); 212 table.put(put1); 213 Thread.sleep(100); 214 215 ts = EnvironmentEdgeManager.currentTime(); 216 put2.addColumn(family, col1, ts, Bytes.toBytes("val2")); 217 put3.addColumn(family, col1, ts, Bytes.toBytes("val3")); 218 table.put(put2); 219 table.put(put3); 220 table.close(); 221 222 String[] args = new String[] { TABLE_NAME_TS_RANGE, COL_FAM + ":" + COL1, "--starttime=" + 0, 223 "--endtime=" + ts }; 224 runRowCount(args, 1); 225 226 args = new String[] { TABLE_NAME_TS_RANGE, COL_FAM + ":" + COL1, "--starttime=" + 0, 227 "--endtime=" + (ts - 10) }; 228 runRowCount(args, 1); 229 230 args = new String[] { TABLE_NAME_TS_RANGE, COL_FAM + ":" + COL1, "--starttime=" + ts, 231 "--endtime=" + (ts + 1000) }; 232 runRowCount(args, 2); 233 234 args = new String[] { TABLE_NAME_TS_RANGE, COL_FAM + ":" + COL1, 235 "--starttime=" + (ts - 30 * 1000), "--endtime=" + (ts + 30 * 1000), }; 236 runRowCount(args, 3); 237 } 238 239 /** 240 * Run the RowCounter map reduce job and verify the row count. 241 * @param args the command line arguments to be used for rowcounter job. 242 * @param expectedCount the expected row count (result of map reduce job). 243 */ 244 private void runRowCount(String[] args, int expectedCount) throws Exception { 245 RowCounter rowCounter = new RowCounter(); 246 rowCounter.setConf(TEST_UTIL.getConfiguration()); 247 args = Arrays.copyOf(args, args.length + 1); 248 args[args.length - 1] = "--expectedCount=" + expectedCount; 249 long start = EnvironmentEdgeManager.currentTime(); 250 int result = rowCounter.run(args); 251 long duration = EnvironmentEdgeManager.currentTime() - start; 252 LOG.debug("row count duration (ms): " + duration); 253 assertTrue(result == 0); 254 } 255 256 /** 257 * Run the RowCounter map reduce job and verify the row count. 258 * @param args the command line arguments to be used for rowcounter job. 259 * @param expectedCount the expected row count (result of map reduce job). 260 * @throws Exception in case of any unexpected error. 261 */ 262 private void runCreateSubmittableJobWithArgs(String[] args, int expectedCount) throws Exception { 263 Job job = RowCounter.createSubmittableJob(TEST_UTIL.getConfiguration(), args); 264 long start = EnvironmentEdgeManager.currentTime(); 265 job.waitForCompletion(true); 266 long duration = EnvironmentEdgeManager.currentTime() - start; 267 LOG.debug("row count duration (ms): " + duration); 268 assertTrue(job.isSuccessful()); 269 Counter counter = job.getCounters().findCounter(RowCounter.RowCounterMapper.Counters.ROWS); 270 assertEquals(expectedCount, counter.getValue()); 271 } 272 273 @Test 274 public void testCreateSubmittableJobWithArgsNoColumn() throws Exception { 275 String[] args = new String[] { TABLE_NAME }; 276 runCreateSubmittableJobWithArgs(args, 10); 277 } 278 279 /** 280 * Test a case when the column specified in command line arguments is exclusive for few rows. 281 * @throws Exception in case of any unexpected error. 282 */ 283 @Test 284 public void testCreateSubmittableJobWithArgsExclusiveColumn() throws Exception { 285 String[] args = new String[] { TABLE_NAME, COL_FAM + ":" + COL1 }; 286 runCreateSubmittableJobWithArgs(args, 8); 287 } 288 289 /** 290 * Test a case when the column specified in command line arguments is one for which the qualifier 291 * contains colons. 292 * @throws Exception in case of any unexpected error. 293 */ 294 @Test 295 public void testCreateSubmittableJobWithArgsColumnWithColonInQualifier() throws Exception { 296 String[] args = new String[] { TABLE_NAME, COL_FAM + ":" + COMPOSITE_COLUMN }; 297 runCreateSubmittableJobWithArgs(args, 8); 298 } 299 300 /** 301 * Test a case when the column specified in command line arguments is not part of first KV for a 302 * row. 303 * @throws Exception in case of any unexpected error. 304 */ 305 @Test 306 public void testCreateSubmittableJobWithArgsHiddenColumn() throws Exception { 307 String[] args = new String[] { TABLE_NAME, COL_FAM + ":" + COL2 }; 308 runCreateSubmittableJobWithArgs(args, 10); 309 } 310 311 /** 312 * Test a case when the column specified in command line arguments is exclusive for few rows and 313 * also a row range filter is specified 314 * @throws Exception in case of any unexpected error. 315 */ 316 @Test 317 public void testCreateSubmittableJobWithArgsColumnAndRowRange() throws Exception { 318 String[] args = new String[] { TABLE_NAME, "--range=\\x00rov,\\x00rox", COL_FAM + ":" + COL1 }; 319 runCreateSubmittableJobWithArgs(args, 8); 320 } 321 322 /** 323 * Test a case when a range is specified with single range of start-end keys 324 * @throws Exception in case of any unexpected error. 325 */ 326 @Test 327 public void testCreateSubmittableJobWithArgsRowSingleRange() throws Exception { 328 String[] args = new String[] { TABLE_NAME, "--range=\\x00row1,\\x00row3" }; 329 runCreateSubmittableJobWithArgs(args, 2); 330 } 331 332 /** 333 * Test a case when a range is specified with single range with end key only 334 * @throws Exception in case of any unexpected error. 335 */ 336 @Test 337 public void testCreateSubmittableJobWithArgsRowSingleRangeUpperBound() throws Exception { 338 String[] args = new String[] { TABLE_NAME, "--range=,\\x00row3" }; 339 runCreateSubmittableJobWithArgs(args, 3); 340 } 341 342 /** 343 * Test a case when a range is specified with two ranges where one range is with end key only 344 * @throws Exception in case of any unexpected error. 345 */ 346 @Test 347 public void testCreateSubmittableJobWithArgsRowMultiRangeUpperBound() throws Exception { 348 String[] args = new String[] { TABLE_NAME, "--range=,\\x00row3;\\x00row5,\\x00row7" }; 349 runCreateSubmittableJobWithArgs(args, 5); 350 } 351 352 /** 353 * Test a case when a range is specified with multiple ranges of start-end keys 354 * @throws Exception in case of any unexpected error. 355 */ 356 @Test 357 public void testCreateSubmittableJobWithArgsRowMultiRange() throws Exception { 358 String[] args = new String[] { TABLE_NAME, "--range=\\x00row1,\\x00row3;\\x00row5,\\x00row8" }; 359 runCreateSubmittableJobWithArgs(args, 5); 360 } 361 362 /** 363 * Test a case when a range is specified with multiple ranges of start-end keys; one range is 364 * filled, another two are not 365 * @throws Exception in case of any unexpected error. 366 */ 367 @Test 368 public void testCreateSubmittableJobWithArgsRowMultiEmptyRange() throws Exception { 369 String[] args = new String[] { TABLE_NAME, "--range=\\x00row1,\\x00row3;;" }; 370 runCreateSubmittableJobWithArgs(args, 2); 371 } 372 373 @Test 374 public void testCreateSubmittableJobWithArgs10kRowRange() throws Exception { 375 String tableName = TABLE_NAME + "CreateSubmittableJobWithArgs10kRowRange"; 376 377 try ( 378 Table table = TEST_UTIL.createTable(TableName.valueOf(tableName), Bytes.toBytes(COL_FAM))) { 379 writeRows(table, 10000, 0); 380 } 381 String[] args = new String[] { tableName, "--range=\\x00row9872,\\x00row9875" }; 382 runCreateSubmittableJobWithArgs(args, 3); 383 } 384 385 /** 386 * Test a case when the timerange is specified with --starttime and --endtime options 387 * @throws Exception in case of any unexpected error. 388 */ 389 @Test 390 public void testCreateSubmittableJobWithArgsTimeRange() throws Exception { 391 final byte[] family = Bytes.toBytes(COL_FAM); 392 final byte[] col1 = Bytes.toBytes(COL1); 393 Put put1 = new Put(Bytes.toBytes("row_timerange_" + 1)); 394 Put put2 = new Put(Bytes.toBytes("row_timerange_" + 2)); 395 Put put3 = new Put(Bytes.toBytes("row_timerange_" + 3)); 396 397 long ts; 398 399 String tableName = TABLE_NAME_TS_RANGE + "CreateSubmittableJobWithArgs"; 400 // clean up content of TABLE_NAME 401 Table table = TEST_UTIL.createTable(TableName.valueOf(tableName), Bytes.toBytes(COL_FAM)); 402 403 ts = EnvironmentEdgeManager.currentTime(); 404 put1.addColumn(family, col1, ts, Bytes.toBytes("val1")); 405 table.put(put1); 406 Thread.sleep(100); 407 408 ts = EnvironmentEdgeManager.currentTime(); 409 put2.addColumn(family, col1, ts, Bytes.toBytes("val2")); 410 put3.addColumn(family, col1, ts, Bytes.toBytes("val3")); 411 table.put(put2); 412 table.put(put3); 413 table.close(); 414 415 String[] args = 416 new String[] { tableName, COL_FAM + ":" + COL1, "--starttime=" + 0, "--endtime=" + ts }; 417 runCreateSubmittableJobWithArgs(args, 1); 418 419 args = new String[] { tableName, COL_FAM + ":" + COL1, "--starttime=" + 0, 420 "--endtime=" + (ts - 10) }; 421 runCreateSubmittableJobWithArgs(args, 1); 422 423 args = new String[] { tableName, COL_FAM + ":" + COL1, "--starttime=" + ts, 424 "--endtime=" + (ts + 1000) }; 425 runCreateSubmittableJobWithArgs(args, 2); 426 427 args = new String[] { tableName, COL_FAM + ":" + COL1, "--starttime=" + (ts - 30 * 1000), 428 "--endtime=" + (ts + 30 * 1000), }; 429 runCreateSubmittableJobWithArgs(args, 3); 430 } 431 432 /** 433 * Writes TOTAL_ROWS number of distinct rows in to the table. Few rows have two columns, Few have 434 * one. 435 */ 436 private static void writeRows(Table table, int totalRows, int rowsWithOneCol) throws IOException { 437 final byte[] family = Bytes.toBytes(COL_FAM); 438 final byte[] value = Bytes.toBytes("abcd"); 439 final byte[] col1 = Bytes.toBytes(COL1); 440 final byte[] col2 = Bytes.toBytes(COL2); 441 final byte[] col3 = Bytes.toBytes(COMPOSITE_COLUMN); 442 ArrayList<Put> rowsUpdate = new ArrayList<>(); 443 // write few rows with two columns 444 int i = 0; 445 for (; i < totalRows - rowsWithOneCol; i++) { 446 // Use binary rows values to test for HBASE-15287. 447 byte[] row = Bytes.toBytesBinary("\\x00row" + i); 448 Put put = new Put(row); 449 put.addColumn(family, col1, value); 450 put.addColumn(family, col2, value); 451 put.addColumn(family, col3, value); 452 rowsUpdate.add(put); 453 } 454 455 // write few rows with only one column 456 for (; i < totalRows; i++) { 457 byte[] row = Bytes.toBytes("row" + i); 458 Put put = new Put(row); 459 put.addColumn(family, col2, value); 460 rowsUpdate.add(put); 461 } 462 table.put(rowsUpdate); 463 } 464 465 /** 466 * test main method. Import should print help and call System.exit 467 */ 468 @Test 469 public void testImportMain() throws Exception { 470 SecurityManager SECURITY_MANAGER = System.getSecurityManager(); 471 LauncherSecurityManager newSecurityManager = new LauncherSecurityManager(); 472 System.setSecurityManager(newSecurityManager); 473 String[] args = {}; 474 try { 475 try { 476 RowCounter.main(args); 477 fail("should be SecurityException"); 478 } catch (SecurityException e) { 479 assertEquals(RowCounter.EXIT_FAILURE, newSecurityManager.getExitCode()); 480 } 481 try { 482 args = new String[2]; 483 args[0] = "table"; 484 args[1] = "--range=1"; 485 RowCounter.main(args); 486 fail("should be SecurityException"); 487 } catch (SecurityException e) { 488 assertEquals(RowCounter.EXIT_FAILURE, newSecurityManager.getExitCode()); 489 } 490 491 } finally { 492 System.setSecurityManager(SECURITY_MANAGER); 493 } 494 } 495 496 @Test 497 public void testHelp() throws Exception { 498 PrintStream oldPrintStream = System.out; 499 try { 500 ByteArrayOutputStream data = new ByteArrayOutputStream(); 501 PrintStream stream = new PrintStream(data); 502 System.setOut(stream); 503 String[] args = { "-h" }; 504 runRowCount(args, 0); 505 assertUsageContent(data.toString()); 506 args = new String[] { "--help" }; 507 runRowCount(args, 0); 508 assertUsageContent(data.toString()); 509 } finally { 510 System.setOut(oldPrintStream); 511 } 512 } 513 514 @Test 515 public void testInvalidTable() throws Exception { 516 try { 517 String[] args = { "invalid" }; 518 runRowCount(args, 0); 519 fail("RowCounter should had failed with invalid table."); 520 } catch (Throwable e) { 521 assertTrue(e instanceof AssertionError); 522 } 523 } 524 525 /** 526 * Step 1: Add 10 rows(row1, row2, row3, row4, row5, row6, row7, row8, row9, row10) to a table. 527 * Each row contains 1 column family and 4 columns and values for two different timestamps - 5 & 528 * 10. 529 * <p> 530 * Step 2: Delete the latest version of column A for row1. --> 1 X Delete 531 * <p> 532 * Step 3: Delete the cell for timestamp 5 of column B for row1. --> 1 X Delete 533 * <p> 534 * Step 4: Delete a column family for row2 and row4. --> 2 X DeleteFamily 535 * <p> 536 * Step 5: Delete all versions of a specific column for row3, row5 and row6. --> 3 X DeleteColumn 537 * <p> 538 * Step 6: Delete all columns for timestamp 5 for row 7. --> 1 X DeleteFamilyVersion 539 * <p> 540 * Case 1: Run row counter without countDeleteMarkers and validate counter values. 541 * <p> 542 * Case 2: Run row counter with countDeleteMarkers flag and validate counter values. 543 * <p> 544 * Case 3: Run row counter with countDeleteMarkers flag for a row range and validate counter 545 * values. 546 */ 547 @Test 548 public void testRowCounterWithCountDeleteMarkersOption() throws Exception { 549 // Test Setup 550 551 final TableName tableName = 552 TableName.valueOf(TABLE_NAME + "_" + "withCountDeleteMarkersOption"); 553 // Row keys are represented in this way because of HBASE-15287 554 final byte[][] rowKeys = { Bytes.toBytesBinary("\\x00row1"), Bytes.toBytesBinary("\\x00row2"), 555 Bytes.toBytesBinary("\\x00row3"), Bytes.toBytesBinary("\\x00row4"), 556 Bytes.toBytesBinary("\\x00row5"), Bytes.toBytesBinary("\\x00row6"), 557 Bytes.toBytesBinary("\\x00row7"), Bytes.toBytesBinary("\\x00row8"), 558 Bytes.toBytesBinary("\\x00row9"), Bytes.toBytesBinary("\\x00row10") }; 559 final byte[] columnFamily = Bytes.toBytes("cf"); 560 final byte[][] columns = 561 { Bytes.toBytes("A"), Bytes.toBytes("B"), Bytes.toBytes("C"), Bytes.toBytes("D") }; 562 final byte[][] values = { Bytes.toBytes("a"), Bytes.toBytes("b") }; 563 564 try (Table table = TEST_UTIL.createTable(tableName, columnFamily)) { 565 // Step 1: Insert rows with columns 566 for (byte[] rowKey : rowKeys) { 567 Put put = new Put(rowKey); 568 for (byte[] col : columns) { 569 long timestamp = 5L; 570 for (byte[] value : values) { 571 put.addColumn(columnFamily, col, timestamp, value); 572 timestamp += 5L; 573 } 574 } 575 table.put(put); 576 } 577 TEST_UTIL.getAdmin().flush(tableName); 578 579 // Steps 2-6 580 Delete deleteA = new Delete(rowKeys[0]).addColumn(columnFamily, columns[0]); 581 Delete deleteB = new Delete(rowKeys[0]).addColumn(columnFamily, columns[1], 5L); 582 Delete deleteC = new Delete(rowKeys[1]).addFamily(columnFamily); 583 Delete deleteD = new Delete(rowKeys[2]).addColumns(columnFamily, columns[0]); 584 Delete deleteE = new Delete(rowKeys[3]).addFamily(columnFamily); 585 Delete deleteF = new Delete(rowKeys[4]).addColumns(columnFamily, columns[0]); 586 Delete deleteG = new Delete(rowKeys[5]).addColumns(columnFamily, columns[0]); 587 Delete deleteH = new Delete(rowKeys[6]).addFamilyVersion(columnFamily, 5L); 588 589 table.delete(deleteA); 590 table.delete(deleteB); 591 table.delete(deleteC); 592 table.delete(deleteD); 593 table.delete(deleteE); 594 table.delete(deleteF); 595 table.delete(deleteG); 596 table.delete(deleteH); 597 TEST_UTIL.getAdmin().flush(tableName); 598 } 599 600 RowCounter rowCounterWithoutCountDeleteMarkers = new RowCounter(); 601 RowCounter rowCounterWithCountDeleteMarkers = new RowCounter(); 602 RowCounter rowCounterForRangeWithCountDeleteMarkers = new RowCounter(); 603 rowCounterWithoutCountDeleteMarkers.setConf(new Configuration(TEST_UTIL.getConfiguration())); 604 rowCounterWithCountDeleteMarkers.setConf(new Configuration(TEST_UTIL.getConfiguration())); 605 rowCounterForRangeWithCountDeleteMarkers 606 .setConf(new Configuration(TEST_UTIL.getConfiguration())); 607 608 // Invocation 609 610 rowCounterWithoutCountDeleteMarkers.run(new String[] { tableName.getNameAsString() }); 611 rowCounterWithCountDeleteMarkers 612 .run(new String[] { tableName.getNameAsString(), "--countDeleteMarkers" }); 613 rowCounterForRangeWithCountDeleteMarkers.run(new String[] { tableName.getNameAsString(), 614 "--countDeleteMarkers", "--range=\\x00row8,\\x00row9" }); 615 616 // Validation 617 618 // Case 1: 619 validateCounterCounts(rowCounterWithoutCountDeleteMarkers.getMapReduceJob().getCounters(), 8, 0, 620 0, 0, 0, 0); 621 622 // Case 2: 623 validateCounterCounts(rowCounterWithCountDeleteMarkers.getMapReduceJob().getCounters(), 10, 7, 624 2, 3, 2, 1); 625 626 // Case 3: 627 validateCounterCounts(rowCounterForRangeWithCountDeleteMarkers.getMapReduceJob().getCounters(), 628 1, 0, 0, 0, 0, 0); 629 } 630 631 private void validateCounterCounts(Counters counters, long rowCount, 632 long rowsWithDeleteMarkersCount, long deleteCount, long deleteColumnCount, 633 long deleteFamilyCount, long deleteFamilyVersionCount) { 634 635 long actualRowCount = 636 counters.findCounter(RowCounter.RowCounterMapper.Counters.ROWS).getValue(); 637 long actualRowsWithDeleteMarkersCount = 638 counters.findCounter(RowCounter.RowCounterMapper.Counters.ROWS_WITH_DELETE_MARKER).getValue(); 639 long actualDeleteCount = 640 counters.findCounter(RowCounter.RowCounterMapper.Counters.DELETE).getValue(); 641 long actualDeleteColumnCount = 642 counters.findCounter(RowCounter.RowCounterMapper.Counters.DELETE_COLUMN).getValue(); 643 long actualDeleteFamilyCount = 644 counters.findCounter(RowCounter.RowCounterMapper.Counters.DELETE_FAMILY).getValue(); 645 long actualDeleteFamilyVersionCount = 646 counters.findCounter(RowCounter.RowCounterMapper.Counters.DELETE_FAMILY_VERSION).getValue(); 647 648 assertEquals(rowCount, actualRowCount); 649 assertEquals(rowsWithDeleteMarkersCount, actualRowsWithDeleteMarkersCount); 650 assertEquals(deleteCount, actualDeleteCount); 651 assertEquals(deleteColumnCount, actualDeleteColumnCount); 652 assertEquals(deleteFamilyCount, actualDeleteFamilyCount); 653 assertEquals(deleteFamilyVersionCount, actualDeleteFamilyVersionCount); 654 } 655 656 private void assertUsageContent(String usage) { 657 assertTrue(usage 658 .contains("usage: hbase rowcounter " + "<tablename> [options] [<column1> <column2>...]")); 659 assertTrue(usage.contains("Options:\n")); 660 assertTrue(usage.contains( 661 "--starttime=<arg> " + "starting time filter to start counting rows from.\n")); 662 assertTrue(usage.contains("--endtime=<arg> " 663 + "end time filter limit, to only count rows up to this timestamp.\n")); 664 assertTrue(usage 665 .contains("--range=<arg> " + "[startKey],[endKey][;[startKey],[endKey]...]]\n")); 666 assertTrue(usage.contains("--expectedCount=<arg> expected number of rows to be count.\n")); 667 assertTrue( 668 usage.contains("For performance, " + "consider the following configuration properties:\n")); 669 assertTrue(usage.contains("-Dhbase.client.scanner.caching=100\n")); 670 assertTrue(usage.contains("-Dmapreduce.map.speculative=false\n")); 671 } 672 673}