001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.mapreduce;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022import static org.junit.Assert.fail;
023
024import java.io.ByteArrayOutputStream;
025import java.io.File;
026import java.io.FileInputStream;
027import java.io.PrintStream;
028import org.apache.commons.io.IOUtils;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.FileUtil;
031import org.apache.hadoop.fs.LocalFileSystem;
032import org.apache.hadoop.fs.Path;
033import org.apache.hadoop.hbase.HBaseClassTestRule;
034import org.apache.hadoop.hbase.HBaseConfiguration;
035import org.apache.hadoop.hbase.HBaseTestingUtility;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.client.Put;
038import org.apache.hadoop.hbase.client.Table;
039import org.apache.hadoop.hbase.testclassification.LargeTests;
040import org.apache.hadoop.hbase.testclassification.MapReduceTests;
041import org.apache.hadoop.hbase.util.Bytes;
042import org.apache.hadoop.hbase.util.LauncherSecurityManager;
043import org.apache.hadoop.util.ToolRunner;
044import org.junit.AfterClass;
045import org.junit.BeforeClass;
046import org.junit.ClassRule;
047import org.junit.Rule;
048import org.junit.Test;
049import org.junit.experimental.categories.Category;
050import org.junit.rules.TestName;
051
052@Category({MapReduceTests.class, LargeTests.class})
053public class TestCellCounter {
054  @ClassRule
055  public static final HBaseClassTestRule CLASS_RULE =
056      HBaseClassTestRule.forClass(TestCellCounter.class);
057
058  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
059  private static final byte[] ROW1 = Bytes.toBytesBinary("\\x01row1");
060  private static final byte[] ROW2 = Bytes.toBytesBinary("\\x01row2");
061  private static final String FAMILY_A_STRING = "a";
062  private static final String FAMILY_B_STRING = "b";
063  private static final byte[] FAMILY_A = Bytes.toBytes(FAMILY_A_STRING);
064  private static final byte[] FAMILY_B = Bytes.toBytes(FAMILY_B_STRING);
065  private static final byte[] QUALIFIER = Bytes.toBytes("q");
066
067  private static Path FQ_OUTPUT_DIR;
068  private static final String OUTPUT_DIR = "target" + File.separator + "test-data" + File.separator
069      + "output";
070  private static long now = System.currentTimeMillis();
071
072  @Rule
073  public TestName name = new TestName();
074
075  @BeforeClass
076  public static void beforeClass() throws Exception {
077    UTIL.startMiniCluster();
078    FQ_OUTPUT_DIR = new Path(OUTPUT_DIR).makeQualified(new LocalFileSystem());
079    FileUtil.fullyDelete(new File(OUTPUT_DIR));
080  }
081
082  @AfterClass
083  public static void afterClass() throws Exception {
084    UTIL.shutdownMiniCluster();
085  }
086
087  /**
088   * Test CellCounter all data should print to output
089   *
090   */
091  @Test
092  public void testCellCounter() throws Exception {
093    final TableName sourceTable = TableName.valueOf(name.getMethodName());
094    byte[][] families = { FAMILY_A, FAMILY_B };
095    try (Table t = UTIL.createTable(sourceTable, families)) {
096      Put p = new Put(ROW1);
097      p.addColumn(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
098      p.addColumn(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
099      p.addColumn(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
100      t.put(p);
101      p = new Put(ROW2);
102      p.addColumn(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
103      p.addColumn(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
104      p.addColumn(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
105      t.put(p);
106      String[] args = { sourceTable.getNameAsString(), FQ_OUTPUT_DIR.toString(), ";", "^row1" };
107      runCount(args);
108      FileInputStream inputStream =
109        new FileInputStream(OUTPUT_DIR + File.separator + "part-r-00000");
110      String data = IOUtils.toString(inputStream);
111      inputStream.close();
112      assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
113      assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
114      assertTrue(data.contains("Total ROWS" + "\t" + "1"));
115      assertTrue(data.contains("b;q" + "\t" + "1"));
116      assertTrue(data.contains("a;q" + "\t" + "1"));
117      assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
118      assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
119    } finally {
120      FileUtil.fullyDelete(new File(OUTPUT_DIR));
121    }
122  }
123
124  /**
125   * Test CellCounter all data should print to output
126   */
127  @Test
128  public void testCellCounterPrefix() throws Exception {
129    final TableName sourceTable = TableName.valueOf(name.getMethodName());
130    byte[][] families = { FAMILY_A, FAMILY_B };
131    try (Table t = UTIL.createTable(sourceTable, families)) {
132      Put p = new Put(ROW1);
133      p.addColumn(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
134      p.addColumn(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
135      p.addColumn(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
136      t.put(p);
137      p = new Put(ROW2);
138      p.addColumn(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
139      p.addColumn(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
140      p.addColumn(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
141      t.put(p);
142      String[] args = { sourceTable.getNameAsString(), FQ_OUTPUT_DIR.toString(), ";", "\\x01row1" };
143      runCount(args);
144      FileInputStream inputStream =
145        new FileInputStream(OUTPUT_DIR + File.separator + "part-r-00000");
146      String data = IOUtils.toString(inputStream);
147      inputStream.close();
148      assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
149      assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
150      assertTrue(data.contains("Total ROWS" + "\t" + "1"));
151      assertTrue(data.contains("b;q" + "\t" + "1"));
152      assertTrue(data.contains("a;q" + "\t" + "1"));
153      assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
154      assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
155    } finally {
156      FileUtil.fullyDelete(new File(OUTPUT_DIR));
157    }
158  }
159
160  /**
161   * Test CellCounter with time range all data should print to output
162   */
163  @Test
164  public void testCellCounterStartTimeRange() throws Exception {
165    final TableName sourceTable = TableName.valueOf(name.getMethodName());
166    byte[][] families = { FAMILY_A, FAMILY_B };
167    try (Table t = UTIL.createTable(sourceTable, families)) {
168      Put p = new Put(ROW1);
169      p.addColumn(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
170      p.addColumn(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
171      p.addColumn(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
172      t.put(p);
173      p = new Put(ROW2);
174      p.addColumn(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
175      p.addColumn(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
176      p.addColumn(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
177      t.put(p);
178      String[] args = { sourceTable.getNameAsString(), FQ_OUTPUT_DIR.toString(), ";", "^row1",
179        "--starttime=" + now, "--endtime=" + now + 2 };
180      runCount(args);
181      FileInputStream inputStream =
182        new FileInputStream(OUTPUT_DIR + File.separator + "part-r-00000");
183      String data = IOUtils.toString(inputStream);
184      inputStream.close();
185      assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
186      assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
187      assertTrue(data.contains("Total ROWS" + "\t" + "1"));
188      assertTrue(data.contains("b;q" + "\t" + "1"));
189      assertTrue(data.contains("a;q" + "\t" + "1"));
190      assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
191      assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
192    } finally {
193      FileUtil.fullyDelete(new File(OUTPUT_DIR));
194    }
195  }
196
197  /**
198   * Test CellCounter with time range all data should print to output
199   */
200  @Test
201  public void testCellCounteEndTimeRange() throws Exception {
202    final TableName sourceTable = TableName.valueOf(name.getMethodName());
203    byte[][] families = { FAMILY_A, FAMILY_B };
204    try (Table t = UTIL.createTable(sourceTable, families)) {
205      Put p = new Put(ROW1);
206      p.addColumn(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
207      p.addColumn(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
208      p.addColumn(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
209      t.put(p);
210      p = new Put(ROW2);
211      p.addColumn(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
212      p.addColumn(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
213      p.addColumn(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
214      t.put(p);
215      String[] args = { sourceTable.getNameAsString(), FQ_OUTPUT_DIR.toString(), ";", "^row1",
216        "--endtime=" + now + 1 };
217      runCount(args);
218      FileInputStream inputStream =
219        new FileInputStream(OUTPUT_DIR + File.separator + "part-r-00000");
220      String data = IOUtils.toString(inputStream);
221      inputStream.close();
222      assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
223      assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
224      assertTrue(data.contains("Total ROWS" + "\t" + "1"));
225      assertTrue(data.contains("b;q" + "\t" + "1"));
226      assertTrue(data.contains("a;q" + "\t" + "1"));
227      assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
228      assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
229    } finally {
230      FileUtil.fullyDelete(new File(OUTPUT_DIR));
231    }
232  }
233
234  /**
235   * Test CellCounter with time range all data should print to output
236   */
237  @Test
238  public void testCellCounteOutOfTimeRange() throws Exception {
239    final TableName sourceTable = TableName.valueOf(name.getMethodName());
240    byte[][] families = { FAMILY_A, FAMILY_B };
241    try (Table t = UTIL.createTable(sourceTable, families)) {
242      Put p = new Put(ROW1);
243      p.addColumn(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
244      p.addColumn(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
245      p.addColumn(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
246      t.put(p);
247      p = new Put(ROW2);
248      p.addColumn(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
249      p.addColumn(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
250      p.addColumn(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
251      t.put(p);
252      String[] args =
253        { sourceTable.getNameAsString(), FQ_OUTPUT_DIR.toString(), ";", "--starttime=" + now + 1,
254          "--endtime=" + now + 2 };
255
256      runCount(args);
257      FileInputStream inputStream =
258        new FileInputStream(OUTPUT_DIR + File.separator + "part-r-00000");
259      String data = IOUtils.toString(inputStream);
260      inputStream.close();
261      // nothing should hace been emitted to the reducer
262      assertTrue(data.isEmpty());
263    } finally {
264      FileUtil.fullyDelete(new File(OUTPUT_DIR));
265    }
266  }
267
268  private boolean runCount(String[] args) throws Exception {
269    // need to make a copy of the configuration because to make sure
270    // different temp dirs are used.
271    int status = ToolRunner.run(new Configuration(UTIL.getConfiguration()), new CellCounter(),
272        args);
273    return status == 0;
274  }
275
276  /**
277   * Test main method of CellCounter
278   */
279  @Test
280  public void testCellCounterMain() throws Exception {
281
282    PrintStream oldPrintStream = System.err;
283    SecurityManager SECURITY_MANAGER = System.getSecurityManager();
284    LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
285    System.setSecurityManager(newSecurityManager);
286    ByteArrayOutputStream data = new ByteArrayOutputStream();
287    String[] args = {};
288    System.setErr(new PrintStream(data));
289    try {
290      System.setErr(new PrintStream(data));
291
292      try {
293        CellCounter.main(args);
294        fail("should be SecurityException");
295      } catch (SecurityException e) {
296        assertEquals(-1, newSecurityManager.getExitCode());
297        assertTrue(data.toString().contains("ERROR: Wrong number of parameters:"));
298        // should be information about usage
299        assertTrue(data.toString().contains("Usage:"));
300      }
301
302    } finally {
303      System.setErr(oldPrintStream);
304      System.setSecurityManager(SECURITY_MANAGER);
305    }
306  }
307
308  /**
309   * Test CellCounter for complete table all data should print to output
310   */
311  @Test
312  public void testCellCounterForCompleteTable() throws Exception {
313    final TableName sourceTable = TableName.valueOf(name.getMethodName());
314    String outputPath = OUTPUT_DIR + sourceTable;
315    LocalFileSystem localFileSystem = new LocalFileSystem();
316    Path outputDir =
317        new Path(outputPath).makeQualified(localFileSystem.getUri(),
318          localFileSystem.getWorkingDirectory());
319    byte[][] families = { FAMILY_A, FAMILY_B };
320    Table t = UTIL.createTable(sourceTable, families);
321    try {
322      Put p = new Put(ROW1);
323      p.addColumn(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
324      p.addColumn(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
325      p.addColumn(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
326      t.put(p);
327      p = new Put(ROW2);
328      p.addColumn(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
329      p.addColumn(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
330      p.addColumn(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
331      t.put(p);
332      String[] args = { sourceTable.getNameAsString(), outputDir.toString(), ";" };
333      runCount(args);
334      FileInputStream inputStream =
335          new FileInputStream(outputPath + File.separator + "part-r-00000");
336      String data = IOUtils.toString(inputStream);
337      inputStream.close();
338      assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
339      assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "4"));
340      assertTrue(data.contains("Total ROWS" + "\t" + "2"));
341      assertTrue(data.contains("b;q" + "\t" + "2"));
342      assertTrue(data.contains("a;q" + "\t" + "2"));
343      assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
344      assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
345      assertTrue(data.contains("row2;a;q_Versions" + "\t" + "1"));
346      assertTrue(data.contains("row2;b;q_Versions" + "\t" + "1"));
347
348      FileUtil.fullyDelete(new File(outputPath));
349      args = new String[] { "-D " + TableInputFormat.SCAN_COLUMN_FAMILY + "=a, b",
350                  sourceTable.getNameAsString(), outputDir.toString(), ";"};
351      runCount(args);
352      inputStream = new FileInputStream(outputPath + File.separator + "part-r-00000");
353      String data2 = IOUtils.toString(inputStream);
354      inputStream.close();
355      assertEquals(data, data2);
356    } finally {
357      t.close();
358      localFileSystem.close();
359      FileUtil.fullyDelete(new File(outputPath));
360    }
361  }
362
363  @Test
364  public void TestCellCounterWithoutOutputDir() throws Exception {
365    String[] args = new String[] { "tableName" };
366    assertEquals("CellCounter should exit with -1 as output directory is not specified.", -1,
367      ToolRunner.run(HBaseConfiguration.create(), new CellCounter(), args));
368  }
369}