001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.mapreduce;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022import static org.junit.Assert.fail;
023
024import java.io.ByteArrayOutputStream;
025import java.io.File;
026import java.io.FileInputStream;
027import java.io.PrintStream;
028import org.apache.commons.io.IOUtils;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.FileUtil;
031import org.apache.hadoop.fs.LocalFileSystem;
032import org.apache.hadoop.fs.Path;
033import org.apache.hadoop.hbase.HBaseClassTestRule;
034import org.apache.hadoop.hbase.HBaseConfiguration;
035import org.apache.hadoop.hbase.HBaseTestingUtility;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.client.Put;
038import org.apache.hadoop.hbase.client.Table;
039import org.apache.hadoop.hbase.testclassification.LargeTests;
040import org.apache.hadoop.hbase.testclassification.MapReduceTests;
041import org.apache.hadoop.hbase.util.Bytes;
042import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
043import org.apache.hadoop.hbase.util.LauncherSecurityManager;
044import org.apache.hadoop.util.ToolRunner;
045import org.junit.AfterClass;
046import org.junit.BeforeClass;
047import org.junit.ClassRule;
048import org.junit.Rule;
049import org.junit.Test;
050import org.junit.experimental.categories.Category;
051import org.junit.rules.TestName;
052
053@Category({ MapReduceTests.class, LargeTests.class })
054public class TestCellCounter {
055  @ClassRule
056  public static final HBaseClassTestRule CLASS_RULE =
057    HBaseClassTestRule.forClass(TestCellCounter.class);
058
059  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
060  private static final byte[] ROW1 = Bytes.toBytesBinary("\\x01row1");
061  private static final byte[] ROW2 = Bytes.toBytesBinary("\\x01row2");
062  private static final String FAMILY_A_STRING = "a";
063  private static final String FAMILY_B_STRING = "b";
064  private static final byte[] FAMILY_A = Bytes.toBytes(FAMILY_A_STRING);
065  private static final byte[] FAMILY_B = Bytes.toBytes(FAMILY_B_STRING);
066  private static final byte[] QUALIFIER = Bytes.toBytes("q");
067
068  private static Path FQ_OUTPUT_DIR;
069  private static final String OUTPUT_DIR =
070    "target" + File.separator + "test-data" + File.separator + "output";
071  private static long now = EnvironmentEdgeManager.currentTime();
072
073  @Rule
074  public TestName name = new TestName();
075
076  @BeforeClass
077  public static void beforeClass() throws Exception {
078    UTIL.startMiniCluster();
079    FQ_OUTPUT_DIR = new Path(OUTPUT_DIR).makeQualified(new LocalFileSystem());
080    FileUtil.fullyDelete(new File(OUTPUT_DIR));
081  }
082
083  @AfterClass
084  public static void afterClass() throws Exception {
085    UTIL.shutdownMiniCluster();
086  }
087
088  /**
089   * Test CellCounter all data should print to output
090   */
091  @Test
092  public void testCellCounter() throws Exception {
093    final TableName sourceTable = TableName.valueOf(name.getMethodName());
094    byte[][] families = { FAMILY_A, FAMILY_B };
095    try (Table t = UTIL.createTable(sourceTable, families)) {
096      Put p = new Put(ROW1);
097      p.addColumn(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
098      p.addColumn(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
099      p.addColumn(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
100      t.put(p);
101      p = new Put(ROW2);
102      p.addColumn(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
103      p.addColumn(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
104      p.addColumn(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
105      t.put(p);
106      String[] args = { sourceTable.getNameAsString(), FQ_OUTPUT_DIR.toString(), ";", "^row1" };
107      runCount(args);
108      FileInputStream inputStream =
109        new FileInputStream(OUTPUT_DIR + File.separator + "part-r-00000");
110      String data = IOUtils.toString(inputStream);
111      inputStream.close();
112      assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
113      assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
114      assertTrue(data.contains("Total ROWS" + "\t" + "1"));
115      assertTrue(data.contains("b;q" + "\t" + "1"));
116      assertTrue(data.contains("a;q" + "\t" + "1"));
117      assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
118      assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
119    } finally {
120      FileUtil.fullyDelete(new File(OUTPUT_DIR));
121    }
122  }
123
124  /**
125   * Test CellCounter all data should print to output
126   */
127  @Test
128  public void testCellCounterPrefix() throws Exception {
129    final TableName sourceTable = TableName.valueOf(name.getMethodName());
130    byte[][] families = { FAMILY_A, FAMILY_B };
131    try (Table t = UTIL.createTable(sourceTable, families)) {
132      Put p = new Put(ROW1);
133      p.addColumn(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
134      p.addColumn(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
135      p.addColumn(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
136      t.put(p);
137      p = new Put(ROW2);
138      p.addColumn(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
139      p.addColumn(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
140      p.addColumn(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
141      t.put(p);
142      String[] args = { sourceTable.getNameAsString(), FQ_OUTPUT_DIR.toString(), ";", "\\x01row1" };
143      runCount(args);
144      FileInputStream inputStream =
145        new FileInputStream(OUTPUT_DIR + File.separator + "part-r-00000");
146      String data = IOUtils.toString(inputStream);
147      inputStream.close();
148      assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
149      assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
150      assertTrue(data.contains("Total ROWS" + "\t" + "1"));
151      assertTrue(data.contains("b;q" + "\t" + "1"));
152      assertTrue(data.contains("a;q" + "\t" + "1"));
153      assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
154      assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
155    } finally {
156      FileUtil.fullyDelete(new File(OUTPUT_DIR));
157    }
158  }
159
160  /**
161   * Test CellCounter with time range all data should print to output
162   */
163  @Test
164  public void testCellCounterStartTimeRange() throws Exception {
165    final TableName sourceTable = TableName.valueOf(name.getMethodName());
166    byte[][] families = { FAMILY_A, FAMILY_B };
167    try (Table t = UTIL.createTable(sourceTable, families)) {
168      Put p = new Put(ROW1);
169      p.addColumn(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
170      p.addColumn(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
171      p.addColumn(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
172      t.put(p);
173      p = new Put(ROW2);
174      p.addColumn(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
175      p.addColumn(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
176      p.addColumn(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
177      t.put(p);
178      String[] args = { sourceTable.getNameAsString(), FQ_OUTPUT_DIR.toString(), ";", "^row1",
179        "--starttime=" + now, "--endtime=" + now + 2 };
180      runCount(args);
181      FileInputStream inputStream =
182        new FileInputStream(OUTPUT_DIR + File.separator + "part-r-00000");
183      String data = IOUtils.toString(inputStream);
184      inputStream.close();
185      assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
186      assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
187      assertTrue(data.contains("Total ROWS" + "\t" + "1"));
188      assertTrue(data.contains("b;q" + "\t" + "1"));
189      assertTrue(data.contains("a;q" + "\t" + "1"));
190      assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
191      assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
192    } finally {
193      FileUtil.fullyDelete(new File(OUTPUT_DIR));
194    }
195  }
196
197  /**
198   * Test CellCounter with time range all data should print to output
199   */
200  @Test
201  public void testCellCounteEndTimeRange() throws Exception {
202    final TableName sourceTable = TableName.valueOf(name.getMethodName());
203    byte[][] families = { FAMILY_A, FAMILY_B };
204    try (Table t = UTIL.createTable(sourceTable, families)) {
205      Put p = new Put(ROW1);
206      p.addColumn(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
207      p.addColumn(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
208      p.addColumn(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
209      t.put(p);
210      p = new Put(ROW2);
211      p.addColumn(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
212      p.addColumn(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
213      p.addColumn(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
214      t.put(p);
215      String[] args = { sourceTable.getNameAsString(), FQ_OUTPUT_DIR.toString(), ";", "^row1",
216        "--endtime=" + now + 1 };
217      runCount(args);
218      FileInputStream inputStream =
219        new FileInputStream(OUTPUT_DIR + File.separator + "part-r-00000");
220      String data = IOUtils.toString(inputStream);
221      inputStream.close();
222      assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
223      assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
224      assertTrue(data.contains("Total ROWS" + "\t" + "1"));
225      assertTrue(data.contains("b;q" + "\t" + "1"));
226      assertTrue(data.contains("a;q" + "\t" + "1"));
227      assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
228      assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
229    } finally {
230      FileUtil.fullyDelete(new File(OUTPUT_DIR));
231    }
232  }
233
234  /**
235   * Test CellCounter with time range all data should print to output
236   */
237  @Test
238  public void testCellCounteOutOfTimeRange() throws Exception {
239    final TableName sourceTable = TableName.valueOf(name.getMethodName());
240    byte[][] families = { FAMILY_A, FAMILY_B };
241    try (Table t = UTIL.createTable(sourceTable, families)) {
242      Put p = new Put(ROW1);
243      p.addColumn(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
244      p.addColumn(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
245      p.addColumn(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
246      t.put(p);
247      p = new Put(ROW2);
248      p.addColumn(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
249      p.addColumn(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
250      p.addColumn(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
251      t.put(p);
252      String[] args = { sourceTable.getNameAsString(), FQ_OUTPUT_DIR.toString(), ";",
253        "--starttime=" + now + 1, "--endtime=" + now + 2 };
254
255      runCount(args);
256      FileInputStream inputStream =
257        new FileInputStream(OUTPUT_DIR + File.separator + "part-r-00000");
258      String data = IOUtils.toString(inputStream);
259      inputStream.close();
260      // nothing should hace been emitted to the reducer
261      assertTrue(data.isEmpty());
262    } finally {
263      FileUtil.fullyDelete(new File(OUTPUT_DIR));
264    }
265  }
266
267  private boolean runCount(String[] args) throws Exception {
268    // need to make a copy of the configuration because to make sure
269    // different temp dirs are used.
270    int status =
271      ToolRunner.run(new Configuration(UTIL.getConfiguration()), new CellCounter(), args);
272    return status == 0;
273  }
274
275  /**
276   * Test main method of CellCounter
277   */
278  @Test
279  public void testCellCounterMain() throws Exception {
280
281    PrintStream oldPrintStream = System.err;
282    SecurityManager SECURITY_MANAGER = System.getSecurityManager();
283    LauncherSecurityManager newSecurityManager = new LauncherSecurityManager();
284    System.setSecurityManager(newSecurityManager);
285    ByteArrayOutputStream data = new ByteArrayOutputStream();
286    String[] args = {};
287    System.setErr(new PrintStream(data));
288    try {
289      System.setErr(new PrintStream(data));
290
291      try {
292        CellCounter.main(args);
293        fail("should be SecurityException");
294      } catch (SecurityException e) {
295        assertEquals(-1, newSecurityManager.getExitCode());
296        assertTrue(data.toString().contains("ERROR: Wrong number of parameters:"));
297        // should be information about usage
298        assertTrue(data.toString().contains("Usage:"));
299      }
300
301    } finally {
302      System.setErr(oldPrintStream);
303      System.setSecurityManager(SECURITY_MANAGER);
304    }
305  }
306
307  /**
308   * Test CellCounter for complete table all data should print to output
309   */
310  @Test
311  public void testCellCounterForCompleteTable() throws Exception {
312    final TableName sourceTable = TableName.valueOf(name.getMethodName());
313    String outputPath = OUTPUT_DIR + sourceTable;
314    LocalFileSystem localFileSystem = new LocalFileSystem();
315    Path outputDir = new Path(outputPath).makeQualified(localFileSystem.getUri(),
316      localFileSystem.getWorkingDirectory());
317    byte[][] families = { FAMILY_A, FAMILY_B };
318    Table t = UTIL.createTable(sourceTable, families);
319    try {
320      Put p = new Put(ROW1);
321      p.addColumn(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
322      p.addColumn(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
323      p.addColumn(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
324      t.put(p);
325      p = new Put(ROW2);
326      p.addColumn(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
327      p.addColumn(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
328      p.addColumn(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
329      t.put(p);
330      String[] args = { sourceTable.getNameAsString(), outputDir.toString(), ";" };
331      runCount(args);
332      FileInputStream inputStream =
333        new FileInputStream(outputPath + File.separator + "part-r-00000");
334      String data = IOUtils.toString(inputStream);
335      inputStream.close();
336      assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
337      assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "4"));
338      assertTrue(data.contains("Total ROWS" + "\t" + "2"));
339      assertTrue(data.contains("b;q" + "\t" + "2"));
340      assertTrue(data.contains("a;q" + "\t" + "2"));
341      assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
342      assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
343      assertTrue(data.contains("row2;a;q_Versions" + "\t" + "1"));
344      assertTrue(data.contains("row2;b;q_Versions" + "\t" + "1"));
345
346      FileUtil.fullyDelete(new File(outputPath));
347      args = new String[] { "-D " + TableInputFormat.SCAN_COLUMN_FAMILY + "=a, b",
348        sourceTable.getNameAsString(), outputDir.toString(), ";" };
349      runCount(args);
350      inputStream = new FileInputStream(outputPath + File.separator + "part-r-00000");
351      String data2 = IOUtils.toString(inputStream);
352      inputStream.close();
353      assertEquals(data, data2);
354    } finally {
355      t.close();
356      localFileSystem.close();
357      FileUtil.fullyDelete(new File(outputPath));
358    }
359  }
360
361  @Test
362  public void TestCellCounterWithoutOutputDir() throws Exception {
363    String[] args = new String[] { "tableName" };
364    assertEquals("CellCounter should exit with -1 as output directory is not specified.", -1,
365      ToolRunner.run(HBaseConfiguration.create(), new CellCounter(), args));
366  }
367}