001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.mapreduce;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022import static org.junit.Assert.fail;
023
024import java.io.ByteArrayOutputStream;
025import java.io.File;
026import java.io.FileInputStream;
027import java.io.PrintStream;
028import org.apache.commons.io.IOUtils;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.FileUtil;
031import org.apache.hadoop.fs.LocalFileSystem;
032import org.apache.hadoop.fs.Path;
033import org.apache.hadoop.hbase.HBaseClassTestRule;
034import org.apache.hadoop.hbase.HBaseConfiguration;
035import org.apache.hadoop.hbase.HBaseTestingUtil;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.client.Put;
038import org.apache.hadoop.hbase.client.Table;
039import org.apache.hadoop.hbase.testclassification.LargeTests;
040import org.apache.hadoop.hbase.testclassification.MapReduceTests;
041import org.apache.hadoop.hbase.util.Bytes;
042import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
043import org.apache.hadoop.hbase.util.LauncherSecurityManager;
044import org.apache.hadoop.util.ToolRunner;
045import org.junit.AfterClass;
046import org.junit.BeforeClass;
047import org.junit.ClassRule;
048import org.junit.Rule;
049import org.junit.Test;
050import org.junit.experimental.categories.Category;
051import org.junit.rules.TestName;
052
053@Category({ MapReduceTests.class, LargeTests.class })
054public class TestCellCounter {
055  @ClassRule
056  public static final HBaseClassTestRule CLASS_RULE =
057    HBaseClassTestRule.forClass(TestCellCounter.class);
058
059  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
060  private static final byte[] ROW1 = Bytes.toBytesBinary("\\x01row1");
061  private static final byte[] ROW2 = Bytes.toBytesBinary("\\x01row2");
062  private static final String FAMILY_A_STRING = "a";
063  private static final String FAMILY_B_STRING = "b";
064  private static final byte[] FAMILY_A = Bytes.toBytes(FAMILY_A_STRING);
065  private static final byte[] FAMILY_B = Bytes.toBytes(FAMILY_B_STRING);
066  private static final byte[] QUALIFIER = Bytes.toBytes("q");
067
068  private static Path FQ_OUTPUT_DIR;
069  private static final String OUTPUT_DIR =
070    "target" + File.separator + "test-data" + File.separator + "output";
071  private static long now = EnvironmentEdgeManager.currentTime();
072
073  @Rule
074  public TestName name = new TestName();
075
076  @BeforeClass
077  public static void beforeClass() throws Exception {
078    UTIL.startMiniCluster();
079    FQ_OUTPUT_DIR = new Path(OUTPUT_DIR).makeQualified(new LocalFileSystem());
080    FileUtil.fullyDelete(new File(OUTPUT_DIR));
081  }
082
083  @AfterClass
084  public static void afterClass() throws Exception {
085    UTIL.shutdownMiniCluster();
086  }
087
088  /**
089   * Test CellCounter all data should print to output
090   */
091  @Test
092  public void testCellCounter() throws Exception {
093    final TableName sourceTable = TableName.valueOf(name.getMethodName());
094    byte[][] families = { FAMILY_A, FAMILY_B };
095    try (Table t = UTIL.createTable(sourceTable, families)) {
096      Put p = new Put(ROW1);
097      p.addColumn(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
098      p.addColumn(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
099      p.addColumn(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
100      t.put(p);
101      p = new Put(ROW2);
102      p.addColumn(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
103      p.addColumn(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
104      p.addColumn(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
105      t.put(p);
106      String[] args = { sourceTable.getNameAsString(), FQ_OUTPUT_DIR.toString(), ";", "^row1" };
107      runCount(args);
108      FileInputStream inputStream =
109        new FileInputStream(OUTPUT_DIR + File.separator + "part-r-00000");
110      String data = IOUtils.toString(inputStream);
111      inputStream.close();
112      assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
113      assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
114      assertTrue(data.contains("Total ROWS" + "\t" + "1"));
115      assertTrue(data.contains("b;q" + "\t" + "1"));
116      assertTrue(data.contains("a;q" + "\t" + "1"));
117      assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
118      assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
119    } finally {
120      FileUtil.fullyDelete(new File(OUTPUT_DIR));
121    }
122  }
123
124  /**
125   * Test CellCounter all data should print to output
126   */
127  @Test
128  public void testCellCounterPrefix() throws Exception {
129    final TableName sourceTable = TableName.valueOf(name.getMethodName());
130    byte[][] families = { FAMILY_A, FAMILY_B };
131    try (Table t = UTIL.createTable(sourceTable, families)) {
132      Put p = new Put(ROW1);
133      p.addColumn(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
134      p.addColumn(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
135      p.addColumn(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
136      t.put(p);
137      p = new Put(ROW2);
138      p.addColumn(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
139      p.addColumn(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
140      p.addColumn(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
141      t.put(p);
142      String[] args = { sourceTable.getNameAsString(), FQ_OUTPUT_DIR.toString(), ";", "\\x01row1" };
143      runCount(args);
144      FileInputStream inputStream =
145        new FileInputStream(OUTPUT_DIR + File.separator + "part-r-00000");
146      String data = IOUtils.toString(inputStream);
147      inputStream.close();
148      assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
149      assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
150      assertTrue(data.contains("Total ROWS" + "\t" + "1"));
151      assertTrue(data.contains("b;q" + "\t" + "1"));
152      assertTrue(data.contains("a;q" + "\t" + "1"));
153      assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
154      assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
155    } finally {
156      FileUtil.fullyDelete(new File(OUTPUT_DIR));
157    }
158  }
159
160  /**
161   * Test CellCounter with time range all data should print to output
162   */
163  @Test
164  public void testCellCounterStartTimeRange() throws Exception {
165    final TableName sourceTable = TableName.valueOf(name.getMethodName());
166    byte[][] families = { FAMILY_A, FAMILY_B };
167    try (Table t = UTIL.createTable(sourceTable, families)) {
168      Put p = new Put(ROW1);
169      p.addColumn(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
170      p.addColumn(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
171      p.addColumn(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
172      t.put(p);
173      p = new Put(ROW2);
174      p.addColumn(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
175      p.addColumn(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
176      p.addColumn(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
177      t.put(p);
178      String[] args = { sourceTable.getNameAsString(), FQ_OUTPUT_DIR.toString(), ";", "^row1",
179        "--starttime=" + now, "--endtime=" + now + 2 };
180      runCount(args);
181      FileInputStream inputStream =
182        new FileInputStream(OUTPUT_DIR + File.separator + "part-r-00000");
183      String data = IOUtils.toString(inputStream);
184      inputStream.close();
185      assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
186      assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
187      assertTrue(data.contains("Total ROWS" + "\t" + "1"));
188      assertTrue(data.contains("b;q" + "\t" + "1"));
189      assertTrue(data.contains("a;q" + "\t" + "1"));
190      assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
191      assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
192    } finally {
193      FileUtil.fullyDelete(new File(OUTPUT_DIR));
194    }
195  }
196
197  /**
198   * Test CellCounter with time range all data should print to output
199   */
200  @Test
201  public void testCellCounteEndTimeRange() throws Exception {
202    final TableName sourceTable = TableName.valueOf(name.getMethodName());
203    byte[][] families = { FAMILY_A, FAMILY_B };
204    try (Table t = UTIL.createTable(sourceTable, families)) {
205      Put p = new Put(ROW1);
206      p.addColumn(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
207      p.addColumn(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
208      p.addColumn(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
209      t.put(p);
210      p = new Put(ROW2);
211      p.addColumn(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
212      p.addColumn(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
213      p.addColumn(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
214      t.put(p);
215      String[] args = { sourceTable.getNameAsString(), FQ_OUTPUT_DIR.toString(), ";", "^row1",
216        "--endtime=" + now + 1 };
217      runCount(args);
218      FileInputStream inputStream =
219        new FileInputStream(OUTPUT_DIR + File.separator + "part-r-00000");
220      String data = IOUtils.toString(inputStream);
221      inputStream.close();
222      assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
223      assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
224      assertTrue(data.contains("Total ROWS" + "\t" + "1"));
225      assertTrue(data.contains("b;q" + "\t" + "1"));
226      assertTrue(data.contains("a;q" + "\t" + "1"));
227      assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
228      assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
229    } finally {
230      FileUtil.fullyDelete(new File(OUTPUT_DIR));
231    }
232  }
233
234  /**
235   * Test CellCounter with time range all data should print to output
236   */
237  @Test
238  public void testCellCounteOutOfTimeRange() throws Exception {
239    final TableName sourceTable = TableName.valueOf(name.getMethodName());
240    byte[][] families = { FAMILY_A, FAMILY_B };
241    try (Table t = UTIL.createTable(sourceTable, families)) {
242      Put p = new Put(ROW1);
243      p.addColumn(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
244      p.addColumn(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
245      p.addColumn(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
246      t.put(p);
247      p = new Put(ROW2);
248      p.addColumn(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
249      p.addColumn(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
250      p.addColumn(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
251      t.put(p);
252      String[] args = { sourceTable.getNameAsString(), FQ_OUTPUT_DIR.toString(), ";",
253        "--starttime=" + now + 1, "--endtime=" + now + 2 };
254
255      runCount(args);
256      FileInputStream inputStream =
257        new FileInputStream(OUTPUT_DIR + File.separator + "part-r-00000");
258      String data = IOUtils.toString(inputStream);
259      inputStream.close();
260      // nothing should hace been emitted to the reducer
261      assertTrue(data.isEmpty());
262    } finally {
263      FileUtil.fullyDelete(new File(OUTPUT_DIR));
264    }
265  }
266
267  private boolean runCount(String[] args) throws Exception {
268    // need to make a copy of the configuration because to make sure
269    // different temp dirs are used.
270    int status =
271      ToolRunner.run(new Configuration(UTIL.getConfiguration()), new CellCounter(), args);
272    return status == 0;
273  }
274
275  /**
276   * Test main method of CellCounter
277   */
278  @Test
279  public void testCellCounterMain() throws Exception {
280    PrintStream oldPrintStream = System.err;
281    SecurityManager SECURITY_MANAGER = System.getSecurityManager();
282    LauncherSecurityManager newSecurityManager = new LauncherSecurityManager();
283    System.setSecurityManager(newSecurityManager);
284    ByteArrayOutputStream data = new ByteArrayOutputStream();
285    String[] args = {};
286    System.setErr(new PrintStream(data));
287    try {
288      System.setErr(new PrintStream(data));
289
290      try {
291        CellCounter.main(args);
292        fail("should be SecurityException");
293      } catch (SecurityException e) {
294        assertEquals(-1, newSecurityManager.getExitCode());
295        assertTrue(data.toString().contains("ERROR: Wrong number of parameters:"));
296        // should be information about usage
297        assertTrue(data.toString().contains("Usage:"));
298      }
299
300    } finally {
301      System.setErr(oldPrintStream);
302      System.setSecurityManager(SECURITY_MANAGER);
303    }
304  }
305
306  /**
307   * Test CellCounter for complete table all data should print to output
308   */
309  @Test
310  public void testCellCounterForCompleteTable() throws Exception {
311    final TableName sourceTable = TableName.valueOf(name.getMethodName());
312    String outputPath = OUTPUT_DIR + sourceTable;
313    LocalFileSystem localFileSystem = new LocalFileSystem();
314    Path outputDir = new Path(outputPath).makeQualified(localFileSystem.getUri(),
315      localFileSystem.getWorkingDirectory());
316    byte[][] families = { FAMILY_A, FAMILY_B };
317    Table t = UTIL.createTable(sourceTable, families);
318    try {
319      Put p = new Put(ROW1);
320      p.addColumn(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
321      p.addColumn(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
322      p.addColumn(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
323      t.put(p);
324      p = new Put(ROW2);
325      p.addColumn(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
326      p.addColumn(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
327      p.addColumn(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
328      t.put(p);
329      String[] args = { sourceTable.getNameAsString(), outputDir.toString(), ";" };
330      runCount(args);
331      FileInputStream inputStream =
332        new FileInputStream(outputPath + File.separator + "part-r-00000");
333      String data = IOUtils.toString(inputStream);
334      inputStream.close();
335      assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
336      assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "4"));
337      assertTrue(data.contains("Total ROWS" + "\t" + "2"));
338      assertTrue(data.contains("b;q" + "\t" + "2"));
339      assertTrue(data.contains("a;q" + "\t" + "2"));
340      assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
341      assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
342      assertTrue(data.contains("row2;a;q_Versions" + "\t" + "1"));
343      assertTrue(data.contains("row2;b;q_Versions" + "\t" + "1"));
344
345      FileUtil.fullyDelete(new File(outputPath));
346      args = new String[] { "-D " + TableInputFormat.SCAN_COLUMN_FAMILY + "=a, b",
347        sourceTable.getNameAsString(), outputDir.toString(), ";" };
348      runCount(args);
349      inputStream = new FileInputStream(outputPath + File.separator + "part-r-00000");
350      String data2 = IOUtils.toString(inputStream);
351      inputStream.close();
352      assertEquals(data, data2);
353    } finally {
354      t.close();
355      localFileSystem.close();
356      FileUtil.fullyDelete(new File(outputPath));
357    }
358  }
359
360  @Test
361  public void TestCellCounterWithoutOutputDir() throws Exception {
362    String[] args = new String[] { "tableName" };
363    assertEquals("CellCounter should exit with -1 as output directory is not specified.", -1,
364      ToolRunner.run(HBaseConfiguration.create(), new CellCounter(), args));
365  }
366}