001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.procedure2.store.region;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotEquals;
022import static org.junit.Assert.fail;
023
024import java.io.BufferedReader;
025import java.io.ByteArrayInputStream;
026import java.io.ByteArrayOutputStream;
027import java.io.IOException;
028import java.io.InputStreamReader;
029import java.io.PrintStream;
030import java.nio.charset.StandardCharsets;
031import java.util.ArrayList;
032import java.util.List;
033import org.apache.commons.lang3.mutable.MutableLong;
034import org.apache.hadoop.fs.FileSystem;
035import org.apache.hadoop.fs.Path;
036import org.apache.hadoop.hbase.HBaseClassTestRule;
037import org.apache.hadoop.hbase.client.RegionInfo;
038import org.apache.hadoop.hbase.io.hfile.HFile;
039import org.apache.hadoop.hbase.master.region.MasterRegionFactory;
040import org.apache.hadoop.hbase.testclassification.MasterTests;
041import org.apache.hadoop.hbase.testclassification.SmallTests;
042import org.apache.hadoop.hbase.util.Bytes;
043import org.apache.hadoop.hbase.util.CommonFSUtils;
044import org.apache.hadoop.util.ToolRunner;
045import org.junit.ClassRule;
046import org.junit.Test;
047import org.junit.experimental.categories.Category;
048import org.slf4j.Logger;
049import org.slf4j.LoggerFactory;
050
051@Category({ MasterTests.class, SmallTests.class })
052public class TestHFileProcedurePrettyPrinter extends RegionProcedureStoreTestBase {
053
054  @ClassRule
055  public static final HBaseClassTestRule CLASS_RULE =
056    HBaseClassTestRule.forClass(TestHFileProcedurePrettyPrinter.class);
057
058  private static final Logger LOG = LoggerFactory.getLogger(TestHFileProcedurePrettyPrinter.class);
059
060  private List<String> checkOutput(BufferedReader reader, MutableLong putCount,
061    MutableLong deleteCount, MutableLong markDeletedCount) throws IOException {
062    putCount.setValue(0);
063    deleteCount.setValue(0);
064    markDeletedCount.setValue(0);
065    List<String> fileScanned = new ArrayList<>();
066    for (;;) {
067      String line = reader.readLine();
068      if (line == null) {
069        return fileScanned;
070      }
071      LOG.info(line);
072      if (line.contains("V: mark deleted")) {
073        markDeletedCount.increment();
074      } else if (line.contains("/Put/")) {
075        putCount.increment();
076      } else if (line.contains("/DeleteFamily/")) {
077        deleteCount.increment();
078      } else if (line.startsWith("Scanning -> ")) {
079        fileScanned.add(line.split(" -> ")[1]);
080      } else {
081        fail("Unrecognized output: " + line);
082      }
083    }
084  }
085
086  @Test
087  public void test() throws Exception {
088    HFileProcedurePrettyPrinter printer = new HFileProcedurePrettyPrinter();
089    // -a or -f is required so passing empty args will cause an error and return a non-zero value.
090    assertNotEquals(0, ToolRunner.run(htu.getConfiguration(), printer, new String[0]));
091    List<RegionProcedureStoreTestProcedure> procs = new ArrayList<>();
092    for (int i = 0; i < 10; i++) {
093      RegionProcedureStoreTestProcedure proc = new RegionProcedureStoreTestProcedure();
094      store.insert(proc, null);
095      procs.add(proc);
096    }
097    store.region.flush(true);
098    for (int i = 0; i < 5; i++) {
099      store.delete(procs.get(i).getProcId());
100    }
101    store.region.flush(true);
102    store.cleanup();
103    store.region.flush(true);
104    Path tableDir = CommonFSUtils.getTableDir(
105      new Path(htu.getDataTestDir(), MasterRegionFactory.MASTER_STORE_DIR), MasterRegionFactory.TABLE_NAME);
106    FileSystem fs = tableDir.getFileSystem(htu.getConfiguration());
107    Path regionDir =
108      fs.listStatus(tableDir, p -> RegionInfo.isEncodedRegionName(Bytes.toBytes(p.getName())))[0]
109        .getPath();
110    List<Path> storefiles = HFile.getStoreFiles(fs, regionDir);
111    ByteArrayOutputStream bos = new ByteArrayOutputStream();
112    PrintStream out = new PrintStream(bos);
113    MutableLong putCount = new MutableLong();
114    MutableLong deleteCount = new MutableLong();
115    MutableLong markDeletedCount = new MutableLong();
116    for (Path file : storefiles) {
117      bos.reset();
118      printer = new HFileProcedurePrettyPrinter(out);
119      assertEquals(0,
120        ToolRunner.run(htu.getConfiguration(), printer, new String[] { "-f", file.toString() }));
121      try (BufferedReader reader =
122        new BufferedReader(new InputStreamReader(new ByteArrayInputStream(bos.toByteArray()),
123          StandardCharsets.UTF_8))) {
124        List<String> fileScanned = checkOutput(reader, putCount, deleteCount, markDeletedCount);
125        assertEquals(1, fileScanned.size());
126        assertEquals(file.toString(), fileScanned.get(0));
127        if (putCount.longValue() == 10) {
128          assertEquals(0, deleteCount.longValue());
129          assertEquals(0, markDeletedCount.longValue());
130        } else if (deleteCount.longValue() == 5) {
131          assertEquals(0, putCount.longValue());
132          assertEquals(0, markDeletedCount.longValue());
133        } else if (markDeletedCount.longValue() == 5) {
134          assertEquals(0, putCount.longValue());
135          assertEquals(0, deleteCount.longValue());
136        } else {
137          fail("Should have entered one of the above 3 branches");
138        }
139      }
140    }
141    bos.reset();
142    printer = new HFileProcedurePrettyPrinter(out);
143    assertEquals(0, ToolRunner.run(htu.getConfiguration(), printer, new String[] { "-a" }));
144    try (BufferedReader reader = new BufferedReader(
145      new InputStreamReader(new ByteArrayInputStream(bos.toByteArray()), StandardCharsets.UTF_8))) {
146      List<String> fileScanned = checkOutput(reader, putCount, deleteCount, markDeletedCount);
147      assertEquals(3, fileScanned.size());
148      assertEquals(10, putCount.longValue());
149      assertEquals(5, deleteCount.longValue());
150      assertEquals(5, markDeletedCount.longValue());
151    }
152  }
153}