001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.procedure2.store.region;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertNotEquals;
022import static org.junit.jupiter.api.Assertions.fail;
023
024import java.io.BufferedReader;
025import java.io.ByteArrayInputStream;
026import java.io.ByteArrayOutputStream;
027import java.io.IOException;
028import java.io.InputStreamReader;
029import java.io.PrintStream;
030import java.nio.charset.StandardCharsets;
031import java.util.ArrayList;
032import java.util.List;
033import org.apache.commons.lang3.mutable.MutableLong;
034import org.apache.hadoop.fs.FileSystem;
035import org.apache.hadoop.fs.Path;
036import org.apache.hadoop.hbase.client.RegionInfo;
037import org.apache.hadoop.hbase.io.hfile.HFile;
038import org.apache.hadoop.hbase.master.region.MasterRegionFactory;
039import org.apache.hadoop.hbase.testclassification.MasterTests;
040import org.apache.hadoop.hbase.testclassification.SmallTests;
041import org.apache.hadoop.hbase.util.Bytes;
042import org.apache.hadoop.hbase.util.CommonFSUtils;
043import org.apache.hadoop.util.ToolRunner;
044import org.junit.jupiter.api.Tag;
045import org.junit.jupiter.api.Test;
046import org.slf4j.Logger;
047import org.slf4j.LoggerFactory;
048
049@Tag(SmallTests.TAG)
050@Tag(MasterTests.TAG)
051public class TestHFileProcedurePrettyPrinter extends RegionProcedureStoreTestBase {
052
053  private static final Logger LOG = LoggerFactory.getLogger(TestHFileProcedurePrettyPrinter.class);
054
055  private List<String> checkOutput(BufferedReader reader, MutableLong putCount,
056    MutableLong deleteCount, MutableLong markDeletedCount) throws IOException {
057    putCount.setValue(0);
058    deleteCount.setValue(0);
059    markDeletedCount.setValue(0);
060    List<String> fileScanned = new ArrayList<>();
061    for (;;) {
062      String line = reader.readLine();
063      if (line == null) {
064        return fileScanned;
065      }
066      LOG.info(line);
067      if (line.contains("V: mark deleted")) {
068        markDeletedCount.increment();
069      } else if (line.contains("/Put/")) {
070        putCount.increment();
071      } else if (line.contains("/DeleteFamily/")) {
072        deleteCount.increment();
073      } else if (line.startsWith("Scanning -> ")) {
074        fileScanned.add(line.split(" -> ")[1]);
075      } else {
076        fail("Unrecognized output: " + line);
077      }
078    }
079  }
080
081  @Test
082  public void test() throws Exception {
083    HFileProcedurePrettyPrinter printer = new HFileProcedurePrettyPrinter();
084    // -a or -f is required so passing empty args will cause an error and return a non-zero value.
085    assertNotEquals(0, ToolRunner.run(htu.getConfiguration(), printer, new String[0]));
086    List<RegionProcedureStoreTestProcedure> procs = new ArrayList<>();
087    for (int i = 0; i < 10; i++) {
088      RegionProcedureStoreTestProcedure proc = new RegionProcedureStoreTestProcedure();
089      store.insert(proc, null);
090      procs.add(proc);
091    }
092    store.region.flush(true);
093    for (int i = 0; i < 5; i++) {
094      store.delete(procs.get(i).getProcId());
095    }
096    store.region.flush(true);
097    store.cleanup();
098    store.region.flush(true);
099    Path tableDir = CommonFSUtils.getTableDir(
100      new Path(htu.getDataTestDir(), MasterRegionFactory.MASTER_STORE_DIR),
101      MasterRegionFactory.TABLE_NAME);
102    FileSystem fs = tableDir.getFileSystem(htu.getConfiguration());
103    Path regionDir =
104      fs.listStatus(tableDir, p -> RegionInfo.isEncodedRegionName(Bytes.toBytes(p.getName())))[0]
105        .getPath();
106    List<Path> storefiles = HFile.getStoreFiles(fs, regionDir);
107    ByteArrayOutputStream bos = new ByteArrayOutputStream();
108    PrintStream out = new PrintStream(bos);
109    MutableLong putCount = new MutableLong();
110    MutableLong deleteCount = new MutableLong();
111    MutableLong markDeletedCount = new MutableLong();
112    for (Path file : storefiles) {
113      bos.reset();
114      printer = new HFileProcedurePrettyPrinter(out);
115      assertEquals(0,
116        ToolRunner.run(htu.getConfiguration(), printer, new String[] { "-f", file.toString() }));
117      try (BufferedReader reader =
118        new BufferedReader(new InputStreamReader(new ByteArrayInputStream(bos.toByteArray()),
119          StandardCharsets.UTF_8))) {
120        List<String> fileScanned = checkOutput(reader, putCount, deleteCount, markDeletedCount);
121        assertEquals(1, fileScanned.size());
122        assertEquals(file.toString(), fileScanned.get(0));
123        if (putCount.longValue() == 10) {
124          assertEquals(0, deleteCount.longValue());
125          assertEquals(0, markDeletedCount.longValue());
126        } else if (deleteCount.longValue() == 5) {
127          assertEquals(0, putCount.longValue());
128          assertEquals(0, markDeletedCount.longValue());
129        } else if (markDeletedCount.longValue() == 5) {
130          assertEquals(0, putCount.longValue());
131          assertEquals(0, deleteCount.longValue());
132        } else {
133          fail("Should have entered one of the above 3 branches");
134        }
135      }
136    }
137    bos.reset();
138    printer = new HFileProcedurePrettyPrinter(out);
139    assertEquals(0, ToolRunner.run(htu.getConfiguration(), printer, new String[] { "-a" }));
140    try (BufferedReader reader = new BufferedReader(
141      new InputStreamReader(new ByteArrayInputStream(bos.toByteArray()), StandardCharsets.UTF_8))) {
142      List<String> fileScanned = checkOutput(reader, putCount, deleteCount, markDeletedCount);
143      assertEquals(3, fileScanned.size());
144      assertEquals(10, putCount.longValue());
145      assertEquals(5, deleteCount.longValue());
146      assertEquals(5, markDeletedCount.longValue());
147    }
148  }
149}