001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021
022import java.util.concurrent.ScheduledThreadPoolExecutor;
023import java.util.concurrent.SynchronousQueue;
024import java.util.concurrent.ThreadPoolExecutor;
025import java.util.concurrent.TimeUnit;
026import org.apache.hadoop.fs.FileSystem;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.HConstants;
029import org.apache.hadoop.hbase.TableName;
030import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
031import org.apache.hadoop.hbase.io.hfile.TestHFile;
032import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
033import org.apache.hadoop.hbase.mob.MobUtils;
034import org.apache.hadoop.hbase.testclassification.MediumTests;
035import org.apache.hadoop.hbase.testclassification.MiscTests;
036import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
037import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil;
038import org.junit.jupiter.api.AfterAll;
039import org.junit.jupiter.api.BeforeAll;
040import org.junit.jupiter.api.BeforeEach;
041import org.junit.jupiter.api.Disabled;
042import org.junit.jupiter.api.Tag;
043import org.junit.jupiter.api.Test;
044import org.junit.jupiter.api.TestInfo;
045
046import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
047
048// revisit later
049@Disabled
050@Tag(MiscTests.TAG)
051@Tag(MediumTests.TAG)
052public class TestHBaseFsckMOB extends BaseTestHBaseFsck {
053
054  @BeforeAll
055  public static void setUpBeforeClass() throws Exception {
056    TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
057      MasterSyncCoprocessor.class.getName());
058
059    conf.setInt("hbase.regionserver.handler.count", 2);
060    conf.setInt("hbase.regionserver.metahandler.count", 30);
061
062    conf.setInt("hbase.htable.threads.max", POOL_SIZE);
063    conf.setInt("hbase.hconnection.threads.max", 2 * POOL_SIZE);
064    conf.setInt("hbase.hbck.close.timeout", 2 * REGION_ONLINE_TIMEOUT);
065    conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 8 * REGION_ONLINE_TIMEOUT);
066    TEST_UTIL.startMiniCluster(1);
067
068    tableExecutorService = new ThreadPoolExecutor(1, POOL_SIZE, 60, TimeUnit.SECONDS,
069      new SynchronousQueue<>(), new ThreadFactoryBuilder().setNameFormat("testhbck-pool-%d")
070        .setDaemon(true).setUncaughtExceptionHandler(Threads.LOGGING_EXCEPTION_HANDLER).build());
071
072    hbfsckExecutorService = new ScheduledThreadPoolExecutor(POOL_SIZE);
073
074    AssignmentManager assignmentManager =
075      TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager();
076    regionStates = assignmentManager.getRegionStates();
077
078    connection = TEST_UTIL.getConnection();
079
080    admin = connection.getAdmin();
081    admin.balancerSwitch(false, true);
082
083    TEST_UTIL.waitUntilAllRegionsAssigned(TableName.META_TABLE_NAME);
084  }
085
086  @AfterAll
087  public static void tearDownAfterClass() throws Exception {
088    tableExecutorService.shutdown();
089    hbfsckExecutorService.shutdown();
090    admin.close();
091    TEST_UTIL.shutdownMiniCluster();
092  }
093
094  @BeforeEach
095  public void setUp() {
096    EnvironmentEdgeManager.reset();
097  }
098
099  /**
100   * This creates a table and then corrupts a mob file. Hbck should quarantine the file.
101   */
102  @SuppressWarnings("deprecation")
103  @Test
104  public void testQuarantineCorruptMobFile(TestInfo testInfo) throws Exception {
105    TableName table = TableName.valueOf(testInfo.getTestMethod().get().getName());
106    try {
107      setupMobTable(table);
108      assertEquals(ROWKEYS.length, countRows());
109      admin.flush(table);
110
111      FileSystem fs = FileSystem.get(conf);
112      Path mobFile = getFlushedMobFile(fs, table);
113      admin.disableTable(table);
114      // create new corrupt mob file.
115      String corruptMobFile = createMobFileName(mobFile.getName());
116      Path corrupt = new Path(mobFile.getParent(), corruptMobFile);
117      TestHFile.truncateFile(fs, mobFile, corrupt);
118      LOG.info("Created corrupted mob file " + corrupt);
119      HBaseFsck.debugLsr(conf, CommonFSUtils.getRootDir(conf));
120      HBaseFsck.debugLsr(conf, MobUtils.getMobHome(conf));
121
122      // A corrupt mob file doesn't abort the start of regions, so we can enable the table.
123      admin.enableTable(table);
124      HBaseFsck res = HbckTestingUtil.doHFileQuarantine(conf, table);
125      assertEquals(0, res.getRetCode());
126      HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
127      assertEquals(4, hfcc.getHFilesChecked());
128      assertEquals(0, hfcc.getCorrupted().size());
129      assertEquals(0, hfcc.getFailures().size());
130      assertEquals(0, hfcc.getQuarantined().size());
131      assertEquals(0, hfcc.getMissing().size());
132      assertEquals(5, hfcc.getMobFilesChecked());
133      assertEquals(1, hfcc.getCorruptedMobFiles().size());
134      assertEquals(0, hfcc.getFailureMobFiles().size());
135      assertEquals(1, hfcc.getQuarantinedMobFiles().size());
136      assertEquals(0, hfcc.getMissedMobFiles().size());
137      String quarantinedMobFile = hfcc.getQuarantinedMobFiles().iterator().next().getName();
138      assertEquals(corruptMobFile, quarantinedMobFile);
139    } finally {
140      cleanupTable(table);
141    }
142  }
143}