001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022import static org.junit.jupiter.api.Assertions.assertNotEquals;
023import static org.junit.jupiter.api.Assertions.assertNotNull;
024import static org.junit.jupiter.api.Assertions.assertNull;
025import static org.junit.jupiter.api.Assertions.assertTrue;
026
027import java.time.Instant;
028import java.util.Collections;
029import java.util.List;
030import java.util.Map;
031import java.util.Optional;
032import java.util.concurrent.Future;
033import org.apache.hadoop.conf.Configuration;
034import org.apache.hadoop.fs.FileSystem;
035import org.apache.hadoop.fs.Path;
036import org.apache.hadoop.hbase.HConstants;
037import org.apache.hadoop.hbase.ServerName;
038import org.apache.hadoop.hbase.TableName;
039import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
040import org.apache.hadoop.hbase.client.RegionInfo;
041import org.apache.hadoop.hbase.client.RegionInfoBuilder;
042import org.apache.hadoop.hbase.client.TableDescriptor;
043import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
044import org.apache.hadoop.hbase.client.TableState;
045import org.apache.hadoop.hbase.master.TableStateManager;
046import org.apache.hadoop.hbase.master.hbck.HbckChore;
047import org.apache.hadoop.hbase.master.hbck.HbckReport;
048import org.apache.hadoop.hbase.regionserver.BloomType;
049import org.apache.hadoop.hbase.regionserver.HRegion;
050import org.apache.hadoop.hbase.testclassification.MasterTests;
051import org.apache.hadoop.hbase.testclassification.MediumTests;
052import org.apache.hadoop.hbase.util.Bytes;
053import org.apache.hadoop.hbase.util.CommonFSUtils;
054import org.apache.hadoop.hbase.util.FSTableDescriptors;
055import org.apache.hadoop.hbase.util.FSUtils;
056import org.apache.hadoop.hbase.util.Pair;
057import org.junit.jupiter.api.BeforeEach;
058import org.junit.jupiter.api.Tag;
059import org.junit.jupiter.api.Test;
060import org.mockito.Mockito;
061
062@Tag(MasterTests.TAG)
063@Tag(MediumTests.TAG)
064public class TestHbckChore extends TestAssignmentManagerBase {
065
066  private HbckChore hbckChore;
067
068  @BeforeEach
069  public void setUp() throws Exception {
070    super.setUp();
071    hbckChore = new HbckChore(master);
072  }
073
074  @Test
075  public void testForMeta() {
076    byte[] metaRegionNameAsBytes = RegionInfoBuilder.FIRST_META_REGIONINFO.getRegionName();
077    String metaRegionName = RegionInfoBuilder.FIRST_META_REGIONINFO.getRegionNameAsString();
078    List<ServerName> serverNames = master.getServerManager().getOnlineServersList();
079    assertEquals(NSERVERS, serverNames.size());
080
081    hbckChore.choreForTesting();
082    Map<String, Pair<ServerName, List<ServerName>>> inconsistentRegions =
083      hbckChore.getLastReport().getInconsistentRegions();
084
085    // Test for case1: Master thought this region opened, but no regionserver reported it.
086    assertTrue(inconsistentRegions.containsKey(metaRegionName));
087    Pair<ServerName, List<ServerName>> pair = inconsistentRegions.get(metaRegionName);
088    ServerName locationInMeta = pair.getFirst();
089    List<ServerName> reportedRegionServers = pair.getSecond();
090    assertTrue(serverNames.contains(locationInMeta));
091    assertEquals(0, reportedRegionServers.size());
092
093    // Reported right region location. Then not in problematic regions.
094    am.reportOnlineRegions(locationInMeta, Collections.singleton(metaRegionNameAsBytes));
095    hbckChore.choreForTesting();
096    inconsistentRegions = hbckChore.getLastReport().getInconsistentRegions();
097    assertFalse(inconsistentRegions.containsKey(metaRegionName));
098  }
099
100  @Test
101  public void testForUserTable() throws Exception {
102    TableName tableName = TableName.valueOf("testForUserTable");
103    RegionInfo hri = createRegionInfo(tableName, 1);
104    String regionName = hri.getRegionNameAsString();
105    rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
106    Future<byte[]> future = submitProcedure(createAssignProcedure(hri));
107    waitOnFuture(future);
108
109    List<ServerName> serverNames = master.getServerManager().getOnlineServersList();
110    assertEquals(NSERVERS, serverNames.size());
111
112    // Test for case1: Master thought this region opened, but no regionserver reported it.
113    hbckChore.choreForTesting();
114    Map<String, Pair<ServerName, List<ServerName>>> inconsistentRegions =
115      hbckChore.getLastReport().getInconsistentRegions();
116    assertTrue(inconsistentRegions.containsKey(regionName));
117    Pair<ServerName, List<ServerName>> pair = inconsistentRegions.get(regionName);
118    ServerName locationInMeta = pair.getFirst();
119    List<ServerName> reportedRegionServers = pair.getSecond();
120    assertTrue(serverNames.contains(locationInMeta));
121    assertEquals(0, reportedRegionServers.size());
122
123    // Test for case2: Master thought this region opened on Server1, but regionserver reported
124    // Server2
125    final ServerName tempLocationInMeta = locationInMeta;
126    final ServerName anotherServer =
127      serverNames.stream().filter(s -> !s.equals(tempLocationInMeta)).findFirst().get();
128    am.reportOnlineRegions(anotherServer, Collections.singleton(hri.getRegionName()));
129    hbckChore.choreForTesting();
130    inconsistentRegions = hbckChore.getLastReport().getInconsistentRegions();
131    assertTrue(inconsistentRegions.containsKey(regionName));
132    pair = inconsistentRegions.get(regionName);
133    locationInMeta = pair.getFirst();
134    reportedRegionServers = pair.getSecond();
135    assertEquals(1, reportedRegionServers.size());
136    assertFalse(reportedRegionServers.contains(locationInMeta));
137    assertTrue(reportedRegionServers.contains(anotherServer));
138
139    // Test for case3: More than one regionservers reported opened this region.
140    am.reportOnlineRegions(locationInMeta, Collections.singleton(hri.getRegionName()));
141    hbckChore.choreForTesting();
142    inconsistentRegions = hbckChore.getLastReport().getInconsistentRegions();
143    assertTrue(inconsistentRegions.containsKey(regionName));
144    pair = inconsistentRegions.get(regionName);
145    locationInMeta = pair.getFirst();
146    reportedRegionServers = pair.getSecond();
147    assertEquals(2, reportedRegionServers.size());
148    assertTrue(reportedRegionServers.contains(locationInMeta));
149    assertTrue(reportedRegionServers.contains(anotherServer));
150
151    // Reported right region location, then not in inconsistent regions.
152    am.reportOnlineRegions(anotherServer, Collections.emptySet());
153    hbckChore.choreForTesting();
154    inconsistentRegions = hbckChore.getLastReport().getInconsistentRegions();
155    assertFalse(inconsistentRegions.containsKey(regionName));
156
157    // Test for case4: No region location for a previously reported region. Probably due to
158    // TRSP bug or bypass.
159    am.offlineRegion(hri);
160    hbckChore.choreForTesting();
161    inconsistentRegions = hbckChore.getLastReport().getInconsistentRegions();
162    assertTrue(inconsistentRegions.containsKey(regionName));
163  }
164
165  @Test
166  public void testForDisabledTable() throws Exception {
167    TableName tableName = TableName.valueOf("testForDisabledTable");
168    RegionInfo hri = createRegionInfo(tableName, 1);
169    String regionName = hri.getRegionNameAsString();
170    rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
171    Future<byte[]> future = submitProcedure(createAssignProcedure(hri));
172    waitOnFuture(future);
173
174    List<ServerName> serverNames = master.getServerManager().getOnlineServersList();
175    assertEquals(NSERVERS, serverNames.size());
176
177    hbckChore.choreForTesting();
178    Map<String, Pair<ServerName, List<ServerName>>> inconsistentRegions =
179      hbckChore.getLastReport().getInconsistentRegions();
180    assertTrue(inconsistentRegions.containsKey(regionName));
181    Pair<ServerName, List<ServerName>> pair = inconsistentRegions.get(regionName);
182    ServerName locationInMeta = pair.getFirst();
183    List<ServerName> reportedRegionServers = pair.getSecond();
184    assertTrue(serverNames.contains(locationInMeta));
185    assertEquals(0, reportedRegionServers.size());
186
187    // Set table state to disabled, then not in inconsistent regions.
188    TableStateManager tableStateManager = master.getTableStateManager();
189    Mockito.when(tableStateManager.isTableState(tableName, TableState.State.DISABLED))
190      .thenReturn(true);
191    hbckChore.choreForTesting();
192    inconsistentRegions = hbckChore.getLastReport().getInconsistentRegions();
193    assertFalse(inconsistentRegions.containsKey(regionName));
194  }
195
196  @Test
197  public void testForSplitParent() throws Exception {
198    TableName tableName = TableName.valueOf("testForSplitParent");
199    RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes(0))
200      .setEndKey(Bytes.toBytes(1)).setSplit(true).setOffline(true).setRegionId(0).build();
201    String regionName = hri.getEncodedName();
202    rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
203    Future<byte[]> future = submitProcedure(createAssignProcedure(hri));
204    waitOnFuture(future);
205
206    List<ServerName> serverNames = master.getServerManager().getOnlineServersList();
207    assertEquals(NSERVERS, serverNames.size());
208
209    hbckChore.choreForTesting();
210    Map<String, Pair<ServerName, List<ServerName>>> inconsistentRegions =
211      hbckChore.getLastReport().getInconsistentRegions();
212    assertFalse(inconsistentRegions.containsKey(regionName));
213  }
214
215  @Test
216  public void testOrphanRegionsOnFS() throws Exception {
217    TableName tableName = TableName.valueOf("testOrphanRegionsOnFS");
218    RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).build();
219    Configuration conf = util.getConfiguration();
220
221    hbckChore.choreForTesting();
222    assertEquals(0, hbckChore.getLastReport().getOrphanRegionsOnFS().size());
223
224    HRegion.createRegionDir(conf, regionInfo, CommonFSUtils.getRootDir(conf));
225    hbckChore.choreForTesting();
226    assertEquals(1, hbckChore.getLastReport().getOrphanRegionsOnFS().size());
227    assertTrue(
228      hbckChore.getLastReport().getOrphanRegionsOnFS().containsKey(regionInfo.getEncodedName()));
229
230    FSUtils.deleteRegionDir(conf, regionInfo);
231    hbckChore.choreForTesting();
232    assertEquals(0, hbckChore.getLastReport().getOrphanRegionsOnFS().size());
233  }
234
235  @Test
236  public void testChoreDisable() {
237    // The way to disable to chore is to set hbase.master.hbck.chore.interval <= 0
238    // When the interval is > 0, the chore should run.
239    Instant lastRunTime = Optional.ofNullable(hbckChore.getLastReport())
240      .map(HbckReport::getCheckingEndTimestamp).orElse(null);
241    hbckChore.choreForTesting();
242    Instant thisRunTime = Optional.ofNullable(hbckChore.getLastReport())
243      .map(HbckReport::getCheckingEndTimestamp).orElse(null);
244    assertNotNull(thisRunTime);
245    assertNotEquals(lastRunTime, thisRunTime);
246
247    // When the interval <= 0, the chore shouldn't run
248    master.getConfiguration().setInt("hbase.master.hbck.chore.interval", 0);
249    HbckChore hbckChoreWithChangedConf = new HbckChore(master);
250    hbckChoreWithChangedConf.choreForTesting();
251    assertNull(hbckChoreWithChangedConf.getLastReport());
252  }
253
254  @Test
255  public void testChoreSkipsForeignMetaTables() throws Exception {
256    FileSystem fs = master.getMasterFileSystem().getFileSystem();
257    Path rootDir = master.getMasterFileSystem().getRootDir();
258    String[] metaTables = { "meta_replica1", "meta" };
259    Path hbaseNamespaceDir = new Path(rootDir, HConstants.BASE_NAMESPACE_DIR + "/hbase");
260    fs.mkdirs(hbaseNamespaceDir);
261
262    for (String metaTable : metaTables) {
263      TableName tableName = TableName.valueOf("hbase", metaTable);
264      Path metaTableDir = new Path(hbaseNamespaceDir, metaTable);
265      fs.mkdirs(metaTableDir);
266      fs.mkdirs(new Path(metaTableDir, FSTableDescriptors.TABLEINFO_DIR));
267      fs.mkdirs(new Path(metaTableDir, "abcdef0123456789"));
268
269      TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
270        .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(HConstants.CATALOG_FAMILY)
271          .setMaxVersions(HConstants.DEFAULT_HBASE_META_VERSIONS).setInMemory(true)
272          .setBlocksize(HConstants.DEFAULT_HBASE_META_BLOCK_SIZE)
273          .setBloomFilterType(BloomType.ROWCOL).build())
274        .build();
275
276      Path tableDir = CommonFSUtils.getTableDir(rootDir, tableName);
277      FSTableDescriptors.createTableDescriptorForTableDirectory(fs, tableDir, tableDescriptor,
278        false);
279    }
280
281    assertTrue(hbckChore.runChore(), "HbckChore should run successfully");
282    HbckReport report = hbckChore.getLastReport();
283    assertNotNull(report, "HbckReport should not be null");
284    boolean hasForeignMetaOrphan = report.getOrphanRegionsOnFS().values().stream()
285      .anyMatch(path -> path.toString().contains("meta_replica1"));
286    assertFalse(hasForeignMetaOrphan, "HbckChore should not report foreign meta tables as orphans");
287  }
288}