Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.janitor;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import java.util.Arrays;
027import java.util.LinkedList;
028import java.util.List;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseTestingUtility;
031import org.apache.hadoop.hbase.HConstants;
032import org.apache.hadoop.hbase.MetaTableAccessor;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.client.Put;
035import org.apache.hadoop.hbase.client.RegionInfo;
036import org.apache.hadoop.hbase.client.RegionInfoBuilder;
037import org.apache.hadoop.hbase.master.assignment.RegionStateStore;
038import org.apache.hadoop.hbase.testclassification.LargeTests;
039import org.apache.hadoop.hbase.testclassification.MasterTests;
040import org.apache.hadoop.hbase.util.Bytes;
041import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
042import org.apache.hadoop.hbase.util.Pair;
043import org.junit.After;
044import org.junit.Before;
045import org.junit.ClassRule;
046import org.junit.Rule;
047import org.junit.Test;
048import org.junit.experimental.categories.Category;
049import org.junit.rules.TestName;
050import org.slf4j.Logger;
051import org.slf4j.LoggerFactory;
052
053@Category({ MasterTests.class, LargeTests.class })
054public class TestCatalogJanitorCluster {
055  private static final Logger LOG = LoggerFactory.getLogger(TestCatalogJanitorCluster.class);
056
057  @ClassRule
058  public static final HBaseClassTestRule CLASS_RULE =
059    HBaseClassTestRule.forClass(TestCatalogJanitorCluster.class);
060
061  @Rule
062  public final TestName name = new TestName();
063
064  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
065  private static final TableName T1 = TableName.valueOf("t1");
066  private static final TableName T2 = TableName.valueOf("t2");
067  private static final TableName T3 = TableName.valueOf("t3");
068  private static final TableName T4 = TableName.valueOf("t4");
069  private static final TableName T5 = TableName.valueOf("t5");
070
071  @Before
072  public void before() throws Exception {
073    TEST_UTIL.startMiniCluster();
074    TEST_UTIL.createMultiRegionTable(T1, new byte[][] { HConstants.CATALOG_FAMILY });
075    TEST_UTIL.createMultiRegionTable(T2, new byte[][] { HConstants.CATALOG_FAMILY });
076    TEST_UTIL.createMultiRegionTable(T3, new byte[][] { HConstants.CATALOG_FAMILY });
077
078    final byte[][] keysForT4 =
079      { Bytes.toBytes("aa"), Bytes.toBytes("bb"), Bytes.toBytes("cc"), Bytes.toBytes("dd") };
080
081    TEST_UTIL.createTable(T4, HConstants.CATALOG_FAMILY, keysForT4);
082
083    final byte[][] keysForT5 = { Bytes.toBytes("bb"), Bytes.toBytes("cc"), Bytes.toBytes("dd") };
084
085    TEST_UTIL.createTable(T5, HConstants.CATALOG_FAMILY, keysForT5);
086  }
087
088  @After
089  public void after() throws Exception {
090    TEST_UTIL.shutdownMiniCluster();
091  }
092
093  /**
094   * Fat method where we start with a fat hbase:meta and then gradually intro problems running
095   * catalogjanitor for each to ensure it triggers complaint. Do one big method because takes a
096   * while to build up the context we need. We create three tables and then make holes, overlaps,
097   * add unknown servers and empty out regioninfo columns. Each should up counts in the
098   * CatalogJanitor.Report produced.
099   */
100  @Test
101  public void testConsistency() throws IOException {
102    CatalogJanitor janitor = TEST_UTIL.getHBaseCluster().getMaster().getCatalogJanitor();
103    RegionStateStore regionStateStore =
104      TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStateStore();
105    janitor.scan();
106    CatalogJanitorReport report = janitor.getLastReport();
107    // Assert no problems.
108    assertTrue(report.isEmpty());
109    // Now remove first region in table t2 to see if catalogjanitor scan notices.
110    List<RegionInfo> t2Ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), T2);
111    regionStateStore.deleteRegion(t2Ris.get(0));
112    janitor.scan();
113    report = janitor.getLastReport();
114    assertFalse(report.isEmpty());
115    assertEquals(1, report.getHoles().size());
116    assertTrue(report.getHoles().get(0).getFirst().getTable()
117      .equals(RegionInfoBuilder.UNDEFINED.getTable()));
118    assertTrue(report.getHoles().get(0).getSecond().getTable().equals(T2));
119    assertEquals(0, report.getOverlaps().size());
120    // Next, add overlaps to first row in t3
121    List<RegionInfo> t3Ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), T3);
122    RegionInfo ri = t3Ris.get(0);
123    RegionInfo newRi1 = RegionInfoBuilder.newBuilder(ri.getTable())
124      .setStartKey(incrementRow(ri.getStartKey())).setEndKey(incrementRow(ri.getEndKey())).build();
125    Put p1 = MetaTableAccessor.makePutFromRegionInfo(newRi1, EnvironmentEdgeManager.currentTime());
126    RegionInfo newRi2 = RegionInfoBuilder.newBuilder(newRi1.getTable())
127      .setStartKey(incrementRow(newRi1.getStartKey())).setEndKey(incrementRow(newRi1.getEndKey()))
128      .build();
129    Put p2 = MetaTableAccessor.makePutFromRegionInfo(newRi2, EnvironmentEdgeManager.currentTime());
130    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(p1, p2));
131    janitor.scan();
132    report = janitor.getLastReport();
133    assertFalse(report.isEmpty());
134    // We added two overlaps so total three.
135    assertEquals(3, report.getOverlaps().size());
136    // Assert hole is still there.
137    assertEquals(1, report.getHoles().size());
138    // Assert other attributes are empty still.
139    assertTrue(report.getEmptyRegionInfo().isEmpty());
140    assertTrue(report.getUnknownServers().isEmpty());
141    // Now make bad server in t1.
142    List<RegionInfo> t1Ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), T1);
143    RegionInfo t1Ri1 = t1Ris.get(1);
144    Put pServer = new Put(t1Ri1.getRegionName());
145    pServer.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(0),
146      Bytes.toBytes("bad.server.example.org:1234"));
147    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(pServer));
148    janitor.scan();
149    report = janitor.getLastReport();
150    assertFalse(report.isEmpty());
151    assertEquals(1, report.getUnknownServers().size());
152    // Test what happens if we blow away an info:server row, if it is null. Should not kill CJ
153    // and we should log the row that had the problem. HBASE-23192. Just make sure we don't
154    // break if this happens.
155    LOG.info("Make null info:server");
156    Put emptyInfoServerPut = new Put(t1Ri1.getRegionName());
157    emptyInfoServerPut.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(0),
158      Bytes.toBytes(""));
159    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(emptyInfoServerPut));
160    janitor.scan();
161    report = janitor.getLastReport();
162    assertEquals(0, report.getUnknownServers().size());
163    // Mke an empty regioninfo in t1.
164    RegionInfo t1Ri2 = t1Ris.get(2);
165    Put pEmptyRI = new Put(t1Ri2.getRegionName());
166    pEmptyRI.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
167      HConstants.EMPTY_BYTE_ARRAY);
168    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(pEmptyRI));
169    janitor.scan();
170    report = janitor.getLastReport();
171    assertEquals(1, report.getEmptyRegionInfo().size());
172
173    int holesReported = report.getHoles().size();
174    int overlapsReported = report.getOverlaps().size();
175
176    // Test the case for T4
177    // r1: [aa, bb), r2: [cc, dd), r3: [a, cc)
178    // Make sure only overlaps and no holes are reported.
179    List<RegionInfo> t4Ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), T4);
180    // delete the region [bb, cc)
181    MetaTableAccessor.deleteRegionInfo(TEST_UTIL.getConnection(), t4Ris.get(2));
182
183    // add a new region [a, cc)
184    RegionInfo newRiT4 = RegionInfoBuilder.newBuilder(T4).setStartKey("a".getBytes())
185      .setEndKey("cc".getBytes()).build();
186    Put putForT4 =
187      MetaTableAccessor.makePutFromRegionInfo(newRiT4, EnvironmentEdgeManager.currentTime());
188    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(putForT4));
189
190    janitor.scan();
191    report = janitor.getLastReport();
192    // there is no new hole reported, 2 more overLaps added.
193    assertEquals(holesReported, report.getHoles().size());
194    assertEquals(overlapsReported + 2, report.getOverlaps().size());
195
196    holesReported = report.getHoles().size();
197    overlapsReported = report.getOverlaps().size();
198
199    // Test the case for T5
200    // r0: [, bb), r1: [a, g), r2: [bb, cc), r3: [dd, )
201    // Make sure only overlaps and no holes are reported.
202    List<RegionInfo> t5Ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), T5);
203    // delete the region [cc, dd)
204    MetaTableAccessor.deleteRegionInfo(TEST_UTIL.getConnection(), t5Ris.get(2));
205
206    // add a new region [a, g)
207    RegionInfo newRiT5 = RegionInfoBuilder.newBuilder(T5).setStartKey("a".getBytes())
208      .setEndKey("g".getBytes()).build();
209    Put putForT5 =
210      MetaTableAccessor.makePutFromRegionInfo(newRiT5, EnvironmentEdgeManager.currentTime());
211    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(putForT5));
212
213    janitor.scan();
214    report = janitor.getLastReport();
215    // there is no new hole reported, 3 more overLaps added.
216    // ([a, g), [, bb)), ([a, g), [bb, cc)), ([a, g), [dd, ))
217    assertEquals(holesReported, report.getHoles().size());
218    assertEquals(overlapsReported + 3, report.getOverlaps().size());
219  }
220
221  /**
222   * Take last byte and add one to it.
223   */
224  private static byte[] incrementRow(byte[] row) {
225    if (row.length == 0) {
226      return new byte[] { '0' };
227    }
228    row[row.length - 1] = (byte) (((int) row[row.length - 1]) + 1);
229    return row;
230  }
231
232  @Test
233  public void testHoles() throws IOException {
234    CatalogJanitor janitor = TEST_UTIL.getHBaseCluster().getMaster().getCatalogJanitor();
235
236    CatalogJanitorReport report = janitor.getLastReport();
237    // Assert no problems.
238    assertTrue(report.isEmpty());
239    // Verify start and end region holes
240    verifyCornerHoles(janitor, T1);
241    // Verify start and end region holes
242    verifyCornerHoles(janitor, T2);
243    verifyMiddleHole(janitor);
244    // Verify that MetaFixer is able to fix these holes
245    fixHoles(janitor);
246  }
247
248  private void fixHoles(CatalogJanitor janitor) throws IOException {
249    MetaFixer metaFixer = new MetaFixer(TEST_UTIL.getHBaseCluster().getMaster());
250    janitor.scan();
251    CatalogJanitorReport report = janitor.getLastReport();
252    // Verify total number of holes, 2 in t1 and t2 each and one in t3
253    assertEquals("Number of holes are not matching", 5, report.getHoles().size());
254    metaFixer.fix();
255    janitor.scan();
256    report = janitor.getLastReport();
257    assertEquals("Holes are not fixed", 0, report.getHoles().size());
258  }
259
260  private void verifyMiddleHole(CatalogJanitor janitor) throws IOException {
261    // Verify middle holes
262    RegionInfo firstRegion = getRegionInfo(T3, "".getBytes());
263    RegionInfo secondRegion = getRegionInfo(T3, "bbb".getBytes());
264    RegionInfo thirdRegion = getRegionInfo(T3, "ccc".getBytes());
265    MetaTableAccessor.deleteRegionInfo(TEST_UTIL.getConnection(), secondRegion);
266    LinkedList<Pair<RegionInfo, RegionInfo>> holes = getHoles(janitor, T3);
267    Pair<RegionInfo, RegionInfo> regionInfoRegionInfoPair = holes.getFirst();
268    assertTrue(regionInfoRegionInfoPair.getFirst().getTable().equals(T3));
269    assertTrue(regionInfoRegionInfoPair.getSecond().getTable().equals(T3));
270    assertTrue(
271      regionInfoRegionInfoPair.getFirst().getEncodedName().equals(firstRegion.getEncodedName()));
272    assertTrue(
273      regionInfoRegionInfoPair.getSecond().getEncodedName().equals(thirdRegion.getEncodedName()));
274  }
275
276  private void verifyCornerHoles(CatalogJanitor janitor, TableName tableName) throws IOException {
277    RegionInfo firstRegion = getRegionInfo(tableName, "".getBytes());
278    RegionInfo secondRegion = getRegionInfo(tableName, "bbb".getBytes());
279    MetaTableAccessor.deleteRegionInfo(TEST_UTIL.getConnection(), firstRegion);
280    LinkedList<Pair<RegionInfo, RegionInfo>> holes = getHoles(janitor, tableName);
281
282    assertEquals(1, holes.size());
283    Pair<RegionInfo, RegionInfo> regionInfoRegionInfoPair = holes.get(0);
284    assertTrue(regionInfoRegionInfoPair.getFirst().getTable()
285      .equals(RegionInfoBuilder.UNDEFINED.getTable()));
286    assertTrue(regionInfoRegionInfoPair.getSecond().getTable().equals(tableName));
287    assertTrue(
288      regionInfoRegionInfoPair.getSecond().getEncodedName().equals(secondRegion.getEncodedName()));
289
290    RegionInfo lastRegion = getRegionInfo(tableName, "zzz".getBytes());
291    RegionInfo secondLastRegion = getRegionInfo(tableName, "yyy".getBytes());
292    MetaTableAccessor.deleteRegionInfo(TEST_UTIL.getConnection(), lastRegion);
293    holes = getHoles(janitor, tableName);
294    assertEquals(2, holes.size());
295    regionInfoRegionInfoPair = holes.get(1);
296    assertTrue(regionInfoRegionInfoPair.getFirst().getEncodedName()
297      .equals(secondLastRegion.getEncodedName()));
298    assertTrue(regionInfoRegionInfoPair.getSecond().getTable()
299      .equals(RegionInfoBuilder.UNDEFINED.getTable()));
300  }
301
302  // Get Holes filter by table
303  private LinkedList<Pair<RegionInfo, RegionInfo>> getHoles(CatalogJanitor janitor,
304    TableName tableName) throws IOException {
305    janitor.scan();
306    CatalogJanitorReport lastReport = janitor.getLastReport();
307    assertFalse(lastReport.isEmpty());
308    LinkedList<Pair<RegionInfo, RegionInfo>> holes = new LinkedList<>();
309    for (Pair<RegionInfo, RegionInfo> hole : lastReport.getHoles()) {
310      if (
311        hole.getFirst().getTable().equals(tableName)
312          || hole.getSecond().getTable().equals(tableName)
313      ) {
314        holes.add(hole);
315      }
316    }
317    return holes;
318  }
319
320  private RegionInfo getRegionInfo(TableName tableName, byte[] row) throws IOException {
321    RegionInfo regionInfo =
322      TEST_UTIL.getConnection().getRegionLocator(tableName).getRegionLocation(row).getRegion();
323    assertNotNull(regionInfo);
324    return regionInfo;
325  }
326}