Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.janitor;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import java.util.Arrays;
027import java.util.LinkedList;
028import java.util.List;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseTestingUtility;
031import org.apache.hadoop.hbase.HConstants;
032import org.apache.hadoop.hbase.MetaTableAccessor;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.client.Put;
035import org.apache.hadoop.hbase.client.RegionInfo;
036import org.apache.hadoop.hbase.client.RegionInfoBuilder;
037import org.apache.hadoop.hbase.master.assignment.RegionStateStore;
038import org.apache.hadoop.hbase.testclassification.LargeTests;
039import org.apache.hadoop.hbase.testclassification.MasterTests;
040import org.apache.hadoop.hbase.util.Bytes;
041import org.apache.hadoop.hbase.util.Pair;
042import org.junit.After;
043import org.junit.Before;
044import org.junit.ClassRule;
045import org.junit.Rule;
046import org.junit.Test;
047import org.junit.experimental.categories.Category;
048import org.junit.rules.TestName;
049import org.slf4j.Logger;
050import org.slf4j.LoggerFactory;
051
052@Category({ MasterTests.class, LargeTests.class })
053public class TestCatalogJanitorCluster {
054  private static final Logger LOG = LoggerFactory.getLogger(TestCatalogJanitorCluster.class);
055
056  @ClassRule
057  public static final HBaseClassTestRule CLASS_RULE =
058    HBaseClassTestRule.forClass(TestCatalogJanitorCluster.class);
059
060  @Rule
061  public final TestName name = new TestName();
062
063  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
064  private static final TableName T1 = TableName.valueOf("t1");
065  private static final TableName T2 = TableName.valueOf("t2");
066  private static final TableName T3 = TableName.valueOf("t3");
067  private static final TableName T4 = TableName.valueOf("t4");
068  private static final TableName T5 = TableName.valueOf("t5");
069
070  @Before
071  public void before() throws Exception {
072    TEST_UTIL.startMiniCluster();
073    TEST_UTIL.createMultiRegionTable(T1, new byte[][] { HConstants.CATALOG_FAMILY });
074    TEST_UTIL.createMultiRegionTable(T2, new byte[][] { HConstants.CATALOG_FAMILY });
075    TEST_UTIL.createMultiRegionTable(T3, new byte[][] { HConstants.CATALOG_FAMILY });
076
077    final byte[][] keysForT4 =
078      { Bytes.toBytes("aa"), Bytes.toBytes("bb"), Bytes.toBytes("cc"), Bytes.toBytes("dd") };
079
080    TEST_UTIL.createTable(T4, HConstants.CATALOG_FAMILY, keysForT4);
081
082    final byte[][] keysForT5 = { Bytes.toBytes("bb"), Bytes.toBytes("cc"), Bytes.toBytes("dd") };
083
084    TEST_UTIL.createTable(T5, HConstants.CATALOG_FAMILY, keysForT5);
085  }
086
087  @After
088  public void after() throws Exception {
089    TEST_UTIL.shutdownMiniCluster();
090  }
091
092  /**
093   * Fat method where we start with a fat hbase:meta and then gradually intro problems running
094   * catalogjanitor for each to ensure it triggers complaint. Do one big method because takes a
095   * while to build up the context we need. We create three tables and then make holes, overlaps,
096   * add unknown servers and empty out regioninfo columns. Each should up counts in the
097   * CatalogJanitor.Report produced.
098   */
099  @Test
100  public void testConsistency() throws IOException {
101    CatalogJanitor janitor = TEST_UTIL.getHBaseCluster().getMaster().getCatalogJanitor();
102    RegionStateStore regionStateStore =
103      TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStateStore();
104    janitor.scan();
105    Report report = janitor.getLastReport();
106    // Assert no problems.
107    assertTrue(report.isEmpty());
108    // Now remove first region in table t2 to see if catalogjanitor scan notices.
109    List<RegionInfo> t2Ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), T2);
110    regionStateStore.deleteRegion(t2Ris.get(0));
111    janitor.scan();
112    report = janitor.getLastReport();
113    assertFalse(report.isEmpty());
114    assertEquals(1, report.getHoles().size());
115    assertTrue(report.getHoles().get(0).getFirst().getTable()
116      .equals(RegionInfoBuilder.UNDEFINED.getTable()));
117    assertTrue(report.getHoles().get(0).getSecond().getTable().equals(T2));
118    assertEquals(0, report.getOverlaps().size());
119    // Next, add overlaps to first row in t3
120    List<RegionInfo> t3Ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), T3);
121    RegionInfo ri = t3Ris.get(0);
122    RegionInfo newRi1 = RegionInfoBuilder.newBuilder(ri.getTable())
123      .setStartKey(incrementRow(ri.getStartKey())).setEndKey(incrementRow(ri.getEndKey())).build();
124    Put p1 = MetaTableAccessor.makePutFromRegionInfo(newRi1, System.currentTimeMillis());
125    RegionInfo newRi2 = RegionInfoBuilder.newBuilder(newRi1.getTable())
126      .setStartKey(incrementRow(newRi1.getStartKey())).setEndKey(incrementRow(newRi1.getEndKey()))
127      .build();
128    Put p2 = MetaTableAccessor.makePutFromRegionInfo(newRi2, System.currentTimeMillis());
129    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(p1, p2));
130    janitor.scan();
131    report = janitor.getLastReport();
132    assertFalse(report.isEmpty());
133    // We added two overlaps so total three.
134    assertEquals(3, report.getOverlaps().size());
135    // Assert hole is still there.
136    assertEquals(1, report.getHoles().size());
137    // Assert other attributes are empty still.
138    assertTrue(report.getEmptyRegionInfo().isEmpty());
139    assertTrue(report.getUnknownServers().isEmpty());
140    // Now make bad server in t1.
141    List<RegionInfo> t1Ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), T1);
142    RegionInfo t1Ri1 = t1Ris.get(1);
143    Put pServer = new Put(t1Ri1.getRegionName());
144    pServer.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(0),
145      Bytes.toBytes("bad.server.example.org:1234"));
146    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(pServer));
147    janitor.scan();
148    report = janitor.getLastReport();
149    assertFalse(report.isEmpty());
150    assertEquals(1, report.getUnknownServers().size());
151    // Test what happens if we blow away an info:server row, if it is null. Should not kill CJ
152    // and we should log the row that had the problem. HBASE-23192. Just make sure we don't
153    // break if this happens.
154    LOG.info("Make null info:server");
155    Put emptyInfoServerPut = new Put(t1Ri1.getRegionName());
156    emptyInfoServerPut.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(0),
157      Bytes.toBytes(""));
158    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(emptyInfoServerPut));
159    janitor.scan();
160    report = janitor.getLastReport();
161    assertEquals(0, report.getUnknownServers().size());
162    // Mke an empty regioninfo in t1.
163    RegionInfo t1Ri2 = t1Ris.get(2);
164    Put pEmptyRI = new Put(t1Ri2.getRegionName());
165    pEmptyRI.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
166      HConstants.EMPTY_BYTE_ARRAY);
167    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(pEmptyRI));
168    janitor.scan();
169    report = janitor.getLastReport();
170    assertEquals(1, report.getEmptyRegionInfo().size());
171
172    int holesReported = report.getHoles().size();
173    int overlapsReported = report.getOverlaps().size();
174
175    // Test the case for T4
176    // r1: [aa, bb), r2: [cc, dd), r3: [a, cc)
177    // Make sure only overlaps and no holes are reported.
178    List<RegionInfo> t4Ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), T4);
179    // delete the region [bb, cc)
180    MetaTableAccessor.deleteRegionInfo(TEST_UTIL.getConnection(), t4Ris.get(2));
181
182    // add a new region [a, cc)
183    RegionInfo newRiT4 = RegionInfoBuilder.newBuilder(T4).setStartKey("a".getBytes())
184      .setEndKey("cc".getBytes()).build();
185    Put putForT4 = MetaTableAccessor.makePutFromRegionInfo(newRiT4, System.currentTimeMillis());
186    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(putForT4));
187
188    janitor.scan();
189    report = janitor.getLastReport();
190    // there is no new hole reported, 2 more overLaps added.
191    assertEquals(holesReported, report.getHoles().size());
192    assertEquals(overlapsReported + 2, report.getOverlaps().size());
193
194    holesReported = report.getHoles().size();
195    overlapsReported = report.getOverlaps().size();
196
197    // Test the case for T5
198    // r0: [, bb), r1: [a, g), r2: [bb, cc), r3: [dd, )
199    // Make sure only overlaps and no holes are reported.
200    List<RegionInfo> t5Ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), T5);
201    // delete the region [cc, dd)
202    MetaTableAccessor.deleteRegionInfo(TEST_UTIL.getConnection(), t5Ris.get(2));
203
204    // add a new region [a, g)
205    RegionInfo newRiT5 = RegionInfoBuilder.newBuilder(T5).setStartKey("a".getBytes())
206      .setEndKey("g".getBytes()).build();
207    Put putForT5 = MetaTableAccessor.makePutFromRegionInfo(newRiT5, System.currentTimeMillis());
208    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(putForT5));
209
210    janitor.scan();
211    report = janitor.getLastReport();
212    // there is no new hole reported, 3 more overLaps added.
213    // ([a, g), [, bb)), ([a, g), [bb, cc)), ([a, g), [dd, ))
214    assertEquals(holesReported, report.getHoles().size());
215    assertEquals(overlapsReported + 3, report.getOverlaps().size());
216  }
217
218  /**
219   * Take last byte and add one to it.
220   */
221  private static byte[] incrementRow(byte[] row) {
222    if (row.length == 0) {
223      return new byte[] { '0' };
224    }
225    row[row.length - 1] = (byte) (((int) row[row.length - 1]) + 1);
226    return row;
227  }
228
229  @Test
230  public void testHoles() throws IOException {
231    CatalogJanitor janitor = TEST_UTIL.getHBaseCluster().getMaster().getCatalogJanitor();
232
233    Report report = janitor.getLastReport();
234    // Assert no problems.
235    assertTrue(report.isEmpty());
236    // Verify start and end region holes
237    verifyCornerHoles(janitor, T1);
238    // Verify start and end region holes
239    verifyCornerHoles(janitor, T2);
240    verifyMiddleHole(janitor);
241    // Verify that MetaFixer is able to fix these holes
242    fixHoles(janitor);
243  }
244
245  private void fixHoles(CatalogJanitor janitor) throws IOException {
246    MetaFixer metaFixer = new MetaFixer(TEST_UTIL.getHBaseCluster().getMaster());
247    janitor.scan();
248    Report report = janitor.getLastReport();
249    // Verify total number of holes, 2 in t1 and t2 each and one in t3
250    assertEquals("Number of holes are not matching", 5, report.getHoles().size());
251    metaFixer.fix();
252    janitor.scan();
253    report = janitor.getLastReport();
254    assertEquals("Holes are not fixed", 0, report.getHoles().size());
255  }
256
257  private void verifyMiddleHole(CatalogJanitor janitor) throws IOException {
258    // Verify middle holes
259    RegionInfo firstRegion = getRegionInfo(T3, "".getBytes());
260    RegionInfo secondRegion = getRegionInfo(T3, "bbb".getBytes());
261    RegionInfo thirdRegion = getRegionInfo(T3, "ccc".getBytes());
262    MetaTableAccessor.deleteRegionInfo(TEST_UTIL.getConnection(), secondRegion);
263    LinkedList<Pair<RegionInfo, RegionInfo>> holes = getHoles(janitor, T3);
264    Pair<RegionInfo, RegionInfo> regionInfoRegionInfoPair = holes.getFirst();
265    assertTrue(regionInfoRegionInfoPair.getFirst().getTable().equals(T3));
266    assertTrue(regionInfoRegionInfoPair.getSecond().getTable().equals(T3));
267    assertTrue(
268      regionInfoRegionInfoPair.getFirst().getEncodedName().equals(firstRegion.getEncodedName()));
269    assertTrue(
270      regionInfoRegionInfoPair.getSecond().getEncodedName().equals(thirdRegion.getEncodedName()));
271  }
272
273  private void verifyCornerHoles(CatalogJanitor janitor, TableName tableName) throws IOException {
274    RegionInfo firstRegion = getRegionInfo(tableName, "".getBytes());
275    RegionInfo secondRegion = getRegionInfo(tableName, "bbb".getBytes());
276    MetaTableAccessor.deleteRegionInfo(TEST_UTIL.getConnection(), firstRegion);
277    LinkedList<Pair<RegionInfo, RegionInfo>> holes = getHoles(janitor, tableName);
278
279    assertEquals(1, holes.size());
280    Pair<RegionInfo, RegionInfo> regionInfoRegionInfoPair = holes.get(0);
281    assertTrue(regionInfoRegionInfoPair.getFirst().getTable()
282      .equals(RegionInfoBuilder.UNDEFINED.getTable()));
283    assertTrue(regionInfoRegionInfoPair.getSecond().getTable().equals(tableName));
284    assertTrue(
285      regionInfoRegionInfoPair.getSecond().getEncodedName().equals(secondRegion.getEncodedName()));
286
287    RegionInfo lastRegion = getRegionInfo(tableName, "zzz".getBytes());
288    RegionInfo secondLastRegion = getRegionInfo(tableName, "yyy".getBytes());
289    MetaTableAccessor.deleteRegionInfo(TEST_UTIL.getConnection(), lastRegion);
290    holes = getHoles(janitor, tableName);
291    assertEquals(2, holes.size());
292    regionInfoRegionInfoPair = holes.get(1);
293    assertTrue(regionInfoRegionInfoPair.getFirst().getEncodedName()
294      .equals(secondLastRegion.getEncodedName()));
295    assertTrue(regionInfoRegionInfoPair.getSecond().getTable()
296      .equals(RegionInfoBuilder.UNDEFINED.getTable()));
297  }
298
299  // Get Holes filter by table
300  private LinkedList<Pair<RegionInfo, RegionInfo>> getHoles(CatalogJanitor janitor,
301    TableName tableName) throws IOException {
302    janitor.scan();
303    Report lastReport = janitor.getLastReport();
304    assertFalse(lastReport.isEmpty());
305    LinkedList<Pair<RegionInfo, RegionInfo>> holes = new LinkedList<>();
306    for (Pair<RegionInfo, RegionInfo> hole : lastReport.getHoles()) {
307      if (hole.getFirst().getTable().equals(tableName) ||
308        hole.getSecond().getTable().equals(tableName)) {
309        holes.add(hole);
310      }
311    }
312    return holes;
313  }
314
315  private RegionInfo getRegionInfo(TableName tableName, byte[] row) throws IOException {
316    RegionInfo regionInfo =
317      TEST_UTIL.getConnection().getRegionLocator(tableName).getRegionLocation(row).getRegion();
318    assertNotNull(regionInfo);
319    return regionInfo;
320  }
321}