Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.junit.jupiter.api.Assertions.assertArrayEquals;
021import static org.junit.jupiter.api.Assertions.assertEquals;
022import static org.junit.jupiter.api.Assertions.assertFalse;
023import static org.junit.jupiter.api.Assertions.assertNotNull;
024import static org.junit.jupiter.api.Assertions.assertTrue;
025import static org.junit.jupiter.api.Assertions.fail;
026
027import java.io.IOException;
028import java.util.List;
029import java.util.Map;
030import java.util.concurrent.atomic.AtomicReference;
031import org.apache.hadoop.conf.Configuration;
032import org.apache.hadoop.fs.FileSystem;
033import org.apache.hadoop.fs.Path;
034import org.apache.hadoop.hbase.CatalogFamilyFormat;
035import org.apache.hadoop.hbase.ClientMetaTableAccessor;
036import org.apache.hadoop.hbase.HBaseTestingUtil;
037import org.apache.hadoop.hbase.HConstants;
038import org.apache.hadoop.hbase.MetaTableAccessor;
039import org.apache.hadoop.hbase.PleaseHoldException;
040import org.apache.hadoop.hbase.ServerName;
041import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
042import org.apache.hadoop.hbase.TableName;
043import org.apache.hadoop.hbase.UnknownRegionException;
044import org.apache.hadoop.hbase.client.Admin;
045import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
046import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
047import org.apache.hadoop.hbase.client.RegionInfo;
048import org.apache.hadoop.hbase.client.RegionInfoBuilder;
049import org.apache.hadoop.hbase.client.Result;
050import org.apache.hadoop.hbase.client.Table;
051import org.apache.hadoop.hbase.client.TableDescriptor;
052import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
053import org.apache.hadoop.hbase.client.TableState;
054import org.apache.hadoop.hbase.testclassification.MasterTests;
055import org.apache.hadoop.hbase.testclassification.MediumTests;
056import org.apache.hadoop.hbase.util.Bytes;
057import org.apache.hadoop.hbase.util.HBaseFsck;
058import org.apache.hadoop.hbase.util.Pair;
059import org.apache.hadoop.hbase.util.Threads;
060import org.apache.hadoop.util.StringUtils;
061import org.junit.jupiter.api.AfterAll;
062import org.junit.jupiter.api.BeforeAll;
063import org.junit.jupiter.api.BeforeEach;
064import org.junit.jupiter.api.Tag;
065import org.junit.jupiter.api.Test;
066import org.junit.jupiter.api.TestInfo;
067import org.slf4j.Logger;
068import org.slf4j.LoggerFactory;
069
070import org.apache.hbase.thirdparty.com.google.common.base.Joiner;
071
072@Tag(MasterTests.TAG)
073@Tag(MediumTests.TAG)
074public class TestMaster {
075
076  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
077  private static final Logger LOG = LoggerFactory.getLogger(TestMaster.class);
078  private static final TableName TABLENAME = TableName.valueOf("TestMaster");
079  private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
080  private static Admin admin;
081  private String testMethodName;
082
083  @BeforeEach
084  public void setTestMethod(TestInfo testInfo) {
085    testMethodName = testInfo.getTestMethod().get().getName();
086  }
087
088  @BeforeAll
089  public static void beforeAllTests() throws Exception {
090    // we will retry operations when PleaseHoldException is thrown
091    TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
092    // Set hbase.min.version.move.system.tables as version 0 so that
093    // testMoveRegionWhenNotInitialized never fails even if hbase-default has valid default
094    // value present for production use-case.
095    // See HBASE-22923 for details.
096    TEST_UTIL.getConfiguration().set("hbase.min.version.move.system.tables", "0.0.0");
097    // Start a cluster of two regionservers.
098    TEST_UTIL.startMiniCluster(2);
099    admin = TEST_UTIL.getAdmin();
100  }
101
102  @AfterAll
103  public static void afterAllTests() throws Exception {
104    TEST_UTIL.shutdownMiniCluster();
105  }
106
107  /**
108   * Return the region and current deployment for the region containing the given row. If the region
109   * cannot be found, returns null. If it is found, but not currently deployed, the second element
110   * of the pair may be null.
111   */
112  private Pair<RegionInfo, ServerName> getTableRegionForRow(HMaster master, TableName tableName,
113    byte[] rowKey) throws IOException {
114    final AtomicReference<Pair<RegionInfo, ServerName>> result = new AtomicReference<>(null);
115
116    ClientMetaTableAccessor.Visitor visitor = new ClientMetaTableAccessor.Visitor() {
117      @Override
118      public boolean visit(Result data) throws IOException {
119        if (data == null || data.size() <= 0) {
120          return true;
121        }
122        Pair<RegionInfo, ServerName> pair = new Pair<>(CatalogFamilyFormat.getRegionInfo(data),
123          CatalogFamilyFormat.getServerName(data, 0));
124        if (!pair.getFirst().getTable().equals(tableName)) {
125          return false;
126        }
127        result.set(pair);
128        return true;
129      }
130    };
131
132    MetaTableAccessor.scanMeta(master.getConnection(), visitor, tableName, rowKey, 1);
133    return result.get();
134  }
135
136  @Test
137  @SuppressWarnings("deprecation")
138  public void testMasterOpsWhileSplitting() throws Exception {
139    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
140    HMaster m = cluster.getMaster();
141
142    try (Table ht = TEST_UTIL.createTable(TABLENAME, FAMILYNAME)) {
143      assertTrue(m.getTableStateManager().isTableState(TABLENAME, TableState.State.ENABLED));
144      TEST_UTIL.loadTable(ht, FAMILYNAME, false);
145    }
146
147    List<Pair<RegionInfo, ServerName>> tableRegions =
148      MetaTableAccessor.getTableRegionsAndLocations(m.getConnection(), TABLENAME);
149    LOG.info("Regions after load: " + Joiner.on(',').join(tableRegions));
150    assertEquals(1, tableRegions.size());
151    assertArrayEquals(HConstants.EMPTY_START_ROW, tableRegions.get(0).getFirst().getStartKey());
152    assertArrayEquals(HConstants.EMPTY_END_ROW, tableRegions.get(0).getFirst().getEndKey());
153
154    // Now trigger a split and stop when the split is in progress
155    LOG.info("Splitting table");
156    TEST_UTIL.getAdmin().split(TABLENAME);
157
158    LOG.info("Making sure we can call getTableRegions while opening");
159    while (tableRegions.size() < 3) {
160      tableRegions =
161        MetaTableAccessor.getTableRegionsAndLocations(m.getConnection(), TABLENAME, false);
162      Thread.sleep(100);
163    }
164    LOG.info("Regions: " + Joiner.on(',').join(tableRegions));
165    // We have three regions because one is split-in-progress
166    assertEquals(3, tableRegions.size());
167    LOG.info("Making sure we can call getTableRegionClosest while opening");
168    Pair<RegionInfo, ServerName> pair = getTableRegionForRow(m, TABLENAME, Bytes.toBytes("cde"));
169    LOG.info("Result is: " + pair);
170    Pair<RegionInfo, ServerName> tableRegionFromName =
171      MetaTableAccessor.getRegion(m.getConnection(), pair.getFirst().getRegionName());
172    assertTrue(RegionInfo.COMPARATOR.compare(tableRegionFromName.getFirst(), pair.getFirst()) == 0);
173  }
174
175  @Test
176  public void testMoveRegionWhenNotInitialized() {
177    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
178    HMaster m = cluster.getMaster();
179    try {
180      m.setInitialized(false); // fake it, set back later
181      RegionInfo meta = RegionInfoBuilder.FIRST_META_REGIONINFO;
182      m.move(meta.getEncodedNameAsBytes(), null);
183      fail("Region should not be moved since master is not initialized");
184    } catch (IOException ioe) {
185      assertTrue(ioe instanceof PleaseHoldException);
186    } finally {
187      m.setInitialized(true);
188    }
189  }
190
191  @Test
192  public void testMoveThrowsUnknownRegionException() throws IOException {
193    final TableName tableName = TableName.valueOf(testMethodName);
194    TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(tableName);
195    ColumnFamilyDescriptor columnFamilyDescriptor =
196      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("value")).build();
197    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
198
199    admin.createTable(tableDescriptorBuilder.build());
200    try {
201      RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("A"))
202        .setEndKey(Bytes.toBytes("Z")).build();
203      admin.move(hri.getEncodedNameAsBytes());
204      fail("Region should not be moved since it is fake");
205    } catch (IOException ioe) {
206      assertTrue(ioe instanceof UnknownRegionException);
207    } finally {
208      TEST_UTIL.deleteTable(tableName);
209    }
210  }
211
212  @Test
213  public void testMoveThrowsPleaseHoldException() throws IOException {
214    final TableName tableName = TableName.valueOf(testMethodName);
215    HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
216    TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(tableName);
217    ColumnFamilyDescriptor columnFamilyDescriptor =
218      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("value")).build();
219    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
220
221    admin.createTable(tableDescriptorBuilder.build());
222    try {
223      List<RegionInfo> tableRegions = admin.getRegions(tableName);
224
225      master.setInitialized(false); // fake it, set back later
226      admin.move(tableRegions.get(0).getEncodedNameAsBytes());
227      fail("Region should not be moved since master is not initialized");
228    } catch (IOException ioe) {
229      assertTrue(StringUtils.stringifyException(ioe).contains("PleaseHoldException"));
230    } finally {
231      master.setInitialized(true);
232      TEST_UTIL.deleteTable(tableName);
233    }
234  }
235
236  @Test
237  public void testFlushedSequenceIdPersistLoad() throws Exception {
238    Configuration conf = TEST_UTIL.getConfiguration();
239    int msgInterval = conf.getInt("hbase.regionserver.msginterval", 100);
240    // insert some data into META
241    TableName tableName = TableName.valueOf("testFlushSeqId");
242    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
243      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(Bytes.toBytes("cf"))).build();
244    Table table = TEST_UTIL.createTable(tableDescriptor, null);
245    // flush META region
246    TEST_UTIL.flush(TableName.META_TABLE_NAME);
247    // wait for regionserver report
248    Threads.sleep(msgInterval * 2);
249    // record flush seqid before cluster shutdown
250    Map<byte[], Long> regionMapBefore =
251      TEST_UTIL.getHBaseCluster().getMaster().getServerManager().getFlushedSequenceIdByRegion();
252    // restart hbase cluster which will cause flushed sequence id persist and reload
253    TEST_UTIL.getMiniHBaseCluster().shutdown();
254    TEST_UTIL.restartHBaseCluster(2);
255    TEST_UTIL.waitUntilNoRegionsInTransition();
256    // Post HBASE-30335 the master seeds flushedSequenceIdByRegion on OPEN via
257    // merge(openSeqNum, Math::max). openSeqNum is monotonic across close/open cycles, so a region
258    // reopened after cluster restart may carry a strictly higher value than what was persisted.
259    // The preserved invariant is: every region persisted at shutdown is loaded on restart, and no
260    // watermark regresses.
261    Map<byte[], Long> regionMapAfter =
262      TEST_UTIL.getHBaseCluster().getMaster().getServerManager().getFlushedSequenceIdByRegion();
263    assertEquals(regionMapBefore.size(), regionMapAfter.size());
264    for (Map.Entry<byte[], Long> before : regionMapBefore.entrySet()) {
265      Long after = regionMapAfter.get(before.getKey());
266      assertNotNull(after,
267        "region missing after restart: " + Bytes.toStringBinary(before.getKey()));
268      assertTrue(after >= before.getValue(),
269        "flushedSequenceId regressed across restart for region "
270          + Bytes.toStringBinary(before.getKey()) + " before=" + before.getValue() + " after="
271          + after);
272    }
273  }
274
275  @Test
276  public void testBlockingHbkc1WithLockFile() throws IOException {
277    // This is how the patch to the lock file is created inside in HBaseFsck. Too hard to use its
278    // actual method without disturbing HBaseFsck... Do the below mimic instead.
279    Path hbckLockPath =
280      new Path(HBaseFsck.getTmpDir(TEST_UTIL.getConfiguration()), HBaseFsck.HBCK_LOCK_FILE);
281    FileSystem fs = TEST_UTIL.getTestFileSystem();
282    assertTrue(fs.exists(hbckLockPath));
283    TEST_UTIL.getMiniHBaseCluster()
284      .killMaster(TEST_UTIL.getMiniHBaseCluster().getMaster().getServerName());
285    assertTrue(fs.exists(hbckLockPath));
286    TEST_UTIL.getMiniHBaseCluster().startMaster();
287    TEST_UTIL.waitFor(30000, () -> TEST_UTIL.getMiniHBaseCluster().getMaster() != null
288      && TEST_UTIL.getMiniHBaseCluster().getMaster().isInitialized());
289    assertTrue(fs.exists(hbckLockPath));
290    // Start a second Master. Should be fine.
291    TEST_UTIL.getMiniHBaseCluster().startMaster();
292    assertTrue(fs.exists(hbckLockPath));
293    fs.delete(hbckLockPath, true);
294    assertFalse(fs.exists(hbckLockPath));
295    // Kill all Masters.
296    TEST_UTIL.getMiniHBaseCluster().getLiveMasterThreads().stream()
297      .map(sn -> sn.getMaster().getServerName()).forEach(sn -> {
298        try {
299          TEST_UTIL.getMiniHBaseCluster().killMaster(sn);
300        } catch (IOException e) {
301          e.printStackTrace();
302        }
303      });
304    // Start a new one.
305    TEST_UTIL.getMiniHBaseCluster().startMaster();
306    TEST_UTIL.waitFor(30000, () -> TEST_UTIL.getMiniHBaseCluster().getMaster() != null
307      && TEST_UTIL.getMiniHBaseCluster().getMaster().isInitialized());
308    // Assert lock gets put in place again.
309    assertTrue(fs.exists(hbckLockPath));
310  }
311
312  @Test
313  public void testInstallShutdownHook() throws IOException {
314    // Test for HBASE-26951
315    assertTrue(TEST_UTIL.getHBaseCluster().getMaster().isShutdownHookInstalled());
316  }
317}