Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.procedure;
019
020import static org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.assertProcNotFailed;
021import static org.junit.jupiter.api.Assertions.assertEquals;
022import static org.junit.jupiter.api.Assertions.assertFalse;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024
025import java.io.IOException;
026import java.util.Arrays;
027import java.util.List;
028import java.util.stream.Stream;
029import org.apache.hadoop.fs.FSDataOutputStream;
030import org.apache.hadoop.fs.FileSystem;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.hbase.HBaseTestingUtil;
033import org.apache.hadoop.hbase.HConstants;
034import org.apache.hadoop.hbase.MetaTableAccessor;
035import org.apache.hadoop.hbase.TableName;
036import org.apache.hadoop.hbase.client.Admin;
037import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
038import org.apache.hadoop.hbase.client.Put;
039import org.apache.hadoop.hbase.client.RegionInfo;
040import org.apache.hadoop.hbase.client.RegionInfoBuilder;
041import org.apache.hadoop.hbase.client.Table;
042import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
043import org.apache.hadoop.hbase.client.TableState;
044import org.apache.hadoop.hbase.master.HMaster;
045import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
046import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
047import org.apache.hadoop.hbase.regionserver.HRegionServer;
048import org.apache.hadoop.hbase.testclassification.LargeTests;
049import org.apache.hadoop.hbase.testclassification.MasterTests;
050import org.apache.hadoop.hbase.util.Bytes;
051import org.apache.hadoop.hbase.util.CommonFSUtils;
052import org.junit.jupiter.api.AfterEach;
053import org.junit.jupiter.api.BeforeEach;
054import org.junit.jupiter.api.Tag;
055import org.junit.jupiter.api.Test;
056
057@Tag(MasterTests.TAG)
058@Tag(LargeTests.TAG)
059public class TestRefreshMetaProcedureIntegration {
060
061  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
062  private Admin admin;
063  private ProcedureExecutor<MasterProcedureEnv> procExecutor;
064  private HMaster master;
065  private HRegionServer regionServer;
066
067  @BeforeEach
068  public void setup() throws Exception {
069    // Start in active mode
070    TEST_UTIL.getConfiguration().setBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY, false);
071
072    TEST_UTIL.startMiniCluster();
073    admin = TEST_UTIL.getAdmin();
074    procExecutor = TEST_UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
075    master = TEST_UTIL.getHBaseCluster().getMaster();
076    regionServer = TEST_UTIL.getHBaseCluster().getRegionServerThreads().get(0).getRegionServer();
077  }
078
079  @AfterEach
080  public void tearDown() throws Exception {
081    if (admin != null) {
082      admin.close();
083    }
084    TEST_UTIL.shutdownMiniCluster();
085  }
086
087  @Test
088  public void testRestoreMissingRegionInMeta() throws Exception {
089
090    TableName tableName = TableName.valueOf("replicaTestTable");
091
092    createTableWithData(tableName);
093
094    List<RegionInfo> activeRegions = admin.getRegions(tableName);
095    assertTrue(activeRegions.size() >= 2, "Should have at least 2 regions after split");
096
097    Table metaTable = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
098    RegionInfo regionToRemove = activeRegions.get(0);
099    admin.unassign(regionToRemove.getRegionName(), false);
100    Thread.sleep(1000);
101
102    org.apache.hadoop.hbase.client.Delete delete =
103      new org.apache.hadoop.hbase.client.Delete(regionToRemove.getRegionName());
104    metaTable.delete(delete);
105    metaTable.close();
106
107    List<RegionInfo> regionsAfterDrift = admin.getRegions(tableName);
108    assertEquals(activeRegions.size() - 1, regionsAfterDrift.size(),
109      "Should have one less region in meta after simulating drift");
110
111    setReadOnlyMode(true);
112
113    boolean writeBlocked = false;
114    try {
115      Table readOnlyTable = TEST_UTIL.getConnection().getTable(tableName);
116      Put testPut = new Put(Bytes.toBytes("test_readonly"));
117      testPut.addColumn(Bytes.toBytes("cf1"), Bytes.toBytes("qual"), Bytes.toBytes("should_fail"));
118      readOnlyTable.put(testPut);
119      readOnlyTable.close();
120    } catch (Exception e) {
121      if (e.getMessage().contains("Operation not allowed in Read-Only Mode")) {
122        writeBlocked = true;
123      }
124    }
125    assertTrue(writeBlocked, "Write operations should be blocked in read-only mode");
126
127    Long procId = admin.refreshMeta();
128
129    waitForProcedureCompletion(procId);
130
131    List<RegionInfo> regionsAfterRefresh = admin.getRegions(tableName);
132    assertEquals(activeRegions.size(), regionsAfterRefresh.size(),
133      "Missing regions should be restored by refresh_meta");
134
135    boolean regionRestored = regionsAfterRefresh.stream()
136      .anyMatch(r -> r.getRegionNameAsString().equals(regionToRemove.getRegionNameAsString()));
137    assertTrue(regionRestored, "Missing region should be restored by refresh_meta");
138
139    setReadOnlyMode(false);
140
141    Table activeTable = TEST_UTIL.getConnection().getTable(tableName);
142    Put testPut = new Put(Bytes.toBytes("test_active_again"));
143    testPut.addColumn(Bytes.toBytes("cf1"), Bytes.toBytes("qual"),
144      Bytes.toBytes("active_mode_again"));
145    activeTable.put(testPut);
146    activeTable.close();
147  }
148
149  @Test
150  public void testPhantomTableCleanup() throws Exception {
151    TableName table1 = TableName.valueOf("table1");
152    TableName phantomTable = TableName.valueOf("phantomTable");
153    createTableWithData(table1);
154    createTableWithData(phantomTable);
155
156    assertTrue(admin.getRegions(table1).size() >= 2, "Table1 should have multiple regions");
157    assertTrue(admin.getRegions(phantomTable).size() >= 2,
158      "phantomTable should have multiple regions");
159
160    deleteTableFromFilesystem(phantomTable);
161    List<TableName> tablesBeforeRefresh = Arrays.asList(admin.listTableNames());
162    assertTrue(tablesBeforeRefresh.contains(phantomTable),
163      "phantomTable should still be listed before refresh_meta");
164    assertTrue(tablesBeforeRefresh.contains(table1), "Table1 should still be listed");
165
166    setReadOnlyMode(true);
167    Long procId = admin.refreshMeta();
168    waitForProcedureCompletion(procId);
169
170    List<TableName> tablesAfterRefresh = Arrays.asList(admin.listTableNames());
171
172    assertFalse(tablesAfterRefresh.contains(phantomTable),
173      "phantomTable should be removed after refresh_meta");
174    assertTrue(tablesAfterRefresh.contains(table1), "Table1 should still be listed");
175    assertTrue(admin.getRegions(phantomTable).isEmpty(),
176      "phantomTable should have no regions after refresh_meta");
177    setReadOnlyMode(false);
178  }
179
180  @Test
181  public void testRestoreTableStateForOrphanRegions() throws Exception {
182    TableName tableName = TableName.valueOf("t1");
183    createTableInFilesystem(tableName);
184
185    assertEquals(0, Stream.of(admin.listTableNames()).filter(tn -> tn.equals(tableName)).count(),
186      "No tables should exist");
187
188    setReadOnlyMode(true);
189    Long procId = admin.refreshMeta();
190    waitForProcedureCompletion(procId);
191
192    TableState tableState = MetaTableAccessor.getTableState(admin.getConnection(), tableName);
193    assert tableState != null;
194    assertEquals(TableState.State.ENABLED, tableState.getState(), "Table state should be ENABLED");
195    assertEquals(1, Stream.of(admin.listTableNames()).filter(tn -> tn.equals(tableName)).count(),
196      "The list should show the new table from the FS");
197    assertFalse(admin.getRegions(tableName).isEmpty(), "Should have at least 1 region");
198    setReadOnlyMode(false);
199  }
200
201  private void createTableInFilesystem(TableName tableName) throws IOException {
202    FileSystem fs = TEST_UTIL.getTestFileSystem();
203    Path rootDir = CommonFSUtils.getRootDir(TEST_UTIL.getConfiguration());
204    Path tableDir = CommonFSUtils.getTableDir(rootDir, tableName);
205    fs.mkdirs(tableDir);
206
207    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
208    TEST_UTIL.getHBaseCluster().getMaster().getTableDescriptors()
209      .update(builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf1")).build(), false);
210
211    Path regionDir = new Path(tableDir, "dab6d1e1c88787c13b97647f11b2c907");
212    Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
213    fs.mkdirs(regionDir);
214
215    RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).setStartKey(new byte[0])
216      .setEndKey(new byte[0]).setRegionId(1757100253228L).build();
217    byte[] regionInfoContent = RegionInfo.toDelimitedByteArray(regionInfo);
218    try (FSDataOutputStream out = fs.create(regionInfoFile, true)) {
219      out.write(regionInfoContent);
220    }
221  }
222
223  private void deleteTableFromFilesystem(TableName tableName) throws IOException {
224    FileSystem fs = TEST_UTIL.getTestFileSystem();
225    Path rootDir = CommonFSUtils.getRootDir(TEST_UTIL.getConfiguration());
226    Path tableDir = CommonFSUtils.getTableDir(rootDir, tableName);
227    if (fs.exists(tableDir)) {
228      fs.delete(tableDir, true);
229    }
230  }
231
232  private void createTableWithData(TableName tableName) throws Exception {
233    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
234    builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf1"));
235    byte[] splitKeyBytes = Bytes.toBytes("split_key");
236    admin.createTable(builder.build(), new byte[][] { splitKeyBytes });
237    TEST_UTIL.waitTableAvailable(tableName);
238    try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
239      for (int i = 0; i < 100; i++) {
240        Put put = new Put(Bytes.toBytes("row_" + String.format("%05d", i)));
241        put.addColumn(Bytes.toBytes("cf1"), Bytes.toBytes("qual"), Bytes.toBytes("value_" + i));
242        table.put(put);
243      }
244    }
245    admin.flush(tableName);
246  }
247
248  private void waitForProcedureCompletion(Long procId) {
249    assertTrue(procId > 0, "Procedure ID should be positive");
250    TEST_UTIL.waitFor(1000, () -> {
251      try {
252        return procExecutor.isFinished(procId);
253      } catch (Exception e) {
254        return false;
255      }
256    });
257    assertProcNotFailed(procExecutor.getResult(procId));
258  }
259
260  private void setReadOnlyMode(boolean isReadOnly) {
261    TEST_UTIL.getConfiguration().setBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY,
262      isReadOnly);
263    notifyConfigurationObservers();
264  }
265
266  private void notifyConfigurationObservers() {
267    master.getConfigurationManager().notifyAllObservers(TEST_UTIL.getConfiguration());
268    regionServer.getConfigurationManager().notifyAllObservers(TEST_UTIL.getConfiguration());
269  }
270}