Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.procedure;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertNotEquals;
022import static org.junit.jupiter.api.Assertions.assertThrows;
023
024import java.util.ArrayList;
025import java.util.HashMap;
026import java.util.HashSet;
027import java.util.List;
028import java.util.Map;
029import java.util.Set;
030import org.apache.hadoop.conf.Configuration;
031import org.apache.hadoop.hbase.HBaseTestingUtil;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.TableNotFoundException;
034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
035import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
036import org.apache.hadoop.hbase.client.RegionInfo;
037import org.apache.hadoop.hbase.client.TableDescriptor;
038import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
039import org.apache.hadoop.hbase.master.HMaster;
040import org.apache.hadoop.hbase.regionserver.HRegion;
041import org.apache.hadoop.hbase.regionserver.MetricsRegionWrapperImpl;
042import org.apache.hadoop.hbase.testclassification.MasterTests;
043import org.apache.hadoop.hbase.testclassification.MediumTests;
044import org.apache.hadoop.hbase.util.Bytes;
045import org.junit.jupiter.api.AfterAll;
046import org.junit.jupiter.api.BeforeAll;
047import org.junit.jupiter.api.Tag;
048import org.junit.jupiter.api.Test;
049
050@Tag(MasterTests.TAG)
051@Tag(MediumTests.TAG)
052public class TestReopenTableRegionsIntegration {
053
054  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
055  private static final TableName TABLE_NAME = TableName.valueOf("testLazyUpdateReopen");
056  private static final byte[] CF = Bytes.toBytes("cf");
057
058  @BeforeAll
059  public static void setupCluster() throws Exception {
060    Configuration conf = UTIL.getConfiguration();
061    conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
062    UTIL.startMiniCluster(1);
063  }
064
065  @AfterAll
066  public static void tearDown() throws Exception {
067    UTIL.shutdownMiniCluster();
068  }
069
070  @Test
071  public void testLazyUpdateThenReopenUpdatesTableDescriptorHash() throws Exception {
072    // Step 1: Create table with column family and 3 regions
073    ColumnFamilyDescriptor cfd =
074      ColumnFamilyDescriptorBuilder.newBuilder(CF).setMaxVersions(1).build();
075
076    TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLE_NAME).setColumnFamily(cfd)
077      .setMaxFileSize(100 * 1024 * 1024L).build();
078
079    UTIL.getAdmin().createTable(td, Bytes.toBytes("a"), Bytes.toBytes("z"), 3);
080    UTIL.waitTableAvailable(TABLE_NAME);
081
082    try {
083      // Step 2: Capture initial tableDescriptorHash from all regions
084      List<HRegion> regions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
085      assertEquals(3, regions.size(), "Expected 3 regions");
086
087      Map<byte[], String> initialHashes = new HashMap<>();
088
089      for (HRegion region : regions) {
090        String hash;
091        try (MetricsRegionWrapperImpl wrapper = new MetricsRegionWrapperImpl(region)) {
092          hash = wrapper.getTableDescriptorHash();
093        }
094        initialHashes.put(region.getRegionInfo().getRegionName(), hash);
095      }
096
097      // Verify all regions have same hash
098      Set<String> uniqueHashes = new HashSet<>(initialHashes.values());
099      assertEquals(1, uniqueHashes.size(), "All regions should have same hash");
100      String initialHash = uniqueHashes.iterator().next();
101
102      // Step 3: Perform lazy table descriptor update
103      ColumnFamilyDescriptor newCfd =
104        ColumnFamilyDescriptorBuilder.newBuilder(cfd).setMaxVersions(5).build();
105
106      TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td).modifyColumnFamily(newCfd)
107        .setMaxFileSize(200 * 1024 * 1024L).build();
108
109      // Perform lazy update (reopenRegions = false)
110      UTIL.getAdmin().modifyTableAsync(newTd, false).get();
111
112      // Wait for modification to complete
113      UTIL.waitFor(30000, () -> {
114        try {
115          TableDescriptor currentTd = UTIL.getAdmin().getDescriptor(TABLE_NAME);
116          return currentTd.getMaxFileSize() == 200 * 1024 * 1024L;
117        } catch (Exception e) {
118          return false;
119        }
120      });
121
122      // Step 4: Verify tableDescriptorHash has NOT changed in region metrics
123      List<HRegion> regionsAfterLazyUpdate = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
124      for (HRegion region : regionsAfterLazyUpdate) {
125        String currentHash;
126        try (MetricsRegionWrapperImpl wrapper = new MetricsRegionWrapperImpl(region)) {
127          currentHash = wrapper.getTableDescriptorHash();
128        }
129        assertEquals(initialHashes.get(region.getRegionInfo().getRegionName()), currentHash,
130          "Hash should NOT change without region reopen");
131      }
132
133      // Verify the table descriptor itself has changed
134      TableDescriptor currentTd = UTIL.getAdmin().getDescriptor(TABLE_NAME);
135      String newDescriptorHash = currentTd.getDescriptorHash();
136      assertNotEquals(initialHash, newDescriptorHash, "Table descriptor should have new hash");
137
138      // Step 5: Use new Admin API to reopen all regions
139      UTIL.getAdmin().reopenTableRegions(TABLE_NAME);
140
141      // Wait for all regions to be reopened
142      UTIL.waitFor(60000, () -> {
143        try {
144          List<HRegion> currentRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
145          if (currentRegions.size() != 3) {
146            return false;
147          }
148
149          // Check if all regions now have the new hash
150          for (HRegion region : currentRegions) {
151            String hash;
152            try (MetricsRegionWrapperImpl wrapper = new MetricsRegionWrapperImpl(region)) {
153              hash = wrapper.getTableDescriptorHash();
154            }
155            if (hash.equals(initialHash)) {
156              return false;
157            }
158          }
159          return true;
160        } catch (Exception e) {
161          return false;
162        }
163      });
164
165      // Step 6: Verify tableDescriptorHash HAS changed in all region metrics
166      List<HRegion> reopenedRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
167      assertEquals(3, reopenedRegions.size(), "Should still have 3 regions");
168
169      for (HRegion region : reopenedRegions) {
170        String currentHash;
171        try (MetricsRegionWrapperImpl wrapper = new MetricsRegionWrapperImpl(region)) {
172          currentHash = wrapper.getTableDescriptorHash();
173        }
174        assertNotEquals(initialHash, currentHash, "Hash SHOULD change after region reopen");
175        assertEquals(newDescriptorHash, currentHash, "Hash should match current table descriptor");
176      }
177
178      // Verify all regions show the same new hash
179      Set<String> newHashes = new HashSet<>();
180      for (HRegion region : reopenedRegions) {
181        try (MetricsRegionWrapperImpl wrapper = new MetricsRegionWrapperImpl(region)) {
182          newHashes.add(wrapper.getTableDescriptorHash());
183        }
184      }
185      assertEquals(1, newHashes.size(), "All regions should have same new hash");
186
187    } finally {
188      UTIL.deleteTable(TABLE_NAME);
189    }
190  }
191
192  @Test
193  public void testLazyUpdateThenReopenSpecificRegions() throws Exception {
194    TableName tableName = TableName.valueOf("testSpecificRegionsReopen");
195
196    // Step 1: Create table with 5 regions
197    ColumnFamilyDescriptor cfd =
198      ColumnFamilyDescriptorBuilder.newBuilder(CF).setMaxVersions(1).build();
199
200    TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(cfd)
201      .setMaxFileSize(100 * 1024 * 1024L).build();
202
203    UTIL.getAdmin().createTable(td, Bytes.toBytes("a"), Bytes.toBytes("z"), 5);
204    UTIL.waitTableAvailable(tableName);
205
206    try {
207      // Step 2: Capture initial hashes
208      List<HRegion> regions = UTIL.getHBaseCluster().getRegions(tableName);
209      assertEquals(5, regions.size(), "Expected 5 regions");
210
211      Map<byte[], String> initialHashes = new HashMap<>();
212
213      for (HRegion region : regions) {
214        String hash;
215        try (MetricsRegionWrapperImpl wrapper = new MetricsRegionWrapperImpl(region)) {
216          hash = wrapper.getTableDescriptorHash();
217        }
218        initialHashes.put(region.getRegionInfo().getRegionName(), hash);
219      }
220
221      String initialHash = initialHashes.values().iterator().next();
222
223      // Step 3: Perform lazy update
224      ColumnFamilyDescriptor newCfd =
225        ColumnFamilyDescriptorBuilder.newBuilder(cfd).setMaxVersions(10).build();
226
227      TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td).modifyColumnFamily(newCfd)
228        .setMaxFileSize(300 * 1024 * 1024L).build();
229
230      UTIL.getAdmin().modifyTableAsync(newTd, false).get();
231
232      UTIL.waitFor(30000, () -> {
233        try {
234          TableDescriptor currentTd = UTIL.getAdmin().getDescriptor(tableName);
235          return currentTd.getMaxFileSize() == 300 * 1024 * 1024L;
236        } catch (Exception e) {
237          return false;
238        }
239      });
240
241      String newDescriptorHash = UTIL.getAdmin().getDescriptor(tableName).getDescriptorHash();
242
243      // Step 4: Reopen only first 2 regions
244      List<RegionInfo> regionsToReopen = new ArrayList<>();
245      regionsToReopen.add(regions.get(0).getRegionInfo());
246      regionsToReopen.add(regions.get(1).getRegionInfo());
247
248      UTIL.getAdmin().reopenTableRegions(tableName, regionsToReopen);
249
250      // Wait for those regions to reopen
251      UTIL.waitFor(60000, () -> {
252        try {
253          List<HRegion> currentRegions = UTIL.getHBaseCluster().getRegions(tableName);
254          int newHashCount = 0;
255          for (HRegion region : currentRegions) {
256            String hash;
257            try (MetricsRegionWrapperImpl wrapper = new MetricsRegionWrapperImpl(region)) {
258              hash = wrapper.getTableDescriptorHash();
259            }
260            if (!hash.equals(initialHash)) {
261              newHashCount++;
262            }
263          }
264          return newHashCount >= 2;
265        } catch (Exception e) {
266          return false;
267        }
268      });
269
270      // Step 5: Verify only reopened regions have new hash
271      List<HRegion> regionsAfterFirstReopen = UTIL.getHBaseCluster().getRegions(tableName);
272      int newHashCount = 0;
273      int oldHashCount = 0;
274
275      for (HRegion region : regionsAfterFirstReopen) {
276        String currentHash;
277        try (MetricsRegionWrapperImpl wrapper = new MetricsRegionWrapperImpl(region)) {
278          currentHash = wrapper.getTableDescriptorHash();
279        }
280
281        if (currentHash.equals(newDescriptorHash)) {
282          newHashCount++;
283        } else if (currentHash.equals(initialHash)) {
284          oldHashCount++;
285        }
286      }
287
288      assertEquals(2, newHashCount, "Should have 2 regions with new hash");
289      assertEquals(3, oldHashCount, "Should have 3 regions with old hash");
290
291      // Step 6: Reopen remaining regions
292      List<RegionInfo> remainingRegions = new ArrayList<>();
293      for (int i = 2; i < regions.size(); i++) {
294        remainingRegions.add(regions.get(i).getRegionInfo());
295      }
296
297      UTIL.getAdmin().reopenTableRegions(tableName, remainingRegions);
298
299      // Wait for all regions to have new hash
300      UTIL.waitFor(60000, () -> {
301        try {
302          List<HRegion> currentRegions = UTIL.getHBaseCluster().getRegions(tableName);
303          for (HRegion region : currentRegions) {
304            String hash;
305            try (MetricsRegionWrapperImpl wrapper = new MetricsRegionWrapperImpl(region)) {
306              hash = wrapper.getTableDescriptorHash();
307            }
308            if (!hash.equals(newDescriptorHash)) {
309              return false;
310            }
311          }
312          return true;
313        } catch (Exception e) {
314          return false;
315        }
316      });
317
318      // Step 7: Verify all regions now have new hash
319      List<HRegion> finalRegions = UTIL.getHBaseCluster().getRegions(tableName);
320      for (HRegion region : finalRegions) {
321        String currentHash;
322        try (MetricsRegionWrapperImpl wrapper = new MetricsRegionWrapperImpl(region)) {
323          currentHash = wrapper.getTableDescriptorHash();
324        }
325
326        assertEquals(newDescriptorHash, currentHash, "All regions should now have new hash");
327      }
328
329    } finally {
330      UTIL.deleteTable(tableName);
331    }
332  }
333
334  @Test
335  public void testReopenTableRegionsFailsCleanlyWhenDescriptorMissing() throws Exception {
336    TableName tableName = TableName.valueOf("testReopenTableRegionsFailsWhenDescriptorMissing");
337    TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName)
338      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(CF)).build();
339    UTIL.getAdmin().createTable(td, Bytes.toBytes("a"), Bytes.toBytes("z"), 3);
340    UTIL.waitTableAvailable(tableName);
341
342    HMaster master = UTIL.getMiniHBaseCluster().getMaster();
343    TableDescriptor removedDescriptor = null;
344    try {
345      removedDescriptor = master.getTableDescriptors().remove(tableName);
346      assertThrows(TableNotFoundException.class,
347        () -> UTIL.getAdmin().reopenTableRegions(tableName));
348    } finally {
349      if (removedDescriptor != null && master.getTableDescriptors().get(tableName) == null) {
350        master.getTableDescriptors().update(removedDescriptor);
351      }
352      UTIL.deleteTable(tableName);
353    }
354  }
355}