Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static java.util.stream.Collectors.toList;
021import static org.apache.hadoop.hbase.HConstants.EMPTY_END_ROW;
022import static org.apache.hadoop.hbase.HConstants.EMPTY_START_ROW;
023import static org.apache.hadoop.hbase.client.RegionReplicaTestHelper.testLocator;
024import static org.hamcrest.CoreMatchers.instanceOf;
025import static org.hamcrest.MatcherAssert.assertThat;
026import static org.junit.jupiter.api.Assertions.assertArrayEquals;
027import static org.junit.jupiter.api.Assertions.assertEquals;
028import static org.junit.jupiter.api.Assertions.assertNotNull;
029import static org.junit.jupiter.api.Assertions.assertNull;
030import static org.junit.jupiter.api.Assertions.assertSame;
031
032import java.io.IOException;
033import java.util.Arrays;
034import java.util.List;
035import java.util.concurrent.CompletableFuture;
036import java.util.concurrent.ExecutionException;
037import java.util.concurrent.ThreadLocalRandom;
038import java.util.stream.IntStream;
039import java.util.stream.Stream;
040import org.apache.hadoop.conf.Configuration;
041import org.apache.hadoop.hbase.CatalogReplicaMode;
042import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate;
043import org.apache.hadoop.hbase.HBaseTestingUtil;
044import org.apache.hadoop.hbase.HRegionLocation;
045import org.apache.hadoop.hbase.MetaTableAccessor;
046import org.apache.hadoop.hbase.NotServingRegionException;
047import org.apache.hadoop.hbase.RegionLocations;
048import org.apache.hadoop.hbase.ServerName;
049import org.apache.hadoop.hbase.TableName;
050import org.apache.hadoop.hbase.TableNotFoundException;
051import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
052import org.apache.hadoop.hbase.client.RegionReplicaTestHelper.Locator;
053import org.apache.hadoop.hbase.security.User;
054import org.apache.hadoop.hbase.testclassification.ClientTests;
055import org.apache.hadoop.hbase.testclassification.MediumTests;
056import org.apache.hadoop.hbase.util.Bytes;
057import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
058import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
059import org.junit.jupiter.api.AfterAll;
060import org.junit.jupiter.api.AfterEach;
061import org.junit.jupiter.api.BeforeAll;
062import org.junit.jupiter.api.BeforeEach;
063import org.junit.jupiter.api.Tag;
064import org.junit.jupiter.api.TestTemplate;
065import org.junit.jupiter.params.provider.Arguments;
066
067import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
068import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
069
070@Tag(MediumTests.TAG)
071@Tag(ClientTests.TAG)
072@HBaseParameterizedTestTemplate(name = "[{index}]: metaReplicaMode = {0}")
073public class TestAsyncNonMetaRegionLocator {
074
075  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
076
077  private static final TableName TABLE_NAME = TableName.valueOf("async");
078
079  private static byte[] FAMILY = Bytes.toBytes("cf");
080  private static final int NB_SERVERS = 4;
081  private static final int NUM_OF_META_REPLICA = NB_SERVERS - 1;
082
083  private static byte[][] SPLIT_KEYS;
084
085  private AsyncConnectionImpl conn;
086  private AsyncNonMetaRegionLocator locator;
087
088  private final CatalogReplicaMode metaReplicaMode;
089
090  public TestAsyncNonMetaRegionLocator(CatalogReplicaMode metaReplicaMode) {
091    this.metaReplicaMode = metaReplicaMode;
092  }
093
094  @BeforeAll
095  public static void setUp() throws Exception {
096    Configuration conf = TEST_UTIL.getConfiguration();
097    // Enable hbase:meta replication.
098    conf.setBoolean(ServerRegionReplicaUtil.REGION_REPLICA_REPLICATION_CATALOG_CONF_KEY, true);
099    conf.setLong("replication.source.sleepforretries", 10); // 10 ms
100
101    TEST_UTIL.startMiniCluster(NB_SERVERS);
102    Admin admin = TEST_UTIL.getAdmin();
103    admin.balancerSwitch(false, true);
104
105    // Enable hbase:meta replication.
106    HBaseTestingUtil.setReplicas(admin, TableName.META_TABLE_NAME, NUM_OF_META_REPLICA);
107    TEST_UTIL.waitFor(30000,
108      () -> TEST_UTIL.getMiniHBaseCluster().getRegions(TableName.META_TABLE_NAME).size()
109          >= NUM_OF_META_REPLICA);
110
111    SPLIT_KEYS = new byte[8][];
112    for (int i = 111; i < 999; i += 111) {
113      SPLIT_KEYS[i / 111 - 1] = Bytes.toBytes(String.format("%03d", i));
114    }
115  }
116
117  @AfterAll
118  public static void tearDown() throws Exception {
119    TEST_UTIL.shutdownMiniCluster();
120  }
121
122  @BeforeEach
123  public void setUpBeforeTest() throws InterruptedException, ExecutionException, IOException {
124    Configuration c = new Configuration(TEST_UTIL.getConfiguration());
125    // Enable meta replica LoadBalance mode for this connection.
126    c.set(RegionLocator.LOCATOR_META_REPLICAS_MODE, metaReplicaMode.toString());
127    ConnectionRegistry registry =
128      ConnectionRegistryFactory.create(TEST_UTIL.getConfiguration(), User.getCurrent());
129    conn =
130      new AsyncConnectionImpl(c, registry, registry.getClusterId().get(), null, User.getCurrent());
131    locator = new AsyncNonMetaRegionLocator(conn, AsyncConnectionImpl.RETRY_TIMER);
132  }
133
134  @AfterEach
135  public void tearDownAfterTest() throws IOException {
136    Admin admin = TEST_UTIL.getAdmin();
137    if (admin.tableExists(TABLE_NAME)) {
138      if (admin.isTableEnabled(TABLE_NAME)) {
139        admin.disableTable(TABLE_NAME);
140      }
141      admin.deleteTable(TABLE_NAME);
142    }
143    Closeables.close(conn, true);
144  }
145
146  public static Stream<Arguments> parameters() {
147    return Stream.of(Arguments.of(CatalogReplicaMode.NONE),
148      Arguments.of(CatalogReplicaMode.LOAD_BALANCE));
149  }
150
151  private void createSingleRegionTable() throws IOException, InterruptedException {
152    TEST_UTIL.createTable(TABLE_NAME, FAMILY);
153    TEST_UTIL.waitTableAvailable(TABLE_NAME);
154  }
155
156  private CompletableFuture<HRegionLocation> getDefaultRegionLocation(TableName tableName,
157    byte[] row, RegionLocateType locateType, boolean reload) {
158    return locator
159      .getRegionLocations(tableName, row, RegionReplicaUtil.DEFAULT_REPLICA_ID, locateType, reload)
160      .thenApply(RegionLocations::getDefaultRegionLocation);
161  }
162
163  @TestTemplate
164  public void testNoTable() throws InterruptedException {
165    for (RegionLocateType locateType : RegionLocateType.values()) {
166      try {
167        getDefaultRegionLocation(TABLE_NAME, EMPTY_START_ROW, locateType, false).get();
168      } catch (ExecutionException e) {
169        assertThat(e.getCause(), instanceOf(TableNotFoundException.class));
170      }
171    }
172  }
173
174  @TestTemplate
175  public void testDisableTable() throws IOException, InterruptedException {
176    createSingleRegionTable();
177    TEST_UTIL.getAdmin().disableTable(TABLE_NAME);
178    for (RegionLocateType locateType : RegionLocateType.values()) {
179      try {
180        getDefaultRegionLocation(TABLE_NAME, EMPTY_START_ROW, locateType, false).get();
181      } catch (ExecutionException e) {
182        assertThat(e.getCause(), instanceOf(TableNotFoundException.class));
183      }
184    }
185  }
186
187  private void assertLocEquals(byte[] startKey, byte[] endKey, ServerName serverName,
188    HRegionLocation loc) {
189    RegionInfo info = loc.getRegion();
190    assertEquals(TABLE_NAME, info.getTable());
191    assertArrayEquals(startKey, info.getStartKey());
192    assertArrayEquals(endKey, info.getEndKey());
193    assertEquals(serverName, loc.getServerName());
194  }
195
196  @TestTemplate
197  public void testSingleRegionTable() throws IOException, InterruptedException, ExecutionException {
198    createSingleRegionTable();
199    ServerName serverName = TEST_UTIL.getRSForFirstRegionInTable(TABLE_NAME).getServerName();
200    for (RegionLocateType locateType : RegionLocateType.values()) {
201      assertLocEquals(EMPTY_START_ROW, EMPTY_END_ROW, serverName,
202        getDefaultRegionLocation(TABLE_NAME, EMPTY_START_ROW, locateType, false).get());
203    }
204    byte[] key = new byte[ThreadLocalRandom.current().nextInt(128)];
205    Bytes.random(key);
206    for (RegionLocateType locateType : RegionLocateType.values()) {
207      assertLocEquals(EMPTY_START_ROW, EMPTY_END_ROW, serverName,
208        getDefaultRegionLocation(TABLE_NAME, key, locateType, false).get());
209    }
210  }
211
212  private void createMultiRegionTable() throws IOException, InterruptedException {
213    TEST_UTIL.createTable(TABLE_NAME, FAMILY, SPLIT_KEYS);
214    TEST_UTIL.waitTableAvailable(TABLE_NAME);
215  }
216
217  private static byte[][] getStartKeys() {
218    byte[][] startKeys = new byte[SPLIT_KEYS.length + 1][];
219    startKeys[0] = EMPTY_START_ROW;
220    System.arraycopy(SPLIT_KEYS, 0, startKeys, 1, SPLIT_KEYS.length);
221    return startKeys;
222  }
223
224  private static byte[][] getEndKeys() {
225    byte[][] endKeys = Arrays.copyOf(SPLIT_KEYS, SPLIT_KEYS.length + 1);
226    endKeys[endKeys.length - 1] = EMPTY_START_ROW;
227    return endKeys;
228  }
229
230  private ServerName[] getLocations(byte[][] startKeys) {
231    ServerName[] serverNames = new ServerName[startKeys.length];
232    TEST_UTIL.getHBaseCluster().getRegionServerThreads().stream().map(t -> t.getRegionServer())
233      .forEach(rs -> {
234        rs.getRegions(TABLE_NAME).forEach(r -> {
235          serverNames[Arrays.binarySearch(startKeys, r.getRegionInfo().getStartKey(),
236            Bytes::compareTo)] = rs.getServerName();
237        });
238      });
239    return serverNames;
240  }
241
242  @TestTemplate
243  public void testMultiRegionTable() throws IOException, InterruptedException {
244    createMultiRegionTable();
245    byte[][] startKeys = getStartKeys();
246    ServerName[] serverNames = getLocations(startKeys);
247    IntStream.range(0, 2).forEach(n -> IntStream.range(0, startKeys.length).forEach(i -> {
248      try {
249        assertLocEquals(startKeys[i], i == startKeys.length - 1 ? EMPTY_END_ROW : startKeys[i + 1],
250          serverNames[i],
251          getDefaultRegionLocation(TABLE_NAME, startKeys[i], RegionLocateType.CURRENT, false)
252            .get());
253      } catch (InterruptedException | ExecutionException e) {
254        throw new RuntimeException(e);
255      }
256    }));
257
258    locator.clearCache(TABLE_NAME);
259    IntStream.range(0, 2).forEach(n -> IntStream.range(0, startKeys.length).forEach(i -> {
260      try {
261        assertLocEquals(startKeys[i], i == startKeys.length - 1 ? EMPTY_END_ROW : startKeys[i + 1],
262          serverNames[i],
263          getDefaultRegionLocation(TABLE_NAME, startKeys[i], RegionLocateType.AFTER, false).get());
264      } catch (InterruptedException | ExecutionException e) {
265        throw new RuntimeException(e);
266      }
267    }));
268
269    locator.clearCache(TABLE_NAME);
270    byte[][] endKeys = getEndKeys();
271    IntStream.range(0, 2).forEach(
272      n -> IntStream.range(0, endKeys.length).map(i -> endKeys.length - 1 - i).forEach(i -> {
273        try {
274          assertLocEquals(i == 0 ? EMPTY_START_ROW : endKeys[i - 1], endKeys[i], serverNames[i],
275            getDefaultRegionLocation(TABLE_NAME, endKeys[i], RegionLocateType.BEFORE, false).get());
276        } catch (InterruptedException | ExecutionException e) {
277          throw new RuntimeException(e);
278        }
279      }));
280  }
281
282  @TestTemplate
283  public void testRegionMove() throws IOException, InterruptedException, ExecutionException {
284    createSingleRegionTable();
285    ServerName serverName = TEST_UTIL.getRSForFirstRegionInTable(TABLE_NAME).getServerName();
286    HRegionLocation loc =
287      getDefaultRegionLocation(TABLE_NAME, EMPTY_START_ROW, RegionLocateType.CURRENT, false).get();
288    assertLocEquals(EMPTY_START_ROW, EMPTY_END_ROW, serverName, loc);
289    ServerName newServerName = TEST_UTIL.getHBaseCluster().getRegionServerThreads().stream()
290      .map(t -> t.getRegionServer().getServerName()).filter(sn -> !sn.equals(serverName)).findAny()
291      .get();
292
293    TEST_UTIL.getAdmin().move(Bytes.toBytes(loc.getRegion().getEncodedName()), newServerName);
294    while (
295      !TEST_UTIL.getRSForFirstRegionInTable(TABLE_NAME).getServerName().equals(newServerName)
296    ) {
297      Thread.sleep(100);
298    }
299    // Should be same as it is in cache
300    assertSame(loc,
301      getDefaultRegionLocation(TABLE_NAME, EMPTY_START_ROW, RegionLocateType.CURRENT, false).get());
302    locator.updateCachedLocationOnError(loc, null);
303    // null error will not trigger a cache cleanup
304    assertSame(loc,
305      getDefaultRegionLocation(TABLE_NAME, EMPTY_START_ROW, RegionLocateType.CURRENT, false).get());
306    locator.updateCachedLocationOnError(loc, new NotServingRegionException());
307    assertLocEquals(EMPTY_START_ROW, EMPTY_END_ROW, newServerName,
308      getDefaultRegionLocation(TABLE_NAME, EMPTY_START_ROW, RegionLocateType.CURRENT, false).get());
309  }
310
311  // usually locate after will return the same result, so we add a test to make it return different
312  // result.
313  @TestTemplate
314  public void testLocateAfter() throws IOException, InterruptedException, ExecutionException {
315    byte[] row = Bytes.toBytes("1");
316    byte[] splitKey = Arrays.copyOf(row, 2);
317    TEST_UTIL.createTable(TABLE_NAME, FAMILY, new byte[][] { splitKey });
318    TEST_UTIL.waitTableAvailable(TABLE_NAME);
319    HRegionLocation currentLoc =
320      getDefaultRegionLocation(TABLE_NAME, row, RegionLocateType.CURRENT, false).get();
321    ServerName currentServerName = TEST_UTIL.getRSForFirstRegionInTable(TABLE_NAME).getServerName();
322    assertLocEquals(EMPTY_START_ROW, splitKey, currentServerName, currentLoc);
323
324    HRegionLocation afterLoc =
325      getDefaultRegionLocation(TABLE_NAME, row, RegionLocateType.AFTER, false).get();
326    ServerName afterServerName =
327      TEST_UTIL.getHBaseCluster().getRegionServerThreads().stream().map(t -> t.getRegionServer())
328        .filter(rs -> rs.getRegions(TABLE_NAME).stream()
329          .anyMatch(r -> Bytes.equals(splitKey, r.getRegionInfo().getStartKey())))
330        .findAny().get().getServerName();
331    assertLocEquals(splitKey, EMPTY_END_ROW, afterServerName, afterLoc);
332
333    assertSame(afterLoc,
334      getDefaultRegionLocation(TABLE_NAME, row, RegionLocateType.AFTER, false).get());
335  }
336
337  // For HBASE-17402
338  @TestTemplate
339  public void testConcurrentLocate() throws IOException, InterruptedException, ExecutionException {
340    createMultiRegionTable();
341    byte[][] startKeys = getStartKeys();
342    byte[][] endKeys = getEndKeys();
343    ServerName[] serverNames = getLocations(startKeys);
344    for (int i = 0; i < 100; i++) {
345      locator.clearCache(TABLE_NAME);
346      List<CompletableFuture<HRegionLocation>> futures =
347        IntStream.range(0, 1000).mapToObj(n -> String.format("%03d", n)).map(s -> Bytes.toBytes(s))
348          .map(r -> getDefaultRegionLocation(TABLE_NAME, r, RegionLocateType.CURRENT, false))
349          .collect(toList());
350      for (int j = 0; j < 1000; j++) {
351        int index = Math.min(8, j / 111);
352        assertLocEquals(startKeys[index], endKeys[index], serverNames[index], futures.get(j).get());
353      }
354    }
355  }
356
357  @TestTemplate
358  public void testReload() throws Exception {
359    createSingleRegionTable();
360    ServerName serverName = TEST_UTIL.getRSForFirstRegionInTable(TABLE_NAME).getServerName();
361    for (RegionLocateType locateType : RegionLocateType.values()) {
362      assertLocEquals(EMPTY_START_ROW, EMPTY_END_ROW, serverName,
363        getDefaultRegionLocation(TABLE_NAME, EMPTY_START_ROW, locateType, false).get());
364    }
365    ServerName newServerName = TEST_UTIL.getHBaseCluster().getRegionServerThreads().stream()
366      .map(t -> t.getRegionServer().getServerName()).filter(sn -> !sn.equals(serverName)).findAny()
367      .get();
368    Admin admin = TEST_UTIL.getAdmin();
369    RegionInfo region = admin.getRegions(TABLE_NAME).stream().findAny().get();
370    admin.move(region.getEncodedNameAsBytes(), newServerName);
371    TEST_UTIL.waitFor(30000, new ExplainingPredicate<Exception>() {
372
373      @Override
374      public boolean evaluate() throws Exception {
375        ServerName newServerName = TEST_UTIL.getRSForFirstRegionInTable(TABLE_NAME).getServerName();
376        return newServerName != null && !newServerName.equals(serverName);
377      }
378
379      @Override
380      public String explainFailure() throws Exception {
381        return region.getRegionNameAsString() + " is still on " + serverName;
382      }
383
384    });
385    // The cached location will not change
386    for (RegionLocateType locateType : RegionLocateType.values()) {
387      assertLocEquals(EMPTY_START_ROW, EMPTY_END_ROW, serverName,
388        getDefaultRegionLocation(TABLE_NAME, EMPTY_START_ROW, locateType, false).get());
389    }
390    // should get the new location when reload = true
391    // when meta replica LoadBalance mode is enabled, it may delay a bit.
392    TEST_UTIL.waitFor(3000, new ExplainingPredicate<Exception>() {
393      @Override
394      public boolean evaluate() throws Exception {
395        HRegionLocation loc =
396          getDefaultRegionLocation(TABLE_NAME, EMPTY_START_ROW, RegionLocateType.CURRENT, true)
397            .get();
398        return newServerName.equals(loc.getServerName());
399      }
400
401      @Override
402      public String explainFailure() throws Exception {
403        return "New location does not show up in meta (replica) region";
404      }
405    });
406
407    // the cached location should be replaced
408    for (RegionLocateType locateType : RegionLocateType.values()) {
409      assertLocEquals(EMPTY_START_ROW, EMPTY_END_ROW, newServerName,
410        getDefaultRegionLocation(TABLE_NAME, EMPTY_START_ROW, locateType, false).get());
411    }
412  }
413
414  // Testcase for HBASE-20822
415  @TestTemplate
416  public void testLocateBeforeLastRegion()
417    throws IOException, InterruptedException, ExecutionException {
418    createMultiRegionTable();
419    getDefaultRegionLocation(TABLE_NAME, SPLIT_KEYS[0], RegionLocateType.CURRENT, false).join();
420    HRegionLocation loc =
421      getDefaultRegionLocation(TABLE_NAME, EMPTY_END_ROW, RegionLocateType.BEFORE, false).get();
422    // should locate to the last region
423    assertArrayEquals(loc.getRegion().getEndKey(), EMPTY_END_ROW);
424  }
425
426  @TestTemplate
427  public void testRegionReplicas() throws Exception {
428    TEST_UTIL.getAdmin().createTable(TableDescriptorBuilder.newBuilder(TABLE_NAME)
429      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).setRegionReplication(3).build());
430    TEST_UTIL.waitUntilAllRegionsAssigned(TABLE_NAME);
431    testLocator(TEST_UTIL, TABLE_NAME, new Locator() {
432
433      @Override
434      public void updateCachedLocationOnError(HRegionLocation loc, Throwable error)
435        throws Exception {
436        locator.updateCachedLocationOnError(loc, error);
437      }
438
439      @Override
440      public RegionLocations getRegionLocations(TableName tableName, int replicaId, boolean reload)
441        throws Exception {
442        return locator.getRegionLocations(tableName, EMPTY_START_ROW, replicaId,
443          RegionLocateType.CURRENT, reload).get();
444      }
445    });
446  }
447
448  // Testcase for HBASE-21961
449  @TestTemplate
450  public void testLocateBeforeInOnlyRegion() throws IOException, InterruptedException {
451    createSingleRegionTable();
452    HRegionLocation loc =
453      getDefaultRegionLocation(TABLE_NAME, Bytes.toBytes(1), RegionLocateType.BEFORE, false).join();
454    // should locate to the only region
455    assertArrayEquals(loc.getRegion().getStartKey(), EMPTY_START_ROW);
456    assertArrayEquals(loc.getRegion().getEndKey(), EMPTY_END_ROW);
457  }
458
459  @TestTemplate
460  public void testConcurrentUpdateCachedLocationOnError() throws Exception {
461    createSingleRegionTable();
462    HRegionLocation loc =
463      getDefaultRegionLocation(TABLE_NAME, EMPTY_START_ROW, RegionLocateType.CURRENT, false).get();
464    IntStream.range(0, 100).parallel()
465      .forEach(i -> locator.updateCachedLocationOnError(loc, new NotServingRegionException()));
466  }
467
468  @TestTemplate
469  public void testCacheLocationWhenGetAllLocations() throws Exception {
470    createMultiRegionTable();
471    AsyncConnectionImpl conn = (AsyncConnectionImpl) ConnectionFactory
472      .createAsyncConnection(TEST_UTIL.getConfiguration()).get();
473    conn.getRegionLocator(TABLE_NAME).getAllRegionLocations().get();
474    List<RegionInfo> regions = TEST_UTIL.getAdmin().getRegions(TABLE_NAME);
475    for (RegionInfo region : regions) {
476      assertNotNull(conn.getLocator().getRegionLocationInCache(TABLE_NAME, region.getStartKey()));
477    }
478  }
479
480  @TestTemplate
481  public void testDoNotCacheLocationWithNullServerNameWhenGetAllLocations() throws Exception {
482    createMultiRegionTable();
483    AsyncConnectionImpl conn = (AsyncConnectionImpl) ConnectionFactory
484      .createAsyncConnection(TEST_UTIL.getConfiguration()).get();
485    List<RegionInfo> regions = TEST_UTIL.getAdmin().getRegions(TABLE_NAME);
486    RegionInfo chosen = regions.get(0);
487
488    // re-populate region cache
489    AsyncTableRegionLocator regionLocator = conn.getRegionLocator(TABLE_NAME);
490    regionLocator.clearRegionLocationCache();
491    regionLocator.getAllRegionLocations().get();
492
493    int tries = 3;
494    // expect all to be non-null at first
495    checkRegionsWithRetries(conn, regions, null, tries);
496
497    // clear servername from region info
498    Put put = MetaTableAccessor.makePutFromRegionInfo(chosen, EnvironmentEdgeManager.currentTime());
499    MetaTableAccessor.addEmptyLocation(put, 0);
500    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Lists.newArrayList(put));
501
502    // re-populate region cache again. check that we succeeded in nulling the servername
503    regionLocator.clearRegionLocationCache();
504    for (HRegionLocation loc : regionLocator.getAllRegionLocations().get()) {
505      if (loc.getRegion().equals(chosen)) {
506        assertNull(loc.getServerName());
507      }
508    }
509
510    // expect all but chosen to be non-null. chosen should be null because serverName was null
511    checkRegionsWithRetries(conn, regions, chosen, tries);
512  }
513
514  // caching of getAllRegionLocations is async. so we give it a couple tries
515  private void checkRegionsWithRetries(AsyncConnectionImpl conn, List<RegionInfo> regions,
516    RegionInfo chosen, int retries) throws InterruptedException {
517    while (true) {
518      try {
519        checkRegions(conn, regions, chosen);
520        break;
521      } catch (AssertionError e) {
522        if (retries-- <= 0) {
523          throw e;
524        }
525        Thread.sleep(500);
526      }
527    }
528  }
529
530  private void checkRegions(AsyncConnectionImpl conn, List<RegionInfo> regions, RegionInfo chosen) {
531    for (RegionInfo region : regions) {
532      RegionLocations fromCache =
533        conn.getLocator().getRegionLocationInCache(TABLE_NAME, region.getStartKey());
534      if (region.equals(chosen)) {
535        assertNull(fromCache);
536      } else {
537        assertNotNull(fromCache);
538      }
539    }
540  }
541}