001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.apache.hadoop.hbase.HConstants.DEFAULT_USE_META_REPLICAS;
021import static org.apache.hadoop.hbase.HConstants.EMPTY_END_ROW;
022import static org.apache.hadoop.hbase.HConstants.NINES;
023import static org.apache.hadoop.hbase.HConstants.USE_META_REPLICAS;
024import static org.apache.hadoop.hbase.HConstants.ZEROES;
025import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME;
026import static org.apache.hadoop.hbase.client.AsyncRegionLocatorHelper.createRegionLocations;
027import static org.apache.hadoop.hbase.client.AsyncRegionLocatorHelper.isGood;
028import static org.apache.hadoop.hbase.client.ConnectionUtils.createClosestRowAfter;
029import static org.apache.hadoop.hbase.client.ConnectionUtils.isEmptyStopRow;
030import static org.apache.hadoop.hbase.client.RegionInfo.createRegionName;
031import static org.apache.hadoop.hbase.client.RegionLocator.LOCATOR_META_REPLICAS_MODE;
032import static org.apache.hadoop.hbase.util.ConcurrentMapUtils.computeIfAbsent;
033
034import java.io.IOException;
035import java.util.ArrayList;
036import java.util.Arrays;
037import java.util.HashSet;
038import java.util.Iterator;
039import java.util.LinkedHashMap;
040import java.util.List;
041import java.util.Map;
042import java.util.Optional;
043import java.util.Set;
044import java.util.concurrent.CompletableFuture;
045import java.util.concurrent.ConcurrentHashMap;
046import java.util.concurrent.ConcurrentMap;
047import java.util.concurrent.TimeUnit;
048import org.apache.commons.lang3.ObjectUtils;
049import org.apache.hadoop.hbase.CatalogFamilyFormat;
050import org.apache.hadoop.hbase.CatalogReplicaMode;
051import org.apache.hadoop.hbase.HBaseIOException;
052import org.apache.hadoop.hbase.HConstants;
053import org.apache.hadoop.hbase.HRegionLocation;
054import org.apache.hadoop.hbase.RegionLocations;
055import org.apache.hadoop.hbase.ServerName;
056import org.apache.hadoop.hbase.TableName;
057import org.apache.hadoop.hbase.TableNotFoundException;
058import org.apache.hadoop.hbase.client.Scan.ReadType;
059import org.apache.hadoop.hbase.util.Bytes;
060import org.apache.yetus.audience.InterfaceAudience;
061import org.slf4j.Logger;
062import org.slf4j.LoggerFactory;
063
064/**
065 * The asynchronous locator for regions other than meta.
066 */
067@InterfaceAudience.Private
068class AsyncNonMetaRegionLocator {
069
070  private static final Logger LOG = LoggerFactory.getLogger(AsyncNonMetaRegionLocator.class);
071
072  static final String MAX_CONCURRENT_LOCATE_REQUEST_PER_TABLE =
073    "hbase.client.meta.max.concurrent.locate.per.table";
074
075  private static final int DEFAULT_MAX_CONCURRENT_LOCATE_REQUEST_PER_TABLE = 8;
076
077  static String LOCATE_PREFETCH_LIMIT = "hbase.client.locate.prefetch.limit";
078
079  private static final int DEFAULT_LOCATE_PREFETCH_LIMIT = 10;
080
081  private final AsyncConnectionImpl conn;
082
083  private final int maxConcurrentLocateRequestPerTable;
084
085  private final int locatePrefetchLimit;
086
087  // The mode tells if HedgedRead, LoadBalance mode is supported.
088  // The default mode is CatalogReplicaMode.None.
089  private CatalogReplicaMode metaReplicaMode;
090  private CatalogReplicaLoadBalanceSelector metaReplicaSelector;
091
092  private final ConcurrentMap<TableName, TableCache> cache = new ConcurrentHashMap<>();
093
094  private static final class LocateRequest {
095
096    private final byte[] row;
097
098    private final RegionLocateType locateType;
099
100    public LocateRequest(byte[] row, RegionLocateType locateType) {
101      this.row = row;
102      this.locateType = locateType;
103    }
104
105    @Override
106    public int hashCode() {
107      return Bytes.hashCode(row) ^ locateType.hashCode();
108    }
109
110    @Override
111    public boolean equals(Object obj) {
112      if (obj == null || obj.getClass() != LocateRequest.class) {
113        return false;
114      }
115      LocateRequest that = (LocateRequest) obj;
116      return locateType.equals(that.locateType) && Bytes.equals(row, that.row);
117    }
118  }
119
120  private static final class RegionLocationsFutureResult {
121    private final CompletableFuture<RegionLocations> future;
122    private final RegionLocations result;
123    private final Throwable e;
124
125    public RegionLocationsFutureResult(CompletableFuture<RegionLocations> future,
126      RegionLocations result, Throwable e) {
127      this.future = future;
128      this.result = result;
129      this.e = e;
130    }
131
132    public void complete() {
133      if (e != null) {
134        future.completeExceptionally(e);
135      }
136      future.complete(result);
137    }
138  }
139
140  private static final class TableCache {
141
142    private final Set<LocateRequest> pendingRequests = new HashSet<>();
143
144    private final Map<LocateRequest, CompletableFuture<RegionLocations>> allRequests =
145      new LinkedHashMap<>();
146    private final AsyncRegionLocationCache regionLocationCache;
147
148    public TableCache(TableName tableName) {
149      regionLocationCache = new AsyncRegionLocationCache(tableName);
150    }
151
152    public boolean hasQuota(int max) {
153      return pendingRequests.size() < max;
154    }
155
156    public boolean isPending(LocateRequest req) {
157      return pendingRequests.contains(req);
158    }
159
160    public void send(LocateRequest req) {
161      pendingRequests.add(req);
162    }
163
164    public Optional<LocateRequest> getCandidate() {
165      return allRequests.keySet().stream().filter(r -> !isPending(r)).findFirst();
166    }
167
168    public List<RegionLocationsFutureResult> clearCompletedRequests(RegionLocations locations) {
169      List<RegionLocationsFutureResult> futureResultList = new ArrayList<>();
170      for (Iterator<Map.Entry<LocateRequest, CompletableFuture<RegionLocations>>> iter =
171        allRequests.entrySet().iterator(); iter.hasNext();) {
172        Map.Entry<LocateRequest, CompletableFuture<RegionLocations>> entry = iter.next();
173        if (tryComplete(entry.getKey(), entry.getValue(), locations, futureResultList)) {
174          iter.remove();
175        }
176      }
177      return futureResultList;
178    }
179
180    private boolean tryComplete(LocateRequest req, CompletableFuture<RegionLocations> future,
181      RegionLocations locations, List<RegionLocationsFutureResult> futureResultList) {
182      if (future.isDone()) {
183        return true;
184      }
185      if (locations == null) {
186        return false;
187      }
188      HRegionLocation loc = ObjectUtils.firstNonNull(locations.getRegionLocations());
189      // we should at least have one location available, otherwise the request should fail and
190      // should not arrive here
191      assert loc != null;
192      boolean completed;
193      if (req.locateType.equals(RegionLocateType.BEFORE)) {
194        // for locating the row before current row, the common case is to find the previous region
195        // in reverse scan, so we check the endKey first. In general, the condition should be
196        // startKey < req.row and endKey >= req.row. Here we split it to endKey == req.row ||
197        // (endKey > req.row && startKey < req.row). The two conditions are equal since startKey <
198        // endKey.
199        byte[] endKey = loc.getRegion().getEndKey();
200        int c = Bytes.compareTo(endKey, req.row);
201        completed = c == 0 || ((c > 0 || Bytes.equals(EMPTY_END_ROW, endKey))
202          && Bytes.compareTo(loc.getRegion().getStartKey(), req.row) < 0);
203      } else {
204        completed = loc.getRegion().containsRow(req.row);
205      }
206      if (completed) {
207        futureResultList.add(new RegionLocationsFutureResult(future, locations, null));
208        return true;
209      } else {
210        return false;
211      }
212    }
213  }
214
215  AsyncNonMetaRegionLocator(AsyncConnectionImpl conn) {
216    this.conn = conn;
217    this.maxConcurrentLocateRequestPerTable = conn.getConfiguration().getInt(
218      MAX_CONCURRENT_LOCATE_REQUEST_PER_TABLE, DEFAULT_MAX_CONCURRENT_LOCATE_REQUEST_PER_TABLE);
219    this.locatePrefetchLimit =
220      conn.getConfiguration().getInt(LOCATE_PREFETCH_LIMIT, DEFAULT_LOCATE_PREFETCH_LIMIT);
221
222    // Get the region locator's meta replica mode.
223    this.metaReplicaMode = CatalogReplicaMode.fromString(
224      conn.getConfiguration().get(LOCATOR_META_REPLICAS_MODE, CatalogReplicaMode.NONE.toString()));
225
226    switch (this.metaReplicaMode) {
227      case LOAD_BALANCE:
228        String replicaSelectorClass =
229          conn.getConfiguration().get(RegionLocator.LOCATOR_META_REPLICAS_MODE_LOADBALANCE_SELECTOR,
230            CatalogReplicaLoadBalanceSimpleSelector.class.getName());
231
232        this.metaReplicaSelector = CatalogReplicaLoadBalanceSelectorFactory
233          .createSelector(replicaSelectorClass, META_TABLE_NAME, conn, () -> {
234            int numOfReplicas = CatalogReplicaLoadBalanceSelector.UNINITIALIZED_NUM_OF_REPLICAS;
235            try {
236              RegionLocations metaLocations = conn.registry.getMetaRegionLocations()
237                .get(conn.connConf.getReadRpcTimeoutNs(), TimeUnit.NANOSECONDS);
238              numOfReplicas = metaLocations.size();
239            } catch (Exception e) {
240              LOG.error("Failed to get table {}'s region replication, ", META_TABLE_NAME, e);
241            }
242            return numOfReplicas;
243          });
244        break;
245      case NONE:
246        // If user does not configure LOCATOR_META_REPLICAS_MODE, let's check the legacy config.
247        boolean useMetaReplicas =
248          conn.getConfiguration().getBoolean(USE_META_REPLICAS, DEFAULT_USE_META_REPLICAS);
249        if (useMetaReplicas) {
250          this.metaReplicaMode = CatalogReplicaMode.HEDGED_READ;
251        }
252        break;
253      default:
254        // Doing nothing
255    }
256  }
257
258  private TableCache getTableCache(TableName tableName) {
259    return computeIfAbsent(cache, tableName, () -> new TableCache(tableName));
260  }
261
262  private void complete(TableName tableName, LocateRequest req, RegionLocations locs,
263    Throwable error) {
264    if (error != null) {
265      LOG.warn("Failed to locate region in '" + tableName + "', row='"
266        + Bytes.toStringBinary(req.row) + "', locateType=" + req.locateType, error);
267    }
268    Optional<LocateRequest> toSend = Optional.empty();
269    TableCache tableCache = getTableCache(tableName);
270    if (locs != null) {
271      RegionLocations addedLocs = tableCache.regionLocationCache.add(locs);
272      List<RegionLocationsFutureResult> futureResultList = new ArrayList<>();
273      synchronized (tableCache) {
274        tableCache.pendingRequests.remove(req);
275        futureResultList.addAll(tableCache.clearCompletedRequests(addedLocs));
276        // Remove a complete locate request in a synchronized block, so the table cache must have
277        // quota to send a candidate request.
278        toSend = tableCache.getCandidate();
279        toSend.ifPresent(r -> tableCache.send(r));
280      }
281      futureResultList.forEach(RegionLocationsFutureResult::complete);
282      toSend.ifPresent(r -> locateInMeta(tableName, r));
283    } else {
284      // we meet an error
285      assert error != null;
286      List<RegionLocationsFutureResult> futureResultList = new ArrayList<>();
287      synchronized (tableCache) {
288        tableCache.pendingRequests.remove(req);
289        // fail the request itself, no matter whether it is a DoNotRetryIOException, as we have
290        // already retried several times
291        CompletableFuture<RegionLocations> future = tableCache.allRequests.remove(req);
292        if (future != null) {
293          futureResultList.add(new RegionLocationsFutureResult(future, null, error));
294        }
295        futureResultList.addAll(tableCache.clearCompletedRequests(null));
296        // Remove a complete locate request in a synchronized block, so the table cache must have
297        // quota to send a candidate request.
298        toSend = tableCache.getCandidate();
299        toSend.ifPresent(r -> tableCache.send(r));
300      }
301      futureResultList.forEach(RegionLocationsFutureResult::complete);
302      toSend.ifPresent(r -> locateInMeta(tableName, r));
303    }
304  }
305
306  // return whether we should stop the scan
307  private boolean onScanNext(TableName tableName, LocateRequest req, Result result) {
308    RegionLocations locs = CatalogFamilyFormat.getRegionLocations(result);
309    if (LOG.isDebugEnabled()) {
310      LOG.debug("The fetched location of '{}', row='{}', locateType={} is {}", tableName,
311        Bytes.toStringBinary(req.row), req.locateType, locs);
312    }
313    // remove HRegionLocation with null location, i.e, getServerName returns null.
314    if (locs != null) {
315      locs = locs.removeElementsWithNullLocation();
316    }
317
318    // the default region location should always be presented when fetching from meta, otherwise
319    // let's fail the request.
320    if (locs == null || locs.getDefaultRegionLocation() == null) {
321      complete(tableName, req, null,
322        new HBaseIOException(String.format("No location found for '%s', row='%s', locateType=%s",
323          tableName, Bytes.toStringBinary(req.row), req.locateType)));
324      return true;
325    }
326    HRegionLocation loc = locs.getDefaultRegionLocation();
327    RegionInfo info = loc.getRegion();
328    if (info == null) {
329      complete(tableName, req, null,
330        new HBaseIOException(String.format("HRegionInfo is null for '%s', row='%s', locateType=%s",
331          tableName, Bytes.toStringBinary(req.row), req.locateType)));
332      return true;
333    }
334    if (info.isSplitParent()) {
335      return false;
336    }
337    complete(tableName, req, locs, null);
338    return true;
339  }
340
341  private void recordCacheHit() {
342    conn.getConnectionMetrics().ifPresent(MetricsConnection::incrMetaCacheHit);
343  }
344
345  private void recordCacheMiss() {
346    conn.getConnectionMetrics().ifPresent(MetricsConnection::incrMetaCacheMiss);
347  }
348
349  private RegionLocations locateRowInCache(TableCache tableCache, byte[] row, int replicaId) {
350    RegionLocations locs = tableCache.regionLocationCache.findForRow(row, replicaId);
351    if (locs == null) {
352      recordCacheMiss();
353    } else {
354      recordCacheHit();
355    }
356    return locs;
357  }
358
359  private RegionLocations locateRowBeforeInCache(TableCache tableCache, byte[] row, int replicaId) {
360    RegionLocations locs = tableCache.regionLocationCache.findForBeforeRow(row, replicaId);
361    if (locs == null) {
362      recordCacheMiss();
363    } else {
364      recordCacheHit();
365    }
366    return locs;
367  }
368
369  private void locateInMeta(TableName tableName, LocateRequest req) {
370    if (LOG.isTraceEnabled()) {
371      LOG.trace("Try locate '" + tableName + "', row='" + Bytes.toStringBinary(req.row)
372        + "', locateType=" + req.locateType + " in meta");
373    }
374    byte[] metaStartKey;
375    if (req.locateType.equals(RegionLocateType.BEFORE)) {
376      if (isEmptyStopRow(req.row)) {
377        byte[] binaryTableName = tableName.getName();
378        metaStartKey = Arrays.copyOf(binaryTableName, binaryTableName.length + 1);
379      } else {
380        metaStartKey = createRegionName(tableName, req.row, ZEROES, false);
381      }
382    } else {
383      metaStartKey = createRegionName(tableName, req.row, NINES, false);
384    }
385    byte[] metaStopKey =
386      RegionInfo.createRegionName(tableName, HConstants.EMPTY_START_ROW, "", false);
387    Scan scan = new Scan().withStartRow(metaStartKey).withStopRow(metaStopKey, true)
388      .addFamily(HConstants.CATALOG_FAMILY).setReversed(true).setCaching(locatePrefetchLimit)
389      .setReadType(ReadType.PREAD);
390
391    switch (this.metaReplicaMode) {
392      case LOAD_BALANCE:
393        int metaReplicaId = this.metaReplicaSelector.select(tableName, req.row, req.locateType);
394        if (metaReplicaId != RegionInfo.DEFAULT_REPLICA_ID) {
395          // If the selector gives a non-primary meta replica region, then go with it.
396          // Otherwise, just go to primary in non-hedgedRead mode.
397          scan.setConsistency(Consistency.TIMELINE);
398          scan.setReplicaId(metaReplicaId);
399        }
400        break;
401      case HEDGED_READ:
402        scan.setConsistency(Consistency.TIMELINE);
403        break;
404      default:
405        // do nothing
406    }
407
408    conn.getTable(META_TABLE_NAME).scan(scan, new AdvancedScanResultConsumer() {
409
410      private boolean completeNormally = false;
411
412      private boolean tableNotFound = true;
413
414      @Override
415      public void onError(Throwable error) {
416        complete(tableName, req, null, error);
417      }
418
419      @Override
420      public void onComplete() {
421        if (tableNotFound) {
422          complete(tableName, req, null, new TableNotFoundException(tableName));
423        } else if (!completeNormally) {
424          complete(tableName, req, null, new IOException(
425            "Unable to find region for '" + Bytes.toStringBinary(req.row) + "' in " + tableName));
426        }
427      }
428
429      @Override
430      public void onNext(Result[] results, ScanController controller) {
431        if (results.length == 0) {
432          return;
433        }
434        tableNotFound = false;
435        int i = 0;
436        for (; i < results.length; i++) {
437          if (onScanNext(tableName, req, results[i])) {
438            completeNormally = true;
439            controller.terminate();
440            i++;
441            break;
442          }
443        }
444        // Add the remaining results into cache
445        if (i < results.length) {
446          TableCache tableCache = getTableCache(tableName);
447          for (; i < results.length; i++) {
448            RegionLocations locs = CatalogFamilyFormat.getRegionLocations(results[i]);
449            if (locs == null) {
450              continue;
451            }
452            HRegionLocation loc = locs.getDefaultRegionLocation();
453            if (loc == null) {
454              continue;
455            }
456            RegionInfo info = loc.getRegion();
457            if (info == null || info.isOffline() || info.isSplitParent()) {
458              continue;
459            }
460            RegionLocations addedLocs = tableCache.regionLocationCache.add(locs);
461            List<RegionLocationsFutureResult> futureResultList = new ArrayList<>();
462            synchronized (tableCache) {
463              futureResultList.addAll(tableCache.clearCompletedRequests(addedLocs));
464            }
465            futureResultList.forEach(RegionLocationsFutureResult::complete);
466          }
467        }
468      }
469    });
470  }
471
472  private RegionLocations locateInCache(TableCache tableCache, byte[] row, int replicaId,
473    RegionLocateType locateType) {
474    return locateType.equals(RegionLocateType.BEFORE)
475      ? locateRowBeforeInCache(tableCache, row, replicaId)
476      : locateRowInCache(tableCache, row, replicaId);
477  }
478
479  // locateToPrevious is true means we will use the start key of a region to locate the region
480  // placed before it. Used for reverse scan. See the comment of
481  // AsyncRegionLocator.getPreviousRegionLocation.
482  private CompletableFuture<RegionLocations> getRegionLocationsInternal(TableName tableName,
483    byte[] row, int replicaId, RegionLocateType locateType, boolean reload) {
484    // AFTER should be convert to CURRENT before calling this method
485    assert !locateType.equals(RegionLocateType.AFTER);
486    TableCache tableCache = getTableCache(tableName);
487    if (!reload) {
488      RegionLocations locs = locateInCache(tableCache, row, replicaId, locateType);
489      if (isGood(locs, replicaId)) {
490        return CompletableFuture.completedFuture(locs);
491      }
492    }
493    CompletableFuture<RegionLocations> future;
494    LocateRequest req;
495    boolean sendRequest = false;
496    synchronized (tableCache) {
497      // check again
498      if (!reload) {
499        RegionLocations locs = locateInCache(tableCache, row, replicaId, locateType);
500        if (isGood(locs, replicaId)) {
501          return CompletableFuture.completedFuture(locs);
502        }
503      }
504      req = new LocateRequest(row, locateType);
505      future = tableCache.allRequests.get(req);
506      if (future == null) {
507        future = new CompletableFuture<>();
508        tableCache.allRequests.put(req, future);
509        if (tableCache.hasQuota(maxConcurrentLocateRequestPerTable) && !tableCache.isPending(req)) {
510          tableCache.send(req);
511          sendRequest = true;
512        }
513      }
514    }
515    if (sendRequest) {
516      locateInMeta(tableName, req);
517    }
518    return future;
519  }
520
521  CompletableFuture<RegionLocations> getRegionLocations(TableName tableName, byte[] row,
522    int replicaId, RegionLocateType locateType, boolean reload) {
523    // as we know the exact row after us, so we can just create the new row, and use the same
524    // algorithm to locate it.
525    if (locateType.equals(RegionLocateType.AFTER)) {
526      row = createClosestRowAfter(row);
527      locateType = RegionLocateType.CURRENT;
528    }
529    return getRegionLocationsInternal(tableName, row, replicaId, locateType, reload);
530  }
531
532  private void recordClearRegionCache() {
533    conn.getConnectionMetrics().ifPresent(MetricsConnection::incrMetaCacheNumClearRegion);
534  }
535
536  private void removeLocationFromCache(HRegionLocation loc) {
537    TableCache tableCache = cache.get(loc.getRegion().getTable());
538    if (tableCache != null) {
539      if (tableCache.regionLocationCache.remove(loc)) {
540        recordClearRegionCache();
541        updateMetaReplicaSelector(loc);
542      }
543    }
544  }
545
546  private void updateMetaReplicaSelector(HRegionLocation loc) {
547    // Tell metaReplicaSelector that the location is stale. It will create a stale entry
548    // with timestamp internally. Next time the client looks up the same location,
549    // it will pick a different meta replica region.
550    if (metaReplicaMode == CatalogReplicaMode.LOAD_BALANCE) {
551      metaReplicaSelector.onError(loc);
552    }
553  }
554
555  void addLocationToCache(HRegionLocation loc) {
556    getTableCache(loc.getRegion().getTable()).regionLocationCache.add(createRegionLocations(loc));
557  }
558
559  private HRegionLocation getCachedLocation(HRegionLocation loc) {
560    TableCache tableCache = cache.get(loc.getRegion().getTable());
561    if (tableCache == null) {
562      return null;
563    }
564    RegionLocations locs = tableCache.regionLocationCache.get(loc.getRegion().getStartKey());
565    return locs != null ? locs.getRegionLocation(loc.getRegion().getReplicaId()) : null;
566  }
567
568  void updateCachedLocationOnError(HRegionLocation loc, Throwable exception) {
569    Optional<MetricsConnection> connectionMetrics = conn.getConnectionMetrics();
570    AsyncRegionLocatorHelper.updateCachedLocationOnError(loc, exception, this::getCachedLocation,
571      this::addLocationToCache, this::removeLocationFromCache, connectionMetrics.orElse(null));
572  }
573
574  void clearCache(TableName tableName) {
575    TableCache tableCache = cache.remove(tableName);
576    if (tableCache == null) {
577      return;
578    }
579    List<RegionLocationsFutureResult> futureResultList = new ArrayList<>();
580    synchronized (tableCache) {
581      if (!tableCache.allRequests.isEmpty()) {
582        IOException error = new IOException("Cache cleared");
583        tableCache.allRequests.values().forEach(f -> {
584          futureResultList.add(new RegionLocationsFutureResult(f, null, error));
585        });
586      }
587    }
588    futureResultList.forEach(RegionLocationsFutureResult::complete);
589    conn.getConnectionMetrics().ifPresent(
590      metrics -> metrics.incrMetaCacheNumClearRegion(tableCache.regionLocationCache.size()));
591  }
592
593  void clearCache() {
594    cache.clear();
595  }
596
597  void clearCache(ServerName serverName) {
598    for (TableCache tableCache : cache.values()) {
599      tableCache.regionLocationCache.removeForServer(serverName);
600    }
601  }
602
603  // only used for testing whether we have cached the location for a region.
604  RegionLocations getRegionLocationInCache(TableName tableName, byte[] row) {
605    TableCache tableCache = cache.get(tableName);
606    if (tableCache == null) {
607      return null;
608    }
609    return locateRowInCache(tableCache, row, RegionReplicaUtil.DEFAULT_REPLICA_ID);
610  }
611
612  // only used for testing whether we have cached the location for a table.
613  int getNumberOfCachedRegionLocations(TableName tableName) {
614    TableCache tableCache = cache.get(tableName);
615    if (tableCache == null) {
616      return 0;
617    }
618    return tableCache.regionLocationCache.getAll().stream()
619      .mapToInt(RegionLocations::numNonNullElements).sum();
620  }
621}