001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import java.util.ArrayList;
021import java.util.Collection;
022import java.util.Collections;
023import java.util.Comparator;
024import java.util.HashMap;
025import java.util.Iterator;
026import java.util.List;
027import java.util.Map;
028import java.util.SortedSet;
029import java.util.TreeSet;
030import java.util.concurrent.ConcurrentHashMap;
031import java.util.concurrent.ConcurrentSkipListMap;
032import java.util.concurrent.atomic.AtomicInteger;
033import java.util.function.Predicate;
034import java.util.stream.Collectors;
035import org.apache.hadoop.hbase.HRegionLocation;
036import org.apache.hadoop.hbase.ServerName;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.client.RegionInfo;
039import org.apache.hadoop.hbase.client.TableState;
040import org.apache.hadoop.hbase.master.RegionState;
041import org.apache.hadoop.hbase.master.RegionState.State;
042import org.apache.hadoop.hbase.master.TableStateManager;
043import org.apache.hadoop.hbase.util.Bytes;
044import org.apache.yetus.audience.InterfaceAudience;
045import org.slf4j.Logger;
046import org.slf4j.LoggerFactory;
047
048import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
049
050/**
051 * RegionStates contains a set of Maps that describes the in-memory state of the AM, with
052 * the regions available in the system, the region in transition, the offline regions and
053 * the servers holding regions.
054 */
055@InterfaceAudience.Private
056public class RegionStates {
057  private static final Logger LOG = LoggerFactory.getLogger(RegionStates.class);
058
059  // This comparator sorts the RegionStates by time stamp then Region name.
060  // Comparing by timestamp alone can lead us to discard different RegionStates that happen
061  // to share a timestamp.
062  private static class RegionStateStampComparator implements Comparator<RegionState> {
063    @Override
064    public int compare(final RegionState l, final RegionState r) {
065      int stampCmp = Long.compare(l.getStamp(), r.getStamp());
066      return stampCmp != 0 ? stampCmp : RegionInfo.COMPARATOR.compare(l.getRegion(), r.getRegion());
067    }
068  }
069
070  public final static RegionStateStampComparator REGION_STATE_STAMP_COMPARATOR =
071      new RegionStateStampComparator();
072
073  private final Object regionsMapLock = new Object();
074
075  // TODO: Replace the ConcurrentSkipListMaps
076  /**
077   * RegionName -- i.e. RegionInfo.getRegionName() -- as bytes to {@link RegionStateNode}
078   */
079  private final ConcurrentSkipListMap<byte[], RegionStateNode> regionsMap =
080      new ConcurrentSkipListMap<>(Bytes.BYTES_COMPARATOR);
081
082  /**
083   * this map is a hack to lookup of region in master by encoded region name is O(n).
084   * must put and remove with regionsMap.
085   */
086  private final ConcurrentSkipListMap<String, RegionStateNode> encodedRegionsMap =
087    new ConcurrentSkipListMap<>();
088
089  private final ConcurrentSkipListMap<RegionInfo, RegionStateNode> regionInTransition =
090    new ConcurrentSkipListMap<>(RegionInfo.COMPARATOR);
091
092  /**
093   * Regions marked as offline on a read of hbase:meta. Unused or at least, once
094   * offlined, regions have no means of coming on line again. TODO.
095   */
096  private final ConcurrentSkipListMap<RegionInfo, RegionStateNode> regionOffline =
097    new ConcurrentSkipListMap<RegionInfo, RegionStateNode>();
098
099  private final ConcurrentSkipListMap<byte[], RegionFailedOpen> regionFailedOpen =
100    new ConcurrentSkipListMap<byte[], RegionFailedOpen>(Bytes.BYTES_COMPARATOR);
101
102  private final ConcurrentHashMap<ServerName, ServerStateNode> serverMap =
103      new ConcurrentHashMap<ServerName, ServerStateNode>();
104
105  public RegionStates() { }
106
107  /**
108   * Called on stop of AssignmentManager.
109   */
110  public void clear() {
111    regionsMap.clear();
112    encodedRegionsMap.clear();
113    regionInTransition.clear();
114    regionOffline.clear();
115    serverMap.clear();
116  }
117
118  @VisibleForTesting
119  public boolean isRegionInRegionStates(final RegionInfo hri) {
120    return (regionsMap.containsKey(hri.getRegionName()) || regionInTransition.containsKey(hri)
121        || regionOffline.containsKey(hri));
122  }
123
124  // ==========================================================================
125  //  RegionStateNode helpers
126  // ==========================================================================
127  @VisibleForTesting
128  RegionStateNode createRegionStateNode(RegionInfo regionInfo) {
129    synchronized (regionsMapLock) {
130      RegionStateNode node = regionsMap.computeIfAbsent(regionInfo.getRegionName(),
131        key -> new RegionStateNode(regionInfo, regionInTransition));
132
133      if (encodedRegionsMap.get(regionInfo.getEncodedName()) != node) {
134        encodedRegionsMap.put(regionInfo.getEncodedName(), node);
135      }
136
137      return node;
138    }
139  }
140
141  public RegionStateNode getOrCreateRegionStateNode(RegionInfo regionInfo) {
142    RegionStateNode node = getRegionStateNodeFromName(regionInfo.getRegionName());
143    return node != null ? node : createRegionStateNode(regionInfo);
144  }
145
146  RegionStateNode getRegionStateNodeFromName(byte[] regionName) {
147    return regionsMap.get(regionName);
148  }
149
150  public RegionStateNode getRegionStateNode(RegionInfo regionInfo) {
151    return getRegionStateNodeFromName(regionInfo.getRegionName());
152  }
153
154  public void deleteRegion(final RegionInfo regionInfo) {
155    synchronized (regionsMapLock) {
156      regionsMap.remove(regionInfo.getRegionName());
157      encodedRegionsMap.remove(regionInfo.getEncodedName());
158    }
159    // See HBASE-20860
160    // After master restarts, merged regions' RIT state may not be cleaned,
161    // making sure they are cleaned here
162    if (regionInTransition.containsKey(regionInfo)) {
163      regionInTransition.remove(regionInfo);
164    }
165    // Remove from the offline regions map too if there.
166    if (this.regionOffline.containsKey(regionInfo)) {
167      if (LOG.isTraceEnabled()) LOG.trace("Removing from regionOffline Map: " + regionInfo);
168      this.regionOffline.remove(regionInfo);
169    }
170  }
171
172  public void deleteRegions(final List<RegionInfo> regionInfos) {
173    regionInfos.forEach(this::deleteRegion);
174  }
175
176  ArrayList<RegionStateNode> getTableRegionStateNodes(final TableName tableName) {
177    final ArrayList<RegionStateNode> regions = new ArrayList<RegionStateNode>();
178    for (RegionStateNode node: regionsMap.tailMap(tableName.getName()).values()) {
179      if (!node.getTable().equals(tableName)) break;
180      regions.add(node);
181    }
182    return regions;
183  }
184
185  ArrayList<RegionState> getTableRegionStates(final TableName tableName) {
186    final ArrayList<RegionState> regions = new ArrayList<RegionState>();
187    for (RegionStateNode node: regionsMap.tailMap(tableName.getName()).values()) {
188      if (!node.getTable().equals(tableName)) break;
189      regions.add(node.toRegionState());
190    }
191    return regions;
192  }
193
194  ArrayList<RegionInfo> getTableRegionsInfo(final TableName tableName) {
195    final ArrayList<RegionInfo> regions = new ArrayList<RegionInfo>();
196    for (RegionStateNode node: regionsMap.tailMap(tableName.getName()).values()) {
197      if (!node.getTable().equals(tableName)) break;
198      regions.add(node.getRegionInfo());
199    }
200    return regions;
201  }
202
203  /** @return A view of region state nodes for all the regions. */
204  public Collection<RegionStateNode> getRegionStateNodes() {
205    return Collections.unmodifiableCollection(regionsMap.values());
206  }
207
208  /** @return A snapshot of region state nodes for all the regions. */
209  public ArrayList<RegionState> getRegionStates() {
210    final ArrayList<RegionState> regions = new ArrayList<>(regionsMap.size());
211    for (RegionStateNode node: regionsMap.values()) {
212      regions.add(node.toRegionState());
213    }
214    return regions;
215  }
216
217  // ==========================================================================
218  //  RegionState helpers
219  // ==========================================================================
220  public RegionState getRegionState(final RegionInfo regionInfo) {
221    RegionStateNode regionStateNode = getRegionStateNode(regionInfo);
222    return regionStateNode == null ? null : regionStateNode.toRegionState();
223  }
224
225  public RegionState getRegionState(final String encodedRegionName) {
226    final RegionStateNode node = encodedRegionsMap.get(encodedRegionName);
227    if (node == null) {
228      return null;
229    }
230    return node.toRegionState();
231  }
232
233  // ============================================================================================
234  //  TODO: helpers
235  // ============================================================================================
236  public boolean hasTableRegionStates(final TableName tableName) {
237    // TODO
238    return !getTableRegionStates(tableName).isEmpty();
239  }
240
241  /**
242   * @return Return online regions of table; does not include OFFLINE or SPLITTING regions.
243   */
244  public List<RegionInfo> getRegionsOfTable(final TableName table) {
245    return getRegionsOfTable(table, false);
246  }
247
248  private HRegionLocation createRegionForReopen(RegionStateNode node) {
249    node.lock();
250    try {
251      if (!include(node, false)) {
252        return null;
253      }
254      if (node.isInState(State.OPEN)) {
255        return new HRegionLocation(node.getRegionInfo(), node.getRegionLocation(),
256          node.getOpenSeqNum());
257      } else if (node.isInState(State.OPENING)) {
258        return new HRegionLocation(node.getRegionInfo(), node.getRegionLocation(), -1);
259      } else {
260        return null;
261      }
262    } finally {
263      node.unlock();
264    }
265  }
266
267  /**
268   * Get the regions to be reopened when modifying a table.
269   * <p/>
270   * Notice that the {@code openSeqNum} in the returned HRegionLocation is also used to indicate the
271   * state of this region, positive means the region is in {@link State#OPEN}, -1 means
272   * {@link State#OPENING}. And for regions in other states we do not need reopen them.
273   */
274  public List<HRegionLocation> getRegionsOfTableForReopen(TableName tableName) {
275    return getTableRegionStateNodes(tableName).stream().map(this::createRegionForReopen)
276      .filter(r -> r != null).collect(Collectors.toList());
277  }
278
279  /**
280   * Check whether the region has been reopened. The meaning of the {@link HRegionLocation} is the
281   * same with {@link #getRegionsOfTableForReopen(TableName)}.
282   * <p/>
283   * For a region which is in {@link State#OPEN} before, if the region state is changed or the open
284   * seq num is changed, we can confirm that it has been reopened.
285   * <p/>
286   * For a region which is in {@link State#OPENING} before, usually it will be in {@link State#OPEN}
287   * now and we will schedule a MRP to reopen it. But there are several exceptions:
288   * <ul>
289   * <li>The region is in state other than {@link State#OPEN} or {@link State#OPENING}.</li>
290   * <li>The location of the region has been changed</li>
291   * </ul>
292   * Of course the region could still be in {@link State#OPENING} state and still on the same
293   * server, then here we will still return a {@link HRegionLocation} for it, just like
294   * {@link #getRegionsOfTableForReopen(TableName)}.
295   * @param oldLoc the previous state/location of this region
296   * @return null if the region has been reopened, otherwise a new {@link HRegionLocation} which
297   *         means we still need to reopen the region.
298   * @see #getRegionsOfTableForReopen(TableName)
299   */
300  public HRegionLocation checkReopened(HRegionLocation oldLoc) {
301    RegionStateNode node = getRegionStateNode(oldLoc.getRegion());
302    // HBASE-20921
303    // if the oldLoc's state node does not exist, that means the region is
304    // merged or split, no need to check it
305    if (node == null) {
306      return null;
307    }
308    node.lock();
309    try {
310      if (oldLoc.getSeqNum() >= 0) {
311        // in OPEN state before
312        if (node.isInState(State.OPEN)) {
313          if (node.getOpenSeqNum() > oldLoc.getSeqNum()) {
314            // normal case, the region has been reopened
315            return null;
316          } else {
317            // the open seq num does not change, need to reopen again
318            return new HRegionLocation(node.getRegionInfo(), node.getRegionLocation(),
319              node.getOpenSeqNum());
320          }
321        } else {
322          // the state has been changed so we can make sure that the region has been reopened(not
323          // finished maybe, but not a problem).
324          return null;
325        }
326      } else {
327        // in OPENING state before
328        if (!node.isInState(State.OPEN, State.OPENING)) {
329          // not in OPEN or OPENING state, then we can make sure that the region has been
330          // reopened(not finished maybe, but not a problem)
331          return null;
332        } else {
333          if (!node.getRegionLocation().equals(oldLoc.getServerName())) {
334            // the region has been moved, so we can make sure that the region has been reopened.
335            return null;
336          }
337          // normal case, we are still in OPENING state, or the reopen has been opened and the state
338          // is changed to OPEN.
339          long openSeqNum = node.isInState(State.OPEN) ? node.getOpenSeqNum() : -1;
340          return new HRegionLocation(node.getRegionInfo(), node.getRegionLocation(), openSeqNum);
341        }
342      }
343    } finally {
344      node.unlock();
345    }
346  }
347
348  /**
349   * @return Return online regions of table; does not include OFFLINE or SPLITTING regions.
350   */
351  public List<RegionInfo> getRegionsOfTable(TableName table, boolean offline) {
352    return getRegionsOfTable(table, state -> include(state, offline));
353  }
354
355  /**
356   * @return Return the regions of the table; does not include OFFLINE unless you set
357   *         <code>offline</code> to true. Does not include regions that are in the
358   *         {@link State#SPLIT} state.
359   */
360  private List<RegionInfo> getRegionsOfTable(TableName table, Predicate<RegionStateNode> filter) {
361    return getTableRegionStateNodes(table).stream().filter(filter).map(n -> n.getRegionInfo())
362      .collect(Collectors.toList());
363  }
364
365  /**
366   * Utility. Whether to include region in list of regions. Default is to
367   * weed out split and offline regions.
368   * @return True if we should include the <code>node</code> (do not include
369   * if split or offline unless <code>offline</code> is set to true.
370   */
371  boolean include(final RegionStateNode node, final boolean offline) {
372    if (LOG.isTraceEnabled()) {
373      LOG.trace("WORKING ON " + node + " " + node.getRegionInfo());
374    }
375    final RegionInfo hri = node.getRegionInfo();
376    if (node.isInState(State.SPLIT) || hri.isSplit()) {
377      return false;
378    }
379    if ((node.isInState(State.OFFLINE) || hri.isOffline()) && !offline) {
380      return false;
381    }
382    return (!hri.isOffline() && !hri.isSplit()) ||
383        ((hri.isOffline() || hri.isSplit()) && offline);
384  }
385
386  // ============================================================================================
387  // Split helpers
388  // These methods will only be called in ServerCrashProcedure, and at the end of SCP we will remove
389  // the ServerStateNode by calling removeServer.
390  // ============================================================================================
391
392  private void setServerState(ServerName serverName, ServerState state) {
393    ServerStateNode serverNode = getOrCreateServer(serverName);
394    synchronized (serverNode) {
395      serverNode.setState(state);
396    }
397  }
398
399  /**
400   * Call this when we start meta log splitting a crashed Server.
401   * @see #metaLogSplit(ServerName)
402   */
403  public void metaLogSplitting(ServerName serverName) {
404    setServerState(serverName, ServerState.SPLITTING_META);
405  }
406
407  /**
408   * Called after we've split the meta logs on a crashed Server.
409   * @see #metaLogSplitting(ServerName)
410   */
411  public void metaLogSplit(ServerName serverName) {
412    setServerState(serverName, ServerState.SPLITTING_META_DONE);
413  }
414
415  /**
416   * Call this when we start log splitting for a crashed Server.
417   * @see #logSplit(ServerName)
418   */
419  public void logSplitting(final ServerName serverName) {
420    setServerState(serverName, ServerState.SPLITTING);
421  }
422
423  /**
424   * Called after we've split all logs on a crashed Server.
425   * @see #logSplitting(ServerName)
426   */
427  public void logSplit(final ServerName serverName) {
428    setServerState(serverName, ServerState.OFFLINE);
429  }
430
431  public void updateRegionState(RegionInfo regionInfo, State state) {
432    RegionStateNode regionNode = getOrCreateRegionStateNode(regionInfo);
433    regionNode.lock();
434    try {
435      regionNode.setState(state);
436    } finally {
437      regionNode.unlock();
438    }
439  }
440
441  // ============================================================================================
442  //  TODO:
443  // ============================================================================================
444  public List<RegionInfo> getAssignedRegions() {
445    final List<RegionInfo> result = new ArrayList<RegionInfo>();
446    for (RegionStateNode node: regionsMap.values()) {
447      if (!node.isInTransition()) {
448        result.add(node.getRegionInfo());
449      }
450    }
451    return result;
452  }
453
454  public boolean isRegionInState(RegionInfo regionInfo, State... state) {
455    RegionStateNode regionNode = getRegionStateNode(regionInfo);
456    if (regionNode != null) {
457      regionNode.lock();
458      try {
459        return regionNode.isInState(state);
460      } finally {
461        regionNode.unlock();
462      }
463    }
464    return false;
465  }
466
467  public boolean isRegionOnline(final RegionInfo regionInfo) {
468    return isRegionInState(regionInfo, State.OPEN);
469  }
470
471  /**
472   * @return True if region is offline (In OFFLINE or CLOSED state).
473   */
474  public boolean isRegionOffline(final RegionInfo regionInfo) {
475    return isRegionInState(regionInfo, State.OFFLINE, State.CLOSED);
476  }
477
478  public Map<ServerName, List<RegionInfo>> getSnapShotOfAssignment(
479      final Collection<RegionInfo> regions) {
480    final Map<ServerName, List<RegionInfo>> result = new HashMap<ServerName, List<RegionInfo>>();
481    if (regions != null) {
482      for (RegionInfo hri : regions) {
483        final RegionStateNode node = getRegionStateNode(hri);
484        if (node == null) {
485          continue;
486        }
487        createSnapshot(node, result);
488      }
489    } else {
490      for (RegionStateNode node : regionsMap.values()) {
491        if (node == null) {
492          continue;
493        }
494        createSnapshot(node, result);
495      }
496    }
497    return result;
498  }
499
500  private void createSnapshot(RegionStateNode node, Map<ServerName, List<RegionInfo>> result) {
501    final ServerName serverName = node.getRegionLocation();
502    if (serverName == null) {
503      return;
504    }
505
506    List<RegionInfo> serverRegions = result.get(serverName);
507    if (serverRegions == null) {
508      serverRegions = new ArrayList<RegionInfo>();
509      result.put(serverName, serverRegions);
510    }
511    serverRegions.add(node.getRegionInfo());
512  }
513
514  public Map<RegionInfo, ServerName> getRegionAssignments() {
515    final HashMap<RegionInfo, ServerName> assignments = new HashMap<RegionInfo, ServerName>();
516    for (RegionStateNode node: regionsMap.values()) {
517      assignments.put(node.getRegionInfo(), node.getRegionLocation());
518    }
519    return assignments;
520  }
521
522  public Map<RegionState.State, List<RegionInfo>> getRegionByStateOfTable(TableName tableName) {
523    final State[] states = State.values();
524    final Map<RegionState.State, List<RegionInfo>> tableRegions =
525        new HashMap<State, List<RegionInfo>>(states.length);
526    for (int i = 0; i < states.length; ++i) {
527      tableRegions.put(states[i], new ArrayList<RegionInfo>());
528    }
529
530    for (RegionStateNode node: regionsMap.values()) {
531      if (node.getTable().equals(tableName)) {
532        tableRegions.get(node.getState()).add(node.getRegionInfo());
533      }
534    }
535    return tableRegions;
536  }
537
538  public ServerName getRegionServerOfRegion(RegionInfo regionInfo) {
539    RegionStateNode regionNode = getRegionStateNode(regionInfo);
540    if (regionNode != null) {
541      regionNode.lock();
542      try {
543        ServerName server = regionNode.getRegionLocation();
544        return server != null ? server : regionNode.getLastHost();
545      } finally {
546        regionNode.unlock();
547      }
548    }
549    return null;
550  }
551
552  /**
553   * This is an EXPENSIVE clone.  Cloning though is the safest thing to do.
554   * Can't let out original since it can change and at least the load balancer
555   * wants to iterate this exported list.  We need to synchronize on regions
556   * since all access to this.servers is under a lock on this.regions.
557   *
558   * @return A clone of current assignments.
559   */
560  public Map<TableName, Map<ServerName, List<RegionInfo>>> getAssignmentsForBalancer(
561      TableStateManager tableStateManager, List<ServerName> onlineServers) {
562    final Map<TableName, Map<ServerName, List<RegionInfo>>> result = new HashMap<>();
563    for (RegionStateNode node : regionsMap.values()) {
564      if (isTableDisabled(tableStateManager, node.getTable())) {
565        continue;
566      }
567      if (node.getRegionInfo().isSplitParent()) {
568        continue;
569      }
570      Map<ServerName, List<RegionInfo>> tableResult =
571          result.computeIfAbsent(node.getTable(), t -> new HashMap<>());
572      final ServerName serverName = node.getRegionLocation();
573      if (serverName == null) {
574        LOG.info("Skipping, no server for " + node);
575        continue;
576      }
577      List<RegionInfo> serverResult =
578          tableResult.computeIfAbsent(serverName, s -> new ArrayList<>());
579      serverResult.add(node.getRegionInfo());
580    }
581    // Add online servers with no assignment for the table.
582    for (Map<ServerName, List<RegionInfo>> table : result.values()) {
583      for (ServerName serverName : onlineServers) {
584        table.computeIfAbsent(serverName, key -> new ArrayList<>());
585      }
586    }
587    return result;
588  }
589
590  private boolean isTableDisabled(final TableStateManager tableStateManager,
591      final TableName tableName) {
592    return tableStateManager.isTableState(tableName, TableState.State.DISABLED,
593      TableState.State.DISABLING);
594  }
595
596  // ==========================================================================
597  //  Region in transition helpers
598  // ==========================================================================
599  public boolean hasRegionsInTransition() {
600    return !regionInTransition.isEmpty();
601  }
602
603  public boolean isRegionInTransition(final RegionInfo regionInfo) {
604    final RegionStateNode node = regionInTransition.get(regionInfo);
605    return node != null ? node.isInTransition() : false;
606  }
607
608  public RegionState getRegionTransitionState(RegionInfo hri) {
609    RegionStateNode node = regionInTransition.get(hri);
610    if (node == null) {
611      return null;
612    }
613
614    node.lock();
615    try {
616      return node.isInTransition() ? node.toRegionState() : null;
617    } finally {
618      node.unlock();
619    }
620  }
621
622  public List<RegionStateNode> getRegionsInTransition() {
623    return new ArrayList<RegionStateNode>(regionInTransition.values());
624  }
625
626  /**
627   * Get the number of regions in transition.
628   */
629  public int getRegionsInTransitionCount() {
630    return regionInTransition.size();
631  }
632
633  public List<RegionState> getRegionsStateInTransition() {
634    final List<RegionState> rit = new ArrayList<RegionState>(regionInTransition.size());
635    for (RegionStateNode node: regionInTransition.values()) {
636      rit.add(node.toRegionState());
637    }
638    return rit;
639  }
640
641  public SortedSet<RegionState> getRegionsInTransitionOrderedByTimestamp() {
642    final SortedSet<RegionState> rit = new TreeSet<RegionState>(REGION_STATE_STAMP_COMPARATOR);
643    for (RegionStateNode node: regionInTransition.values()) {
644      rit.add(node.toRegionState());
645    }
646    return rit;
647  }
648
649  // ==========================================================================
650  //  Region offline helpers
651  // ==========================================================================
652  // TODO: Populated when we read meta but regions never make it out of here.
653  public void addToOfflineRegions(final RegionStateNode regionNode) {
654    LOG.info("Added to offline, CURRENTLY NEVER CLEARED!!! " + regionNode);
655    regionOffline.put(regionNode.getRegionInfo(), regionNode);
656  }
657
658  // TODO: Unused.
659  public void removeFromOfflineRegions(final RegionInfo regionInfo) {
660    regionOffline.remove(regionInfo);
661  }
662
663  // ==========================================================================
664  //  Region FAIL_OPEN helpers
665  // ==========================================================================
666  public static final class RegionFailedOpen {
667    private final RegionStateNode regionNode;
668
669    private volatile Exception exception = null;
670    private AtomicInteger retries = new AtomicInteger();
671
672    public RegionFailedOpen(final RegionStateNode regionNode) {
673      this.regionNode = regionNode;
674    }
675
676    public RegionStateNode getRegionStateNode() {
677      return regionNode;
678    }
679
680    public RegionInfo getRegionInfo() {
681      return regionNode.getRegionInfo();
682    }
683
684    public int incrementAndGetRetries() {
685      return this.retries.incrementAndGet();
686    }
687
688    public int getRetries() {
689      return retries.get();
690    }
691
692    public void setException(final Exception exception) {
693      this.exception = exception;
694    }
695
696    public Exception getException() {
697      return this.exception;
698    }
699  }
700
701  public RegionFailedOpen addToFailedOpen(final RegionStateNode regionNode) {
702    final byte[] key = regionNode.getRegionInfo().getRegionName();
703    return regionFailedOpen.computeIfAbsent(key, (k) -> new RegionFailedOpen(regionNode));
704  }
705
706  public RegionFailedOpen getFailedOpen(final RegionInfo regionInfo) {
707    return regionFailedOpen.get(regionInfo.getRegionName());
708  }
709
710  public void removeFromFailedOpen(final RegionInfo regionInfo) {
711    regionFailedOpen.remove(regionInfo.getRegionName());
712  }
713
714  public List<RegionState> getRegionFailedOpen() {
715    if (regionFailedOpen.isEmpty()) return Collections.emptyList();
716
717    ArrayList<RegionState> regions = new ArrayList<RegionState>(regionFailedOpen.size());
718    for (RegionFailedOpen r: regionFailedOpen.values()) {
719      regions.add(r.getRegionStateNode().toRegionState());
720    }
721    return regions;
722  }
723
724  // ==========================================================================
725  //  Servers
726  // ==========================================================================
727
728  /**
729   * Be judicious calling this method. Do it on server register ONLY otherwise
730   * you could mess up online server accounting. TOOD: Review usage and convert
731   * to {@link #getServerNode(ServerName)} where we can.
732   */
733  public ServerStateNode getOrCreateServer(final ServerName serverName) {
734    return serverMap.computeIfAbsent(serverName, key -> new ServerStateNode(key));
735  }
736
737  /**
738   * Called by SCP at end of successful processing.
739   */
740  public void removeServer(final ServerName serverName) {
741    serverMap.remove(serverName);
742  }
743
744  /**
745   * @return Pertinent ServerStateNode or NULL if none found (Do not make modifications).
746   */
747  @VisibleForTesting
748  public ServerStateNode getServerNode(final ServerName serverName) {
749    return serverMap.get(serverName);
750  }
751
752  public double getAverageLoad() {
753    int numServers = 0;
754    int totalLoad = 0;
755    for (ServerStateNode node: serverMap.values()) {
756      totalLoad += node.getRegionCount();
757      numServers++;
758    }
759    return numServers == 0 ? 0.0: (double)totalLoad / (double)numServers;
760  }
761
762  public ServerStateNode addRegionToServer(final RegionStateNode regionNode) {
763    ServerStateNode serverNode = getOrCreateServer(regionNode.getRegionLocation());
764    serverNode.addRegion(regionNode);
765    return serverNode;
766  }
767
768  public ServerStateNode removeRegionFromServer(final ServerName serverName,
769      final RegionStateNode regionNode) {
770    ServerStateNode serverNode = getOrCreateServer(serverName);
771    serverNode.removeRegion(regionNode);
772    return serverNode;
773  }
774
775  // ==========================================================================
776  //  ToString helpers
777  // ==========================================================================
778  public static String regionNamesToString(final Collection<byte[]> regions) {
779    final StringBuilder sb = new StringBuilder();
780    final Iterator<byte[]> it = regions.iterator();
781    sb.append("[");
782    if (it.hasNext()) {
783      sb.append(Bytes.toStringBinary(it.next()));
784      while (it.hasNext()) {
785        sb.append(", ");
786        sb.append(Bytes.toStringBinary(it.next()));
787      }
788    }
789    sb.append("]");
790    return sb.toString();
791  }
792}