001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.rsgroup;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.Collections;
023import java.util.HashMap;
024import java.util.HashSet;
025import java.util.LinkedList;
026import java.util.List;
027import java.util.Map;
028import java.util.Set;
029import java.util.concurrent.Future;
030import java.util.stream.Collectors;
031import org.apache.commons.lang3.StringUtils;
032import org.apache.hadoop.hbase.HConstants;
033import org.apache.hadoop.hbase.NamespaceDescriptor;
034import org.apache.hadoop.hbase.ServerName;
035import org.apache.hadoop.hbase.TableName;
036import org.apache.hadoop.hbase.client.BalanceRequest;
037import org.apache.hadoop.hbase.client.BalanceResponse;
038import org.apache.hadoop.hbase.client.RegionInfo;
039import org.apache.hadoop.hbase.client.TableDescriptor;
040import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
041import org.apache.hadoop.hbase.client.TableState;
042import org.apache.hadoop.hbase.constraint.ConstraintException;
043import org.apache.hadoop.hbase.master.HMaster;
044import org.apache.hadoop.hbase.master.LoadBalancer;
045import org.apache.hadoop.hbase.master.MasterServices;
046import org.apache.hadoop.hbase.master.RegionPlan;
047import org.apache.hadoop.hbase.master.RegionState;
048import org.apache.hadoop.hbase.master.ServerManager;
049import org.apache.hadoop.hbase.master.TableStateManager;
050import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
051import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
052import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
053import org.apache.hadoop.hbase.net.Address;
054import org.apache.hadoop.hbase.procedure2.Procedure;
055import org.apache.hadoop.hbase.util.Pair;
056import org.apache.yetus.audience.InterfaceAudience;
057import org.slf4j.Logger;
058import org.slf4j.LoggerFactory;
059
060import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
061
062/**
063 * Service to support Region Server Grouping (HBase-6721).
064 */
065@InterfaceAudience.Private
066public class RSGroupAdminServer implements RSGroupAdmin {
067  private static final Logger LOG = LoggerFactory.getLogger(RSGroupAdminServer.class);
068  public static final String KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE =
069    "should keep at least " + "one server in 'default' RSGroup.";
070
071  private MasterServices master;
072  private final RSGroupInfoManager rsGroupInfoManager;
073
074  public RSGroupAdminServer(MasterServices master, RSGroupInfoManager rsGroupInfoManager) {
075    this.master = master;
076    this.rsGroupInfoManager = rsGroupInfoManager;
077  }
078
079  @Override
080  public RSGroupInfo getRSGroupInfo(String groupName) throws IOException {
081    return rsGroupInfoManager.getRSGroup(groupName);
082  }
083
084  @Override
085  public RSGroupInfo getRSGroupInfoOfTable(TableName tableName) throws IOException {
086    // We are reading across two Maps in the below with out synchronizing across
087    // them; should be safe most of the time.
088    String groupName = rsGroupInfoManager.getRSGroupOfTable(tableName);
089    return groupName == null ? null : rsGroupInfoManager.getRSGroup(groupName);
090  }
091
092  private void checkOnlineServersOnly(Set<Address> servers) throws ConstraintException {
093    // This uglyness is because we only have Address, not ServerName.
094    // Online servers are keyed by ServerName.
095    Set<Address> onlineServers = new HashSet<>();
096    for (ServerName server : master.getServerManager().getOnlineServers().keySet()) {
097      onlineServers.add(server.getAddress());
098    }
099    for (Address address : servers) {
100      if (!onlineServers.contains(address)) {
101        throw new ConstraintException(
102          "Server " + address + " is not an online server in 'default' RSGroup.");
103      }
104    }
105  }
106
107  /**
108   * Check passed name. Fail if nulls or if corresponding RSGroupInfo not found.
109   * @return The RSGroupInfo named <code>name</code>
110   */
111  private RSGroupInfo getAndCheckRSGroupInfo(String name) throws IOException {
112    if (StringUtils.isEmpty(name)) {
113      throw new ConstraintException("RSGroup cannot be null.");
114    }
115    RSGroupInfo rsGroupInfo = getRSGroupInfo(name);
116    if (rsGroupInfo == null) {
117      throw new ConstraintException("RSGroup does not exist: " + name);
118    }
119    return rsGroupInfo;
120  }
121
122  /** Returns List of Regions associated with this <code>server</code>. */
123  private List<RegionInfo> getRegions(final Address server) {
124    LinkedList<RegionInfo> regions = new LinkedList<>();
125    for (Map.Entry<RegionInfo, ServerName> el : master.getAssignmentManager().getRegionStates()
126      .getRegionAssignments().entrySet()) {
127      if (el.getValue() == null) {
128        continue;
129      }
130
131      if (el.getValue().getAddress().equals(server)) {
132        addRegion(regions, el.getKey());
133      }
134    }
135    for (RegionStateNode state : master.getAssignmentManager().getRegionsInTransition()) {
136      if (
137        state.getRegionLocation() != null && state.getRegionLocation().getAddress().equals(server)
138      ) {
139        addRegion(regions, state.getRegionInfo());
140      }
141    }
142    return regions;
143  }
144
145  private void addRegion(final LinkedList<RegionInfo> regions, RegionInfo hri) {
146    // If meta, move it last otherwise other unassigns fail because meta is not
147    // online for them to update state in. This is dodgy. Needs to be made more
148    // robust. See TODO below.
149    if (hri.isMetaRegion()) {
150      regions.addLast(hri);
151    } else {
152      regions.addFirst(hri);
153    }
154  }
155
156  /**
157   * Check servers and tables.
158   * @param servers         servers to move
159   * @param tables          tables to move
160   * @param targetGroupName target group name
161   * @throws IOException if nulls or if servers and tables not belong to the same group
162   */
163  private void checkServersAndTables(Set<Address> servers, Set<TableName> tables,
164    String targetGroupName) throws IOException {
165    // Presume first server's source group. Later ensure all servers are from this group.
166    Address firstServer = servers.iterator().next();
167    RSGroupInfo tmpSrcGrp = rsGroupInfoManager.getRSGroupOfServer(firstServer);
168    if (tmpSrcGrp == null) {
169      // Be careful. This exception message is tested for in TestRSGroupsAdmin2...
170      throw new ConstraintException(
171        "Server " + firstServer + " is either offline or it does not exist.");
172    }
173    RSGroupInfo srcGrp = new RSGroupInfo(tmpSrcGrp);
174
175    // Only move online servers
176    checkOnlineServersOnly(servers);
177
178    // Ensure all servers are of same rsgroup.
179    for (Address server : servers) {
180      String tmpGroup = rsGroupInfoManager.getRSGroupOfServer(server).getName();
181      if (!tmpGroup.equals(srcGrp.getName())) {
182        throw new ConstraintException("Move server request should only come from one source "
183          + "RSGroup. Expecting only " + srcGrp.getName() + " but contains " + tmpGroup);
184      }
185    }
186
187    // Ensure all tables and servers are of same rsgroup.
188    for (TableName table : tables) {
189      String tmpGroup = rsGroupInfoManager.getRSGroupOfTable(table);
190      if (!tmpGroup.equals(srcGrp.getName())) {
191        throw new ConstraintException("Move table request should only come from one source "
192          + "RSGroup. Expecting only " + srcGrp.getName() + " but contains " + tmpGroup);
193      }
194    }
195
196    if (srcGrp.getServers().size() <= servers.size() && srcGrp.getTables().size() > tables.size()) {
197      throw new ConstraintException("Cannot leave a RSGroup " + srcGrp.getName()
198        + " that contains tables without servers to host them.");
199    }
200  }
201
202  /**
203   * Move every region from servers which are currently located on these servers, but should not be
204   * located there.
205   * @param movedServers    the servers that are moved to new group
206   * @param movedTables     the tables that are moved to new group
207   * @param srcGrpServers   all servers in the source group, excluding the movedServers
208   * @param targetGroupName the target group
209   * @param sourceGroupName the source group
210   * @throws IOException if any error while moving regions
211   */
212  private void moveServerRegionsFromGroup(Set<Address> movedServers, Set<TableName> movedTables,
213    Set<Address> srcGrpServers, String targetGroupName, String sourceGroupName) throws IOException {
214    // Get server names corresponding to given Addresses
215    List<ServerName> movedServerNames = new ArrayList<>(movedServers.size());
216    List<ServerName> srcGrpServerNames = new ArrayList<>(srcGrpServers.size());
217    for (ServerName serverName : master.getServerManager().getOnlineServers().keySet()) {
218      // In case region move failed in previous attempt, regionsOwners and newRegionsOwners
219      // can have the same servers. So for all servers below both conditions to be checked
220      if (srcGrpServers.contains(serverName.getAddress())) {
221        srcGrpServerNames.add(serverName);
222      }
223      if (movedServers.contains(serverName.getAddress())) {
224        movedServerNames.add(serverName);
225      }
226    }
227    // Set true to indicate at least one region movement failed
228    boolean errorInRegionMove;
229    List<Pair<RegionInfo, Future<byte[]>>> assignmentFutures = new ArrayList<>();
230    int retry = 0;
231    do {
232      errorInRegionMove = false;
233      for (ServerName server : movedServerNames) {
234        List<RegionInfo> regionsOnServer = getRegions(server.getAddress());
235        for (RegionInfo region : regionsOnServer) {
236          if (
237            !movedTables.contains(region.getTable())
238              && !srcGrpServers.contains(getRegionAddress(region))
239          ) {
240            LOG.info("Moving server region {}, which do not belong to RSGroup {}",
241              region.getShortNameToLog(), targetGroupName);
242            // Move region back to source RSGroup servers
243            ServerName dest =
244              this.master.getLoadBalancer().randomAssignment(region, srcGrpServerNames);
245            if (dest == null) {
246              errorInRegionMove = true;
247              continue;
248            }
249            RegionPlan rp = new RegionPlan(region, server, dest);
250            try {
251              Future<byte[]> future = this.master.getAssignmentManager().moveAsync(rp);
252              assignmentFutures.add(Pair.newPair(region, future));
253            } catch (Exception ioe) {
254              errorInRegionMove = true;
255              LOG.error("Move region {} failed, will retry, current retry time is {}",
256                region.getShortNameToLog(), retry, ioe);
257            }
258          }
259        }
260      }
261      boolean allRegionsMoved = waitForRegionMovement(assignmentFutures, sourceGroupName, retry);
262      if (allRegionsMoved && !errorInRegionMove) {
263        LOG.info("All regions from {} are moved back to {}", movedServerNames, sourceGroupName);
264        return;
265      } else {
266        retry++;
267        try {
268          rsGroupInfoManager.wait(1000);
269        } catch (InterruptedException e) {
270          LOG.warn("Sleep interrupted", e);
271          Thread.currentThread().interrupt();
272        }
273      }
274    } while (retry <= 50);
275  }
276
277  private Address getRegionAddress(RegionInfo hri) {
278    ServerName sn = master.getAssignmentManager().getRegionStates().getRegionServerOfRegion(hri);
279    return sn.getAddress();
280  }
281
282  /**
283   * Wait for all the region move to complete. Keep waiting for other region movement completion
284   * even if some region movement fails.
285   */
286  private boolean waitForRegionMovement(List<Pair<RegionInfo, Future<byte[]>>> regionMoveFutures,
287    String groupName, int retryCount) {
288    LOG.info("Moving {} region(s) to group {}, current retry={}", regionMoveFutures.size(),
289      groupName, retryCount);
290    boolean allRegionsMoved = true;
291    for (Pair<RegionInfo, Future<byte[]>> pair : regionMoveFutures) {
292      try {
293        pair.getSecond().get();
294        if (
295          master.getAssignmentManager().getRegionStates().getRegionState(pair.getFirst())
296            .isFailedOpen()
297        ) {
298          allRegionsMoved = false;
299        }
300      } catch (InterruptedException e) {
301        LOG.warn("Sleep interrupted", e);
302        // Dont return form there lets wait for other regions to complete movement.
303        allRegionsMoved = false;
304      } catch (Exception e) {
305        allRegionsMoved = false;
306        LOG.error("Move region {} to group {} failed, will retry on next attempt",
307          pair.getFirst().getShortNameToLog(), groupName, e);
308      }
309    }
310    return allRegionsMoved;
311  }
312
313  /**
314   * Moves regions of tables which are not on target group servers.
315   * @param tables    the tables that will move to new group
316   * @param targetGrp the target group
317   * @throws IOException if moving the region fails
318   */
319  private void moveTableRegionsToGroup(Set<TableName> tables, RSGroupInfo targetGrp)
320    throws IOException {
321    List<ServerName> targetGrpSevers = new ArrayList<>(targetGrp.getServers().size());
322    for (ServerName serverName : master.getServerManager().getOnlineServers().keySet()) {
323      if (targetGrp.getServers().contains(serverName.getAddress())) {
324        targetGrpSevers.add(serverName);
325      }
326    }
327    // Set true to indicate at least one region movement failed
328    boolean errorInRegionMove;
329    int retry = 0;
330    List<Pair<RegionInfo, Future<byte[]>>> assignmentFutures = new ArrayList<>();
331    do {
332      errorInRegionMove = false;
333      for (TableName table : tables) {
334        if (
335          master.getTableStateManager().isTableState(table, TableState.State.DISABLED,
336            TableState.State.DISABLING)
337        ) {
338          LOG.debug("Skipping move regions because the table {} is disabled", table);
339          continue;
340        }
341        LOG.info("Moving region(s) for table {} to RSGroup {}", table, targetGrp.getName());
342        for (RegionInfo region : master.getAssignmentManager().getRegionStates()
343          .getRegionsOfTable(table)) {
344          ServerName sn =
345            master.getAssignmentManager().getRegionStates().getRegionServerOfRegion(region);
346          if (!targetGrp.containsServer(sn.getAddress())) {
347            LOG.info("Moving region {} to RSGroup {}", region.getShortNameToLog(),
348              targetGrp.getName());
349            ServerName dest =
350              this.master.getLoadBalancer().randomAssignment(region, targetGrpSevers);
351            if (dest == null) {
352              errorInRegionMove = true;
353              continue;
354            }
355            RegionPlan rp = new RegionPlan(region, sn, dest);
356            try {
357              Future<byte[]> future = this.master.getAssignmentManager().moveAsync(rp);
358              assignmentFutures.add(Pair.newPair(region, future));
359            } catch (Exception ioe) {
360              errorInRegionMove = true;
361              LOG.error("Move region {} to group failed, will retry, current retry time is {}",
362                region.getShortNameToLog(), retry, ioe);
363            }
364
365          }
366        }
367      }
368      boolean allRegionsMoved =
369        waitForRegionMovement(assignmentFutures, targetGrp.getName(), retry);
370      if (allRegionsMoved && !errorInRegionMove) {
371        LOG.info("All regions from table(s) {} moved to target group {}.", tables,
372          targetGrp.getName());
373        return;
374      } else {
375        retry++;
376        try {
377          rsGroupInfoManager.wait(1000);
378        } catch (InterruptedException e) {
379          LOG.warn("Sleep interrupted", e);
380          Thread.currentThread().interrupt();
381        }
382      }
383    } while (retry <= 50);
384  }
385
386  @edu.umd.cs.findbugs.annotations.SuppressWarnings(
387      value = "RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE",
388      justification = "Ignoring complaint because don't know what it is complaining about")
389  @Override
390  public void moveServers(Set<Address> servers, String targetGroupName) throws IOException {
391    if (servers == null) {
392      throw new ConstraintException("The list of servers to move cannot be null.");
393    }
394    if (servers.isEmpty()) {
395      // For some reason this difference between null servers and isEmpty is important distinction.
396      // TODO. Why? Stuff breaks if I equate them.
397      return;
398    }
399    // check target group
400    getAndCheckRSGroupInfo(targetGroupName);
401
402    // Hold a lock on the manager instance while moving servers to prevent
403    // another writer changing our state while we are working.
404    synchronized (rsGroupInfoManager) {
405      // Presume first server's source group. Later ensure all servers are from this group.
406      Address firstServer = servers.iterator().next();
407      RSGroupInfo srcGrp = rsGroupInfoManager.getRSGroupOfServer(firstServer);
408      if (srcGrp == null) {
409        // Be careful. This exception message is tested for in TestRSGroupsAdmin2...
410        throw new ConstraintException(
411          "Server " + firstServer + " is either offline or it does not exist.");
412      }
413      // Only move online servers (when moving from 'default') or servers from other
414      // groups. This prevents bogus servers from entering groups
415      if (RSGroupInfo.DEFAULT_GROUP.equals(srcGrp.getName())) {
416        if (srcGrp.getServers().size() <= servers.size()) {
417          throw new ConstraintException(KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE);
418        }
419        checkOnlineServersOnly(servers);
420      }
421      // Ensure all servers are of same rsgroup.
422      for (Address server : servers) {
423        String tmpGroup = rsGroupInfoManager.getRSGroupOfServer(server).getName();
424        if (!tmpGroup.equals(srcGrp.getName())) {
425          throw new ConstraintException("Move server request should only come from one source "
426            + "RSGroup. Expecting only " + srcGrp.getName() + " but contains " + tmpGroup);
427        }
428      }
429      if (srcGrp.getServers().size() <= servers.size() && srcGrp.getTables().size() > 0) {
430        throw new ConstraintException("Cannot leave a RSGroup " + srcGrp.getName()
431          + " that contains tables without servers to host them.");
432      }
433
434      // MovedServers may be < passed in 'servers'.
435      Set<Address> movedServers =
436        rsGroupInfoManager.moveServers(servers, srcGrp.getName(), targetGroupName);
437      moveServerRegionsFromGroup(movedServers, Collections.emptySet(),
438        rsGroupInfoManager.getRSGroup(srcGrp.getName()).getServers(), targetGroupName,
439        srcGrp.getName());
440      LOG.info("Move servers done: {} => {}", srcGrp.getName(), targetGroupName);
441    }
442  }
443
444  @Override
445  public void moveTables(Set<TableName> tables, String targetGroup) throws IOException {
446    if (tables == null) {
447      throw new ConstraintException("The list of tables cannot be null.");
448    }
449    if (tables.size() < 1) {
450      LOG.debug("moveTables() passed an empty set. Ignoring.");
451      return;
452    }
453
454    // Hold a lock on the manager instance while moving servers to prevent
455    // another writer changing our state while we are working.
456    synchronized (rsGroupInfoManager) {
457      if (targetGroup != null) {
458        RSGroupInfo destGroup = rsGroupInfoManager.getRSGroup(targetGroup);
459        if (destGroup == null) {
460          throw new ConstraintException("Target " + targetGroup + " RSGroup does not exist.");
461        }
462        if (destGroup.getServers().size() < 1) {
463          throw new ConstraintException("Target RSGroup must have at least one server.");
464        }
465      }
466      rsGroupInfoManager.moveTables(tables, targetGroup);
467
468      // targetGroup is null when a table is being deleted. In this case no further
469      // action is required.
470      if (targetGroup != null) {
471        modifyOrMoveTables(tables, rsGroupInfoManager.getRSGroup(targetGroup));
472      }
473    }
474  }
475
476  @Override
477  public void addRSGroup(String name) throws IOException {
478    rsGroupInfoManager.addRSGroup(new RSGroupInfo(name));
479  }
480
481  @Override
482  public void removeRSGroup(String name) throws IOException {
483    // Hold a lock on the manager instance while moving servers to prevent
484    // another writer changing our state while we are working.
485    synchronized (rsGroupInfoManager) {
486      RSGroupInfo rsGroupInfo = rsGroupInfoManager.getRSGroup(name);
487      if (rsGroupInfo == null) {
488        throw new ConstraintException("RSGroup " + name + " does not exist");
489      }
490      int tableCount = rsGroupInfo.getTables().size();
491      if (tableCount > 0) {
492        throw new ConstraintException("RSGroup " + name + " has " + tableCount
493          + " tables; you must remove these tables from the rsgroup before "
494          + "the rsgroup can be removed.");
495      }
496      int serverCount = rsGroupInfo.getServers().size();
497      if (serverCount > 0) {
498        throw new ConstraintException("RSGroup " + name + " has " + serverCount
499          + " servers; you must remove these servers from the RSGroup before"
500          + "the RSGroup can be removed.");
501      }
502      for (NamespaceDescriptor ns : master.getClusterSchema().getNamespaces()) {
503        String nsGroup = ns.getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP);
504        if (nsGroup != null && nsGroup.equals(name)) {
505          throw new ConstraintException(
506            "RSGroup " + name + " is referenced by namespace: " + ns.getName());
507        }
508      }
509      rsGroupInfoManager.removeRSGroup(name);
510    }
511  }
512
513  @Override
514  public BalanceResponse balanceRSGroup(String groupName, BalanceRequest request)
515    throws IOException {
516    ServerManager serverManager = master.getServerManager();
517    LoadBalancer balancer = master.getLoadBalancer();
518
519    BalanceResponse.Builder responseBuilder = BalanceResponse.newBuilder();
520
521    synchronized (balancer) {
522      // If balance not true, don't run balancer.
523      if (!((HMaster) master).isBalancerOn() && !request.isDryRun()) {
524        return responseBuilder.build();
525      }
526
527      if (getRSGroupInfo(groupName) == null) {
528        throw new ConstraintException("RSGroup does not exist: " + groupName);
529      }
530
531      // Only allow one balance run at at time.
532      Map<String, RegionState> groupRIT = rsGroupGetRegionsInTransition(groupName);
533      if (groupRIT.size() > 0 && !request.isIgnoreRegionsInTransition()) {
534        LOG.debug("Not running balancer because {} region(s) in transition: {}", groupRIT.size(),
535          StringUtils.abbreviate(
536            master.getAssignmentManager().getRegionStates().getRegionsInTransition().toString(),
537            256));
538        return responseBuilder.build();
539      }
540
541      if (serverManager.areDeadServersInProgress()) {
542        LOG.debug("Not running balancer because processing dead regionserver(s): {}",
543          serverManager.getDeadServers());
544        return responseBuilder.build();
545      }
546
547      // We balance per group instead of per table
548      Map<TableName, Map<ServerName, List<RegionInfo>>> assignmentsByTable =
549        getRSGroupAssignmentsByTable(master.getTableStateManager(), groupName);
550      List<RegionPlan> plans = balancer.balanceCluster(assignmentsByTable);
551      boolean balancerRan = !plans.isEmpty();
552
553      responseBuilder.setBalancerRan(balancerRan).setMovesCalculated(plans.size());
554
555      if (balancerRan && !request.isDryRun()) {
556        LOG.info("RSGroup balance {} starting with plan count: {}", groupName, plans.size());
557        List<RegionPlan> executed = master.executeRegionPlansWithThrottling(plans);
558        responseBuilder.setMovesExecuted(executed.size());
559        LOG.info("RSGroup balance " + groupName + " completed");
560      }
561
562      return responseBuilder.build();
563    }
564  }
565
566  @Override
567  public List<RSGroupInfo> listRSGroups() throws IOException {
568    return rsGroupInfoManager.listRSGroups();
569  }
570
571  @Override
572  public RSGroupInfo getRSGroupOfServer(Address hostPort) throws IOException {
573    return rsGroupInfoManager.getRSGroupOfServer(hostPort);
574  }
575
576  @Override
577  public void moveServersAndTables(Set<Address> servers, Set<TableName> tables, String targetGroup)
578    throws IOException {
579    if (servers == null || servers.isEmpty()) {
580      throw new ConstraintException("The list of servers to move cannot be null or empty.");
581    }
582    if (tables == null || tables.isEmpty()) {
583      throw new ConstraintException("The list of tables to move cannot be null or empty.");
584    }
585
586    // check target group
587    getAndCheckRSGroupInfo(targetGroup);
588
589    // Hold a lock on the manager instance while moving servers and tables to prevent
590    // another writer changing our state while we are working.
591    synchronized (rsGroupInfoManager) {
592      // check servers and tables status
593      checkServersAndTables(servers, tables, targetGroup);
594
595      // Move servers and tables to a new group.
596      String srcGroup = getRSGroupOfServer(servers.iterator().next()).getName();
597      rsGroupInfoManager.moveServersAndTables(servers, tables, srcGroup, targetGroup);
598
599      // move regions on these servers which do not belong to group tables
600      moveServerRegionsFromGroup(servers, tables,
601        rsGroupInfoManager.getRSGroup(srcGroup).getServers(), targetGroup, srcGroup);
602      // move regions of these tables which are not on group servers
603      modifyOrMoveTables(tables, rsGroupInfoManager.getRSGroup(targetGroup));
604    }
605    LOG.info("Move servers and tables done. Severs: {}, Tables: {} => {}", servers, tables,
606      targetGroup);
607  }
608
609  @Override
610  public void removeServers(Set<Address> servers) throws IOException {
611    {
612      if (servers == null || servers.isEmpty()) {
613        throw new ConstraintException("The set of servers to remove cannot be null or empty.");
614      }
615      // Hold a lock on the manager instance while moving servers to prevent
616      // another writer changing our state while we are working.
617      synchronized (rsGroupInfoManager) {
618        // check the set of servers
619        checkForDeadOrOnlineServers(servers);
620        rsGroupInfoManager.removeServers(servers);
621        LOG.info("Remove decommissioned servers {} from RSGroup done", servers);
622      }
623    }
624  }
625
626  @Override
627  public void renameRSGroup(String oldName, String newName) throws IOException {
628    synchronized (rsGroupInfoManager) {
629      rsGroupInfoManager.renameRSGroup(oldName, newName);
630      Set<TableDescriptor> updateTables = master.getTableDescriptors().getAll().values().stream()
631        .filter(t -> oldName.equals(t.getRegionServerGroup().orElse(null)))
632        .collect(Collectors.toSet());
633      // Update rs group info into table descriptors
634      modifyTablesAndWaitForCompletion(updateTables, newName);
635    }
636  }
637
638  @Override
639  public void updateRSGroupConfig(String groupName, Map<String, String> configuration)
640    throws IOException {
641    synchronized (rsGroupInfoManager) {
642      rsGroupInfoManager.updateRSGroupConfig(groupName, configuration);
643    }
644  }
645
646  /**
647   * Because the {@link RSGroupAdminClient#updateConfiguration(String)} calls
648   * {@link org.apache.hadoop.hbase.client.Admin#updateConfiguration(ServerName)} method, the
649   * implementation of this method on the Server side is empty.
650   */
651  @Override
652  public void updateConfiguration(String groupName) throws IOException {
653  }
654
655  private Map<String, RegionState> rsGroupGetRegionsInTransition(String groupName)
656    throws IOException {
657    Map<String, RegionState> rit = Maps.newTreeMap();
658    AssignmentManager am = master.getAssignmentManager();
659    for (TableName tableName : getRSGroupInfo(groupName).getTables()) {
660      for (RegionInfo regionInfo : am.getRegionStates().getRegionsOfTable(tableName)) {
661        RegionState state = am.getRegionStates().getRegionTransitionState(regionInfo);
662        if (state != null) {
663          rit.put(regionInfo.getEncodedName(), state);
664        }
665      }
666    }
667    return rit;
668  }
669
670  /**
671   * This is an EXPENSIVE clone. Cloning though is the safest thing to do. Can't let out original
672   * since it can change and at least the load balancer wants to iterate this exported list. Load
673   * balancer should iterate over this list because cloned list will ignore disabled table and split
674   * parent region cases. This method is invoked by {@link #balanceRSGroup}
675   * @return A clone of current assignments for this group.
676   */
677  Map<TableName, Map<ServerName, List<RegionInfo>>> getRSGroupAssignmentsByTable(
678    TableStateManager tableStateManager, String groupName) throws IOException {
679    Map<TableName, Map<ServerName, List<RegionInfo>>> result = Maps.newHashMap();
680    RSGroupInfo rsGroupInfo = getRSGroupInfo(groupName);
681    Map<TableName, Map<ServerName, List<RegionInfo>>> assignments = Maps.newHashMap();
682    for (Map.Entry<RegionInfo, ServerName> entry : master.getAssignmentManager().getRegionStates()
683      .getRegionAssignments().entrySet()) {
684      TableName currTable = entry.getKey().getTable();
685      ServerName currServer = entry.getValue();
686      RegionInfo currRegion = entry.getKey();
687      if (rsGroupInfo.getTables().contains(currTable)) {
688        if (
689          tableStateManager.isTableState(currTable, TableState.State.DISABLED,
690            TableState.State.DISABLING)
691        ) {
692          continue;
693        }
694        if (currRegion.isSplitParent()) {
695          continue;
696        }
697        assignments.putIfAbsent(currTable, new HashMap<>());
698        assignments.get(currTable).putIfAbsent(currServer, new ArrayList<>());
699        assignments.get(currTable).get(currServer).add(currRegion);
700      }
701    }
702
703    Map<ServerName, List<RegionInfo>> serverMap = Maps.newHashMap();
704    for (ServerName serverName : master.getServerManager().getOnlineServers().keySet()) {
705      if (rsGroupInfo.getServers().contains(serverName.getAddress())) {
706        serverMap.put(serverName, Collections.emptyList());
707      }
708    }
709
710    // add all tables that are members of the group
711    for (TableName tableName : rsGroupInfo.getTables()) {
712      if (assignments.containsKey(tableName)) {
713        result.put(tableName, new HashMap<>());
714        result.get(tableName).putAll(serverMap);
715        result.get(tableName).putAll(assignments.get(tableName));
716        LOG.debug("Adding assignments for {}: {}", tableName, assignments.get(tableName));
717      }
718    }
719
720    return result;
721  }
722
723  /**
724   * Check if the set of servers are belong to dead servers list or online servers list.
725   * @param servers servers to remove
726   */
727  private void checkForDeadOrOnlineServers(Set<Address> servers) throws ConstraintException {
728    // This uglyness is because we only have Address, not ServerName.
729    Set<Address> onlineServers = new HashSet<>();
730    List<ServerName> drainingServers = master.getServerManager().getDrainingServersList();
731    for (ServerName server : master.getServerManager().getOnlineServers().keySet()) {
732      // Only online but not decommissioned servers are really online
733      if (!drainingServers.contains(server)) {
734        onlineServers.add(server.getAddress());
735      }
736    }
737
738    Set<Address> deadServers = new HashSet<>();
739    for (ServerName server : master.getServerManager().getDeadServers().copyServerNames()) {
740      deadServers.add(server.getAddress());
741    }
742
743    for (Address address : servers) {
744      if (onlineServers.contains(address)) {
745        throw new ConstraintException(
746          "Server " + address + " is an online server, not allowed to remove.");
747      }
748      if (deadServers.contains(address)) {
749        throw new ConstraintException("Server " + address + " is on the dead servers list,"
750          + " Maybe it will come back again, not allowed to remove.");
751      }
752    }
753  }
754
755  // Modify table or move table's regions
756  void modifyOrMoveTables(Set<TableName> tables, RSGroupInfo targetGroup) throws IOException {
757    Set<TableName> tablesToBeMoved = new HashSet<>(tables.size());
758    Set<TableDescriptor> tablesToBeModified = new HashSet<>(tables.size());
759    // Segregate tables into to be modified or to be moved category
760    for (TableName tableName : tables) {
761      TableDescriptor descriptor = master.getTableDescriptors().get(tableName);
762      if (descriptor == null) {
763        LOG.error(
764          "TableDescriptor of table {} not found. Skipping the region movement of this table.");
765        continue;
766      }
767      if (descriptor.getRegionServerGroup().isPresent()) {
768        tablesToBeModified.add(descriptor);
769      } else {
770        tablesToBeMoved.add(tableName);
771      }
772    }
773    List<Long> procedureIds = null;
774    if (!tablesToBeModified.isEmpty()) {
775      procedureIds = modifyTables(tablesToBeModified, targetGroup.getName());
776    }
777    if (!tablesToBeMoved.isEmpty()) {
778      moveTableRegionsToGroup(tablesToBeMoved, targetGroup);
779    }
780    // By this time moveTableRegionsToGroup is finished, lets wait for modifyTables completion
781    if (procedureIds != null) {
782      waitForProcedureCompletion(procedureIds);
783    }
784  }
785
786  private void modifyTablesAndWaitForCompletion(Set<TableDescriptor> tableDescriptors,
787    String targetGroup) throws IOException {
788    final List<Long> procIds = modifyTables(tableDescriptors, targetGroup);
789    waitForProcedureCompletion(procIds);
790  }
791
792  // Modify table internally moves the regions as well. So separate region movement is not needed
793  private List<Long> modifyTables(Set<TableDescriptor> tableDescriptors, String targetGroup)
794    throws IOException {
795    List<Long> procIds = new ArrayList<>(tableDescriptors.size());
796    for (TableDescriptor oldTd : tableDescriptors) {
797      TableDescriptor newTd =
798        TableDescriptorBuilder.newBuilder(oldTd).setRegionServerGroup(targetGroup).build();
799      procIds.add(
800        master.modifyTable(oldTd.getTableName(), newTd, HConstants.NO_NONCE, HConstants.NO_NONCE));
801    }
802    return procIds;
803  }
804
805  private void waitForProcedureCompletion(List<Long> procIds) throws IOException {
806    for (long procId : procIds) {
807      Procedure<?> proc = master.getMasterProcedureExecutor().getProcedure(procId);
808      if (proc == null) {
809        continue;
810      }
811      ProcedureSyncWait.waitForProcedureToCompleteIOE(master.getMasterProcedureExecutor(), proc,
812        Long.MAX_VALUE);
813    }
814  }
815}