001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.rsgroup;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.Collections;
023import java.util.HashMap;
024import java.util.HashSet;
025import java.util.Iterator;
026import java.util.LinkedList;
027import java.util.List;
028import java.util.Map;
029import java.util.Set;
030import org.apache.commons.lang3.StringUtils;
031import org.apache.hadoop.hbase.NamespaceDescriptor;
032import org.apache.hadoop.hbase.ServerName;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.client.RegionInfo;
035import org.apache.hadoop.hbase.constraint.ConstraintException;
036import org.apache.hadoop.hbase.master.HMaster;
037import org.apache.hadoop.hbase.master.LoadBalancer;
038import org.apache.hadoop.hbase.master.MasterServices;
039import org.apache.hadoop.hbase.master.RegionPlan;
040import org.apache.hadoop.hbase.master.RegionState;
041import org.apache.hadoop.hbase.master.ServerManager;
042import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
043import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
044import org.apache.hadoop.hbase.net.Address;
045import org.apache.yetus.audience.InterfaceAudience;
046import org.slf4j.Logger;
047import org.slf4j.LoggerFactory;
048
049import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
050
051/**
052 * Service to support Region Server Grouping (HBase-6721).
053 */
054@InterfaceAudience.Private
055public class RSGroupAdminServer implements RSGroupAdmin {
056  private static final Logger LOG = LoggerFactory.getLogger(RSGroupAdminServer.class);
057  public static final String KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE = "should keep at least " +
058          "one server in 'default' RSGroup.";
059
060  private MasterServices master;
061  private final RSGroupInfoManager rsGroupInfoManager;
062
063  public RSGroupAdminServer(MasterServices master, RSGroupInfoManager rsGroupInfoManager) {
064    this.master = master;
065    this.rsGroupInfoManager = rsGroupInfoManager;
066  }
067
068  @Override
069  public RSGroupInfo getRSGroupInfo(String groupName) throws IOException {
070    return rsGroupInfoManager.getRSGroup(groupName);
071  }
072
073  @Override
074  public RSGroupInfo getRSGroupInfoOfTable(TableName tableName) throws IOException {
075    // We are reading across two Maps in the below with out synchronizing across
076    // them; should be safe most of the time.
077    String groupName = rsGroupInfoManager.getRSGroupOfTable(tableName);
078    return groupName == null? null: rsGroupInfoManager.getRSGroup(groupName);
079  }
080
081  private void checkOnlineServersOnly(Set<Address> servers) throws ConstraintException {
082    // This uglyness is because we only have Address, not ServerName.
083    // Online servers are keyed by ServerName.
084    Set<Address> onlineServers = new HashSet<>();
085    for(ServerName server: master.getServerManager().getOnlineServers().keySet()) {
086      onlineServers.add(server.getAddress());
087    }
088    for (Address address: servers) {
089      if (!onlineServers.contains(address)) {
090        throw new ConstraintException(
091            "Server " + address + " is not an online server in 'default' RSGroup.");
092      }
093    }
094  }
095
096  /**
097   * Check passed name. Fail if nulls or if corresponding RSGroupInfo not found.
098   * @return The RSGroupInfo named <code>name</code>
099   */
100  private RSGroupInfo getAndCheckRSGroupInfo(String name) throws IOException {
101    if (StringUtils.isEmpty(name)) {
102      throw new ConstraintException("RSGroup cannot be null.");
103    }
104    RSGroupInfo rsGroupInfo = getRSGroupInfo(name);
105    if (rsGroupInfo == null) {
106      throw new ConstraintException("RSGroup does not exist: " + name);
107    }
108    return rsGroupInfo;
109  }
110
111  /**
112   * @return List of Regions associated with this <code>server</code>.
113   */
114  private List<RegionInfo> getRegions(final Address server) {
115    LinkedList<RegionInfo> regions = new LinkedList<>();
116    for (Map.Entry<RegionInfo, ServerName> el :
117        master.getAssignmentManager().getRegionStates().getRegionAssignments().entrySet()) {
118      if (el.getValue() == null) {
119        continue;
120      }
121
122      if (el.getValue().getAddress().equals(server)) {
123        addRegion(regions, el.getKey());
124      }
125    }
126    for (RegionStateNode state : master.getAssignmentManager().getRegionsInTransition()) {
127      if (state.getRegionLocation() != null &&
128          state.getRegionLocation().getAddress().equals(server)) {
129        addRegion(regions, state.getRegionInfo());
130      }
131    }
132    return regions;
133  }
134
135  private void addRegion(final LinkedList<RegionInfo> regions, RegionInfo hri) {
136    // If meta, move it last otherwise other unassigns fail because meta is not
137    // online for them to update state in. This is dodgy. Needs to be made more
138    // robust. See TODO below.
139    if (hri.isMetaRegion()) {
140      regions.addLast(hri);
141    } else {
142      regions.addFirst(hri);
143    }
144  }
145
146  /**
147   * Check servers and tables.
148   *
149   * @param servers servers to move
150   * @param tables tables to move
151   * @param targetGroupName target group name
152   * @throws IOException if nulls or if servers and tables not belong to the same group
153   */
154  private void checkServersAndTables(Set<Address> servers, Set<TableName> tables,
155                                     String targetGroupName) throws IOException {
156    // Presume first server's source group. Later ensure all servers are from this group.
157    Address firstServer = servers.iterator().next();
158    RSGroupInfo tmpSrcGrp = rsGroupInfoManager.getRSGroupOfServer(firstServer);
159    if (tmpSrcGrp == null) {
160      // Be careful. This exception message is tested for in TestRSGroupsBase...
161      throw new ConstraintException("Source RSGroup for server " + firstServer
162              + " does not exist.");
163    }
164    RSGroupInfo srcGrp = new RSGroupInfo(tmpSrcGrp);
165    if (srcGrp.getName().equals(targetGroupName)) {
166      throw new ConstraintException("Target RSGroup " + targetGroupName +
167              " is same as source " + srcGrp.getName() + " RSGroup.");
168    }
169    // Only move online servers
170    checkOnlineServersOnly(servers);
171
172    // Ensure all servers are of same rsgroup.
173    for (Address server: servers) {
174      String tmpGroup = rsGroupInfoManager.getRSGroupOfServer(server).getName();
175      if (!tmpGroup.equals(srcGrp.getName())) {
176        throw new ConstraintException("Move server request should only come from one source " +
177                "RSGroup. Expecting only " + srcGrp.getName() + " but contains " + tmpGroup);
178      }
179    }
180
181    // Ensure all tables and servers are of same rsgroup.
182    for (TableName table : tables) {
183      String tmpGroup = rsGroupInfoManager.getRSGroupOfTable(table);
184      if (!tmpGroup.equals(srcGrp.getName())) {
185        throw new ConstraintException("Move table request should only come from one source " +
186                "RSGroup. Expecting only " + srcGrp.getName() + " but contains " + tmpGroup);
187      }
188    }
189
190    if (srcGrp.getServers().size() <= servers.size() && srcGrp.getTables().size() > tables.size()) {
191      throw new ConstraintException("Cannot leave a RSGroup " + srcGrp.getName() +
192              " that contains tables without servers to host them.");
193    }
194  }
195
196  /**
197   * Move every region from servers which are currently located on these servers,
198   * but should not be located there.
199   *
200   * @param servers the servers that will move to new group
201   * @param targetGroupName the target group name
202   * @throws IOException if moving the server and tables fail
203   */
204  private void moveServerRegionsFromGroup(Set<Address> servers, String targetGroupName)
205      throws IOException {
206    boolean hasRegionsToMove;
207    RSGroupInfo targetGrp = getRSGroupInfo(targetGroupName);
208    Set<Address> allSevers = new HashSet<>(servers);
209    do {
210      hasRegionsToMove = false;
211      for (Iterator<Address> iter = allSevers.iterator(); iter.hasNext();) {
212        Address rs = iter.next();
213        // Get regions that are associated with this server and filter regions by group tables.
214        for (RegionInfo region : getRegions(rs)) {
215          if (!targetGrp.containsTable(region.getTable())) {
216            LOG.info("Moving server region {}, which do not belong to RSGroup {}",
217                region.getShortNameToLog(), targetGroupName);
218            this.master.getAssignmentManager().move(region);
219            if (master.getAssignmentManager().getRegionStates().
220                getRegionState(region).isFailedOpen()) {
221              continue;
222            }
223            hasRegionsToMove = true;
224          }
225        }
226
227        if (!hasRegionsToMove) {
228          LOG.info("Server {} has no more regions to move for RSGroup", rs.getHostname());
229          iter.remove();
230        }
231      }
232      try {
233        rsGroupInfoManager.wait(1000);
234      } catch (InterruptedException e) {
235        LOG.warn("Sleep interrupted", e);
236        Thread.currentThread().interrupt();
237      }
238    } while (hasRegionsToMove);
239  }
240
241  /**
242   * Moves regions of tables which are not on target group servers.
243   *
244   * @param tables the tables that will move to new group
245   * @param targetGroupName the target group name
246   * @throws IOException if moving the region fails
247   */
248  private void moveTableRegionsToGroup(Set<TableName> tables, String targetGroupName)
249      throws IOException {
250    RSGroupInfo targetGrp = getRSGroupInfo(targetGroupName);
251    for (TableName table : tables) {
252      if (master.getAssignmentManager().isTableDisabled(table)) {
253        LOG.debug("Skipping move regions because the table {} is disabled", table);
254        continue;
255      }
256      LOG.info("Moving region(s) for table {} to RSGroup {}", table, targetGroupName);
257      for (RegionInfo region : master.getAssignmentManager().getRegionStates()
258          .getRegionsOfTable(table)) {
259        ServerName sn =
260            master.getAssignmentManager().getRegionStates().getRegionServerOfRegion(region);
261        if (!targetGrp.containsServer(sn.getAddress())) {
262          LOG.info("Moving region {} to RSGroup {}", region.getShortNameToLog(), targetGroupName);
263          master.getAssignmentManager().move(region);
264        }
265      }
266    }
267  }
268
269  @edu.umd.cs.findbugs.annotations.SuppressWarnings(
270      value="RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE",
271      justification="Ignoring complaint because don't know what it is complaining about")
272  @Override
273  public void moveServers(Set<Address> servers, String targetGroupName) throws IOException {
274    if (servers == null) {
275      throw new ConstraintException("The list of servers to move cannot be null.");
276    }
277    if (servers.isEmpty()) {
278      // For some reason this difference between null servers and isEmpty is important distinction.
279      // TODO. Why? Stuff breaks if I equate them.
280      return;
281    }
282    //check target group
283    getAndCheckRSGroupInfo(targetGroupName);
284
285    // Hold a lock on the manager instance while moving servers to prevent
286    // another writer changing our state while we are working.
287    synchronized (rsGroupInfoManager) {
288      // Presume first server's source group. Later ensure all servers are from this group.
289      Address firstServer = servers.iterator().next();
290      RSGroupInfo srcGrp = rsGroupInfoManager.getRSGroupOfServer(firstServer);
291      if (srcGrp == null) {
292        // Be careful. This exception message is tested for in TestRSGroupsBase...
293        throw new ConstraintException("Source RSGroup for server " + firstServer
294            + " does not exist.");
295      }
296      if (srcGrp.getName().equals(targetGroupName)) {
297        throw new ConstraintException("Target RSGroup " + targetGroupName +
298            " is same as source " + srcGrp + " RSGroup.");
299      }
300      // Only move online servers (when moving from 'default') or servers from other
301      // groups. This prevents bogus servers from entering groups
302      if (RSGroupInfo.DEFAULT_GROUP.equals(srcGrp.getName())) {
303        if (srcGrp.getServers().size() <= servers.size()) {
304          throw new ConstraintException(KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE);
305        }
306        checkOnlineServersOnly(servers);
307      }
308      // Ensure all servers are of same rsgroup.
309      for (Address server: servers) {
310        String tmpGroup = rsGroupInfoManager.getRSGroupOfServer(server).getName();
311        if (!tmpGroup.equals(srcGrp.getName())) {
312          throw new ConstraintException("Move server request should only come from one source " +
313              "RSGroup. Expecting only " + srcGrp.getName() + " but contains " + tmpGroup);
314        }
315      }
316      if (srcGrp.getServers().size() <= servers.size() && srcGrp.getTables().size() > 0) {
317        throw new ConstraintException("Cannot leave a RSGroup " + srcGrp.getName() +
318            " that contains tables without servers to host them.");
319      }
320
321      // MovedServers may be < passed in 'servers'.
322      Set<Address> movedServers = rsGroupInfoManager.moveServers(servers, srcGrp.getName(),
323          targetGroupName);
324      moveServerRegionsFromGroup(movedServers, targetGroupName);
325      LOG.info("Move servers done: {} => {}", srcGrp.getName(), targetGroupName);
326    }
327  }
328
329  @Override
330  public void moveTables(Set<TableName> tables, String targetGroup) throws IOException {
331    if (tables == null) {
332      throw new ConstraintException("The list of servers cannot be null.");
333    }
334    if (tables.size() < 1) {
335      LOG.debug("moveTables() passed an empty set. Ignoring.");
336      return;
337    }
338
339    // Hold a lock on the manager instance while moving servers to prevent
340    // another writer changing our state while we are working.
341    synchronized (rsGroupInfoManager) {
342      if(targetGroup != null) {
343        RSGroupInfo destGroup = rsGroupInfoManager.getRSGroup(targetGroup);
344        if(destGroup == null) {
345          throw new ConstraintException("Target " + targetGroup + " RSGroup does not exist.");
346        }
347        if(destGroup.getServers().size() < 1) {
348          throw new ConstraintException("Target RSGroup must have at least one server.");
349        }
350      }
351
352      for (TableName table : tables) {
353        String srcGroup = rsGroupInfoManager.getRSGroupOfTable(table);
354        if(srcGroup != null && srcGroup.equals(targetGroup)) {
355          throw new ConstraintException(
356              "Source RSGroup " + srcGroup + " is same as target " + targetGroup +
357              " RSGroup for table " + table);
358        }
359        LOG.info("Moving table {} to RSGroup {}", table.getNameAsString(), targetGroup);
360      }
361      rsGroupInfoManager.moveTables(tables, targetGroup);
362
363      // targetGroup is null when a table is being deleted. In this case no further
364      // action is required.
365      if (targetGroup != null) {
366        moveTableRegionsToGroup(tables, targetGroup);
367      }
368    }
369  }
370
371  @Override
372  public void addRSGroup(String name) throws IOException {
373    rsGroupInfoManager.addRSGroup(new RSGroupInfo(name));
374  }
375
376  @Override
377  public void removeRSGroup(String name) throws IOException {
378    // Hold a lock on the manager instance while moving servers to prevent
379    // another writer changing our state while we are working.
380    synchronized (rsGroupInfoManager) {
381      RSGroupInfo rsGroupInfo = rsGroupInfoManager.getRSGroup(name);
382      if (rsGroupInfo == null) {
383        throw new ConstraintException("RSGroup " + name + " does not exist");
384      }
385      int tableCount = rsGroupInfo.getTables().size();
386      if (tableCount > 0) {
387        throw new ConstraintException("RSGroup " + name + " has " + tableCount +
388            " tables; you must remove these tables from the rsgroup before " +
389            "the rsgroup can be removed.");
390      }
391      int serverCount = rsGroupInfo.getServers().size();
392      if (serverCount > 0) {
393        throw new ConstraintException("RSGroup " + name + " has " + serverCount +
394            " servers; you must remove these servers from the RSGroup before" +
395            "the RSGroup can be removed.");
396      }
397      for (NamespaceDescriptor ns : master.getClusterSchema().getNamespaces()) {
398        String nsGroup = ns.getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP);
399        if (nsGroup != null && nsGroup.equals(name)) {
400          throw new ConstraintException(
401            "RSGroup " + name + " is referenced by namespace: " + ns.getName());
402        }
403      }
404      rsGroupInfoManager.removeRSGroup(name);
405    }
406  }
407
408  @Override
409  public boolean balanceRSGroup(String groupName) throws IOException {
410    ServerManager serverManager = master.getServerManager();
411    LoadBalancer balancer = master.getLoadBalancer();
412
413    synchronized (balancer) {
414      // If balance not true, don't run balancer.
415      if (!((HMaster) master).isBalancerOn()) {
416        return false;
417      }
418
419      if (getRSGroupInfo(groupName) == null) {
420        throw new ConstraintException("RSGroup does not exist: "+groupName);
421      }
422      // Only allow one balance run at at time.
423      Map<String, RegionState> groupRIT = rsGroupGetRegionsInTransition(groupName);
424      if (groupRIT.size() > 0) {
425        LOG.debug("Not running balancer because {} region(s) in transition: {}", groupRIT.size(),
426            StringUtils.abbreviate(
427              master.getAssignmentManager().getRegionStates().getRegionsInTransition().toString(),
428              256));
429        return false;
430      }
431      if (serverManager.areDeadServersInProgress()) {
432        LOG.debug("Not running balancer because processing dead regionserver(s): {}",
433            serverManager.getDeadServers());
434        return false;
435      }
436
437      //We balance per group instead of per table
438      List<RegionPlan> plans = new ArrayList<>();
439      for(Map.Entry<TableName, Map<ServerName, List<RegionInfo>>> tableMap:
440          getRSGroupAssignmentsByTable(groupName).entrySet()) {
441        LOG.info("Creating partial plan for table {} : {}", tableMap.getKey(), tableMap.getValue());
442        List<RegionPlan> partialPlans = balancer.balanceCluster(tableMap.getValue());
443        LOG.info("Partial plan for table {} : {}", tableMap.getKey(), partialPlans);
444        if (partialPlans != null) {
445          plans.addAll(partialPlans);
446        }
447      }
448      boolean balancerRan = !plans.isEmpty();
449      if (balancerRan) {
450        LOG.info("RSGroup balance {} starting with plan count: {}", groupName, plans.size());
451        master.executeRegionPlansWithThrottling(plans);
452        LOG.info("RSGroup balance " + groupName + " completed");
453      }
454      return balancerRan;
455    }
456  }
457
458  @Override
459  public List<RSGroupInfo> listRSGroups() throws IOException {
460    return rsGroupInfoManager.listRSGroups();
461  }
462
463  @Override
464  public RSGroupInfo getRSGroupOfServer(Address hostPort) throws IOException {
465    return rsGroupInfoManager.getRSGroupOfServer(hostPort);
466  }
467
468  @Override
469  public void moveServersAndTables(Set<Address> servers, Set<TableName> tables, String targetGroup)
470      throws IOException {
471    if (servers == null || servers.isEmpty()) {
472      throw new ConstraintException("The list of servers to move cannot be null or empty.");
473    }
474    if (tables == null || tables.isEmpty()) {
475      throw new ConstraintException("The list of tables to move cannot be null or empty.");
476    }
477
478    //check target group
479    getAndCheckRSGroupInfo(targetGroup);
480
481    // Hold a lock on the manager instance while moving servers and tables to prevent
482    // another writer changing our state while we are working.
483    synchronized (rsGroupInfoManager) {
484      //check servers and tables status
485      checkServersAndTables(servers, tables, targetGroup);
486
487      //Move servers and tables to a new group.
488      String srcGroup = getRSGroupOfServer(servers.iterator().next()).getName();
489      rsGroupInfoManager.moveServersAndTables(servers, tables, srcGroup, targetGroup);
490
491      //move regions on these servers which do not belong to group tables
492      moveServerRegionsFromGroup(servers, targetGroup);
493      //move regions of these tables which are not on group servers
494      moveTableRegionsToGroup(tables, targetGroup);
495    }
496    LOG.info("Move servers and tables done. Severs: {}, Tables: {} => {}", servers, tables,
497        targetGroup);
498  }
499
500  @Override
501  public void removeServers(Set<Address> servers) throws IOException {
502    {
503      if (servers == null || servers.isEmpty()) {
504        throw new ConstraintException("The set of servers to remove cannot be null or empty.");
505      }
506      // Hold a lock on the manager instance while moving servers to prevent
507      // another writer changing our state while we are working.
508      synchronized (rsGroupInfoManager) {
509        //check the set of servers
510        checkForDeadOrOnlineServers(servers);
511        rsGroupInfoManager.removeServers(servers);
512        LOG.info("Remove decommissioned servers {} from RSGroup done", servers);
513      }
514    }
515  }
516
517  private Map<String, RegionState> rsGroupGetRegionsInTransition(String groupName)
518      throws IOException {
519    Map<String, RegionState> rit = Maps.newTreeMap();
520    AssignmentManager am = master.getAssignmentManager();
521    for(TableName tableName : getRSGroupInfo(groupName).getTables()) {
522      for(RegionInfo regionInfo: am.getRegionStates().getRegionsOfTable(tableName)) {
523        RegionState state = am.getRegionStates().getRegionTransitionState(regionInfo);
524        if(state != null) {
525          rit.put(regionInfo.getEncodedName(), state);
526        }
527      }
528    }
529    return rit;
530  }
531
532  private Map<TableName, Map<ServerName, List<RegionInfo>>>
533      getRSGroupAssignmentsByTable(String groupName) throws IOException {
534    Map<TableName, Map<ServerName, List<RegionInfo>>> result = Maps.newHashMap();
535    RSGroupInfo rsGroupInfo = getRSGroupInfo(groupName);
536    Map<TableName, Map<ServerName, List<RegionInfo>>> assignments = Maps.newHashMap();
537    for(Map.Entry<RegionInfo, ServerName> entry:
538        master.getAssignmentManager().getRegionStates().getRegionAssignments().entrySet()) {
539      TableName currTable = entry.getKey().getTable();
540      ServerName currServer = entry.getValue();
541      RegionInfo currRegion = entry.getKey();
542      if (rsGroupInfo.getTables().contains(currTable)) {
543        assignments.putIfAbsent(currTable, new HashMap<>());
544        assignments.get(currTable).putIfAbsent(currServer, new ArrayList<>());
545        assignments.get(currTable).get(currServer).add(currRegion);
546      }
547    }
548
549    Map<ServerName, List<RegionInfo>> serverMap = Maps.newHashMap();
550    for(ServerName serverName: master.getServerManager().getOnlineServers().keySet()) {
551      if(rsGroupInfo.getServers().contains(serverName.getAddress())) {
552        serverMap.put(serverName, Collections.emptyList());
553      }
554    }
555
556    // add all tables that are members of the group
557    for(TableName tableName : rsGroupInfo.getTables()) {
558      if(assignments.containsKey(tableName)) {
559        result.put(tableName, new HashMap<>());
560        result.get(tableName).putAll(serverMap);
561        result.get(tableName).putAll(assignments.get(tableName));
562        LOG.debug("Adding assignments for {}: {}", tableName, assignments.get(tableName));
563      }
564    }
565
566    return result;
567  }
568
569  /**
570   * Check if the set of servers are belong to dead servers list or online servers list.
571   * @param servers servers to remove
572   */
573  private void checkForDeadOrOnlineServers(Set<Address> servers) throws ConstraintException {
574    // This uglyness is because we only have Address, not ServerName.
575    Set<Address> onlineServers = new HashSet<>();
576    List<ServerName> drainingServers = master.getServerManager().getDrainingServersList();
577    for (ServerName server : master.getServerManager().getOnlineServers().keySet()) {
578      // Only online but not decommissioned servers are really online
579      if (!drainingServers.contains(server)) {
580        onlineServers.add(server.getAddress());
581      }
582    }
583
584    Set<Address> deadServers = new HashSet<>();
585    for(ServerName server: master.getServerManager().getDeadServers().copyServerNames()) {
586      deadServers.add(server.getAddress());
587    }
588
589    for (Address address: servers) {
590      if (onlineServers.contains(address)) {
591        throw new ConstraintException(
592            "Server " + address + " is an online server, not allowed to remove.");
593      }
594      if (deadServers.contains(address)) {
595        throw new ConstraintException(
596            "Server " + address + " is on the dead servers list,"
597                + " Maybe it will come back again, not allowed to remove.");
598      }
599    }
600  }
601}