001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.rsgroup;
019
020import static org.junit.Assert.assertTrue;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.EnumSet;
025import java.util.HashSet;
026import java.util.LinkedList;
027import java.util.List;
028import java.util.Map;
029import java.util.Optional;
030import java.util.Set;
031import java.util.TreeMap;
032import java.util.concurrent.ThreadLocalRandom;
033import java.util.regex.Pattern;
034import org.apache.hadoop.conf.Configuration;
035import org.apache.hadoop.hbase.ClusterMetrics;
036import org.apache.hadoop.hbase.ClusterMetrics.Option;
037import org.apache.hadoop.hbase.HBaseClusterInterface;
038import org.apache.hadoop.hbase.HBaseTestingUtil;
039import org.apache.hadoop.hbase.NamespaceDescriptor;
040import org.apache.hadoop.hbase.ServerName;
041import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
042import org.apache.hadoop.hbase.TableName;
043import org.apache.hadoop.hbase.Waiter;
044import org.apache.hadoop.hbase.client.AbstractTestUpdateConfiguration;
045import org.apache.hadoop.hbase.client.Admin;
046import org.apache.hadoop.hbase.client.BalanceRequest;
047import org.apache.hadoop.hbase.client.BalanceResponse;
048import org.apache.hadoop.hbase.client.RegionInfo;
049import org.apache.hadoop.hbase.client.TableDescriptor;
050import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
051import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
052import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
053import org.apache.hadoop.hbase.coprocessor.MasterObserver;
054import org.apache.hadoop.hbase.coprocessor.ObserverContext;
055import org.apache.hadoop.hbase.master.HMaster;
056import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
057import org.apache.hadoop.hbase.master.ServerManager;
058import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
059import org.apache.hadoop.hbase.net.Address;
060import org.apache.hadoop.hbase.quotas.QuotaUtil;
061import org.junit.Rule;
062import org.junit.rules.TestName;
063import org.slf4j.Logger;
064import org.slf4j.LoggerFactory;
065
066import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
067import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
068
069public abstract class TestRSGroupsBase extends AbstractTestUpdateConfiguration {
070  protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsBase.class);
071
072  // shared
073  protected static final String GROUP_PREFIX = "Group";
074  protected static final String TABLE_PREFIX = "Group";
075
076  // shared, cluster type specific
077  protected static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
078  protected static Admin ADMIN;
079  protected static HBaseClusterInterface CLUSTER;
080  protected static HMaster MASTER;
081  protected boolean INIT = false;
082  protected static CPMasterObserver OBSERVER;
083
084  public final static long WAIT_TIMEOUT = 60000;
085  public final static int NUM_SLAVES_BASE = 4; // number of slaves for the smallest cluster
086  public static int NUM_DEAD_SERVERS = 0;
087
088  // Per test variables
089  @Rule
090  public TestName name = new TestName();
091  protected TableName tableName;
092
093  public static String getNameWithoutIndex(String name) {
094    return name.split("\\[")[0];
095  }
096
097  public static void setUpTestBeforeClass() throws Exception {
098    Configuration conf = TEST_UTIL.getConfiguration();
099    conf.setFloat("hbase.master.balancer.stochastic.tableSkewCost", 6000);
100    if (conf.get(RSGroupUtil.RS_GROUP_ENABLED) == null) {
101      RSGroupUtil.enableRSGroup(conf);
102    }
103    if (conf.get(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY) != null) {
104      conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
105        conf.get(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY) + ","
106          + CPMasterObserver.class.getName());
107    } else {
108      conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, CPMasterObserver.class.getName());
109    }
110
111    conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, NUM_SLAVES_BASE);
112    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
113    conf.setInt("hbase.rpc.timeout", 100000);
114
115    TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE);
116    initialize();
117  }
118
119  protected static void initialize() throws Exception {
120    ADMIN = new VerifyingRSGroupAdmin(TEST_UTIL.getConfiguration());
121    CLUSTER = TEST_UTIL.getHBaseCluster();
122    MASTER = TEST_UTIL.getMiniHBaseCluster().getMaster();
123
124    // wait for balancer to come online
125    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
126      @Override
127      public boolean evaluate() throws Exception {
128        return MASTER.isInitialized()
129          && ((RSGroupBasedLoadBalancer) MASTER.getLoadBalancer()).isOnline();
130      }
131    });
132    ADMIN.balancerSwitch(false, true);
133    MasterCoprocessorHost host = MASTER.getMasterCoprocessorHost();
134    OBSERVER = (CPMasterObserver) host.findCoprocessor(CPMasterObserver.class.getName());
135  }
136
137  public static void tearDownAfterClass() throws Exception {
138    TEST_UTIL.shutdownMiniCluster();
139  }
140
141  public void setUpBeforeMethod() throws Exception {
142    LOG.info(name.getMethodName());
143    tableName = TableName.valueOf(TABLE_PREFIX + "_" + name.getMethodName().split("\\[")[0]);
144    if (!INIT) {
145      INIT = true;
146      tearDownAfterMethod();
147    }
148    OBSERVER.resetFlags();
149  }
150
151  public void tearDownAfterMethod() throws Exception {
152    deleteTableIfNecessary();
153    deleteNamespaceIfNecessary();
154    deleteGroups();
155
156    for (ServerName sn : ADMIN.listDecommissionedRegionServers()) {
157      ADMIN.recommissionRegionServer(sn, null);
158    }
159    assertTrue(ADMIN.listDecommissionedRegionServers().isEmpty());
160
161    int missing = NUM_SLAVES_BASE - getNumServers();
162    LOG.info("Restoring servers: " + missing);
163    for (int i = 0; i < missing; i++) {
164      ((SingleProcessHBaseCluster) CLUSTER).startRegionServer();
165    }
166    ADMIN.addRSGroup("master");
167    ServerName masterServerName = ((SingleProcessHBaseCluster) CLUSTER).getMaster().getServerName();
168    try {
169      ADMIN.moveServersToRSGroup(Sets.newHashSet(masterServerName.getAddress()), "master");
170    } catch (Exception ex) {
171      LOG.warn("Got this on setup, FYI", ex);
172    }
173    assertTrue(OBSERVER.preMoveServersCalled);
174    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
175      @Override
176      public boolean evaluate() throws Exception {
177        LOG.info("Waiting for cleanup to finish " + ADMIN.listRSGroups());
178        // Might be greater since moving servers back to default
179        // is after starting a server
180
181        return ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().size() == NUM_SLAVES_BASE;
182      }
183    });
184  }
185
186  protected final RSGroupInfo addGroup(String groupName, int serverCount)
187    throws IOException, InterruptedException {
188    RSGroupInfo defaultInfo = ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP);
189    ADMIN.addRSGroup(groupName);
190    Set<Address> set = new HashSet<>();
191    for (Address server : defaultInfo.getServers()) {
192      if (set.size() == serverCount) {
193        break;
194      }
195      set.add(server);
196    }
197    ADMIN.moveServersToRSGroup(set, groupName);
198    RSGroupInfo result = ADMIN.getRSGroup(groupName);
199    return result;
200  }
201
202  protected final void removeGroup(String groupName) throws IOException {
203    Set<TableName> tables = new HashSet<>();
204    for (TableDescriptor td : ADMIN.listTableDescriptors(true)) {
205      RSGroupInfo groupInfo = ADMIN.getRSGroup(td.getTableName());
206      if (groupInfo != null && groupInfo.getName().equals(groupName)) {
207        tables.add(td.getTableName());
208      }
209    }
210    ADMIN.setRSGroup(tables, RSGroupInfo.DEFAULT_GROUP);
211    for (NamespaceDescriptor nd : ADMIN.listNamespaceDescriptors()) {
212      if (groupName.equals(nd.getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP))) {
213        nd.removeConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP);
214        ADMIN.modifyNamespace(nd);
215      }
216    }
217    RSGroupInfo groupInfo = ADMIN.getRSGroup(groupName);
218    ADMIN.moveServersToRSGroup(groupInfo.getServers(), RSGroupInfo.DEFAULT_GROUP);
219    ADMIN.removeRSGroup(groupName);
220  }
221
222  protected final void deleteTableIfNecessary() throws IOException {
223    for (TableDescriptor desc : TEST_UTIL.getAdmin()
224      .listTableDescriptors(Pattern.compile(TABLE_PREFIX + ".*"))) {
225      TEST_UTIL.deleteTable(desc.getTableName());
226    }
227  }
228
229  protected final void deleteNamespaceIfNecessary() throws IOException {
230    for (NamespaceDescriptor desc : TEST_UTIL.getAdmin().listNamespaceDescriptors()) {
231      if (desc.getName().startsWith(TABLE_PREFIX)) {
232        ADMIN.deleteNamespace(desc.getName());
233      }
234    }
235  }
236
237  protected final void deleteGroups() throws IOException {
238    for (RSGroupInfo groupInfo : ADMIN.listRSGroups()) {
239      if (!groupInfo.getName().equals(RSGroupInfo.DEFAULT_GROUP)) {
240        removeGroup(groupInfo.getName());
241      }
242    }
243  }
244
245  protected Map<TableName, List<String>> getTableRegionMap() throws IOException {
246    Map<TableName, List<String>> map = Maps.newTreeMap();
247    Map<TableName, Map<ServerName, List<String>>> tableServerRegionMap = getTableServerRegionMap();
248    for (TableName tableName : tableServerRegionMap.keySet()) {
249      if (!map.containsKey(tableName)) {
250        map.put(tableName, new LinkedList<>());
251      }
252      for (List<String> subset : tableServerRegionMap.get(tableName).values()) {
253        map.get(tableName).addAll(subset);
254      }
255    }
256    return map;
257  }
258
259  protected Map<TableName, Map<ServerName, List<String>>> getTableServerRegionMap()
260    throws IOException {
261    Map<TableName, Map<ServerName, List<String>>> map = Maps.newTreeMap();
262    Admin admin = TEST_UTIL.getAdmin();
263    ClusterMetrics metrics =
264      admin.getClusterMetrics(EnumSet.of(ClusterMetrics.Option.SERVERS_NAME));
265    for (ServerName serverName : metrics.getServersName()) {
266      for (RegionInfo region : admin.getRegions(serverName)) {
267        TableName tableName = region.getTable();
268        map.computeIfAbsent(tableName, k -> new TreeMap<>())
269          .computeIfAbsent(serverName, k -> new ArrayList<>()).add(region.getRegionNameAsString());
270      }
271    }
272    return map;
273  }
274
275  // return the real number of region servers, excluding the master embedded region server in 2.0+
276  protected int getNumServers() throws IOException {
277    ClusterMetrics status = ADMIN.getClusterMetrics(EnumSet.of(Option.MASTER, Option.LIVE_SERVERS));
278    ServerName masterName = status.getMasterName();
279    int count = 0;
280    for (ServerName sn : status.getLiveServerMetrics().keySet()) {
281      if (!sn.equals(masterName)) {
282        count++;
283      }
284    }
285    return count;
286  }
287
288  protected final String getGroupName(String baseName) {
289    return GROUP_PREFIX + "_" + getNameWithoutIndex(baseName) + "_"
290      + ThreadLocalRandom.current().nextInt(Integer.MAX_VALUE);
291  }
292
293  /**
294   * The server name in group does not contain the start code, this method will find out the start
295   * code and construct the ServerName object.
296   */
297  protected final ServerName getServerName(Address addr) {
298    return TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().stream()
299      .map(t -> t.getRegionServer().getServerName()).filter(sn -> sn.getAddress().equals(addr))
300      .findFirst().get();
301  }
302
303  protected final void toggleQuotaCheckAndRestartMiniCluster(boolean enable) throws Exception {
304    TEST_UTIL.shutdownMiniCluster();
305    TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, enable);
306    TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE);
307    TEST_UTIL.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
308      NUM_SLAVES_BASE);
309    TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
310    initialize();
311  }
312
313  public static class CPMasterObserver implements MasterCoprocessor, MasterObserver {
314    boolean preBalanceRSGroupCalled = false;
315    boolean postBalanceRSGroupCalled = false;
316    boolean preMoveServersCalled = false;
317    boolean postMoveServersCalled = false;
318    boolean preMoveTablesCalled = false;
319    boolean postMoveTablesCalled = false;
320    boolean preAddRSGroupCalled = false;
321    boolean postAddRSGroupCalled = false;
322    boolean preRemoveRSGroupCalled = false;
323    boolean postRemoveRSGroupCalled = false;
324    boolean preRemoveServersCalled = false;
325    boolean postRemoveServersCalled = false;
326    boolean preMoveServersAndTables = false;
327    boolean postMoveServersAndTables = false;
328    boolean preGetRSGroupInfoCalled = false;
329    boolean postGetRSGroupInfoCalled = false;
330    boolean preGetRSGroupInfoOfTableCalled = false;
331    boolean postGetRSGroupInfoOfTableCalled = false;
332    boolean preListRSGroupsCalled = false;
333    boolean postListRSGroupsCalled = false;
334    boolean preGetRSGroupInfoOfServerCalled = false;
335    boolean postGetRSGroupInfoOfServerCalled = false;
336    boolean preSetRSGroupForTablesCalled = false;
337    boolean postSetRSGroupForTablesCalled = false;
338    boolean preListTablesInRSGroupCalled = false;
339    boolean postListTablesInRSGroupCalled = false;
340    boolean preGetConfiguredNamespacesAndTablesInRSGroupCalled = false;
341    boolean postGetConfiguredNamespacesAndTablesInRSGroupCalled = false;
342    boolean preRenameRSGroup = false;
343    boolean postRenameRSGroup = false;
344    boolean preUpdateRSGroupConfig = false;
345    boolean postUpdateRSGroupConfig = false;
346
347    public void resetFlags() {
348      preBalanceRSGroupCalled = false;
349      postBalanceRSGroupCalled = false;
350      preMoveServersCalled = false;
351      postMoveServersCalled = false;
352      preMoveTablesCalled = false;
353      postMoveTablesCalled = false;
354      preAddRSGroupCalled = false;
355      postAddRSGroupCalled = false;
356      preRemoveRSGroupCalled = false;
357      postRemoveRSGroupCalled = false;
358      preRemoveServersCalled = false;
359      postRemoveServersCalled = false;
360      preMoveServersAndTables = false;
361      postMoveServersAndTables = false;
362      preGetRSGroupInfoCalled = false;
363      postGetRSGroupInfoCalled = false;
364      preGetRSGroupInfoOfTableCalled = false;
365      postGetRSGroupInfoOfTableCalled = false;
366      preListRSGroupsCalled = false;
367      postListRSGroupsCalled = false;
368      preGetRSGroupInfoOfServerCalled = false;
369      postGetRSGroupInfoOfServerCalled = false;
370      preSetRSGroupForTablesCalled = false;
371      postSetRSGroupForTablesCalled = false;
372      preListTablesInRSGroupCalled = false;
373      postListTablesInRSGroupCalled = false;
374      preGetConfiguredNamespacesAndTablesInRSGroupCalled = false;
375      postGetConfiguredNamespacesAndTablesInRSGroupCalled = false;
376      preRenameRSGroup = false;
377      postRenameRSGroup = false;
378      preUpdateRSGroupConfig = false;
379      postUpdateRSGroupConfig = false;
380    }
381
382    @Override
383    public Optional<MasterObserver> getMasterObserver() {
384      return Optional.of(this);
385    }
386
387    @Override
388    public void preMoveServersAndTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
389      Set<Address> servers, Set<TableName> tables, String targetGroup) throws IOException {
390      preMoveServersAndTables = true;
391    }
392
393    @Override
394    public void postMoveServersAndTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
395      Set<Address> servers, Set<TableName> tables, String targetGroup) throws IOException {
396      postMoveServersAndTables = true;
397    }
398
399    @Override
400    public void preRemoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx,
401      Set<Address> servers) throws IOException {
402      preRemoveServersCalled = true;
403    }
404
405    @Override
406    public void postRemoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx,
407      Set<Address> servers) throws IOException {
408      postRemoveServersCalled = true;
409    }
410
411    @Override
412    public void preRemoveRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
413      String name) throws IOException {
414      preRemoveRSGroupCalled = true;
415    }
416
417    @Override
418    public void postRemoveRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
419      String name) throws IOException {
420      postRemoveRSGroupCalled = true;
421    }
422
423    @Override
424    public void preAddRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx, String name)
425      throws IOException {
426      preAddRSGroupCalled = true;
427    }
428
429    @Override
430    public void postAddRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx, String name)
431      throws IOException {
432      postAddRSGroupCalled = true;
433    }
434
435    @Override
436    public void preMoveTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
437      Set<TableName> tables, String targetGroup) throws IOException {
438      preMoveTablesCalled = true;
439    }
440
441    @Override
442    public void postMoveTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
443      Set<TableName> tables, String targetGroup) throws IOException {
444      postMoveTablesCalled = true;
445    }
446
447    @Override
448    public void preMoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx,
449      Set<Address> servers, String targetGroup) throws IOException {
450      preMoveServersCalled = true;
451    }
452
453    @Override
454    public void postMoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx,
455      Set<Address> servers, String targetGroup) throws IOException {
456      postMoveServersCalled = true;
457    }
458
459    @Override
460    public void preBalanceRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
461      String groupName, BalanceRequest request) throws IOException {
462      preBalanceRSGroupCalled = true;
463    }
464
465    @Override
466    public void postBalanceRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
467      String groupName, BalanceRequest request, BalanceResponse response) throws IOException {
468      postBalanceRSGroupCalled = true;
469    }
470
471    @Override
472    public void preGetRSGroupInfo(final ObserverContext<MasterCoprocessorEnvironment> ctx,
473      final String groupName) throws IOException {
474      preGetRSGroupInfoCalled = true;
475    }
476
477    @Override
478    public void postGetRSGroupInfo(final ObserverContext<MasterCoprocessorEnvironment> ctx,
479      final String groupName) throws IOException {
480      postGetRSGroupInfoCalled = true;
481    }
482
483    @Override
484    public void preGetRSGroupInfoOfTable(final ObserverContext<MasterCoprocessorEnvironment> ctx,
485      final TableName tableName) throws IOException {
486      preGetRSGroupInfoOfTableCalled = true;
487    }
488
489    @Override
490    public void postGetRSGroupInfoOfTable(final ObserverContext<MasterCoprocessorEnvironment> ctx,
491      final TableName tableName) throws IOException {
492      postGetRSGroupInfoOfTableCalled = true;
493    }
494
495    @Override
496    public void preListRSGroups(final ObserverContext<MasterCoprocessorEnvironment> ctx)
497      throws IOException {
498      preListRSGroupsCalled = true;
499    }
500
501    @Override
502    public void postListRSGroups(final ObserverContext<MasterCoprocessorEnvironment> ctx)
503      throws IOException {
504      postListRSGroupsCalled = true;
505    }
506
507    @Override
508    public void preGetRSGroupInfoOfServer(final ObserverContext<MasterCoprocessorEnvironment> ctx,
509      final Address server) throws IOException {
510      preGetRSGroupInfoOfServerCalled = true;
511    }
512
513    @Override
514    public void postGetRSGroupInfoOfServer(final ObserverContext<MasterCoprocessorEnvironment> ctx,
515      final Address server) throws IOException {
516      postGetRSGroupInfoOfServerCalled = true;
517    }
518
519    @Override
520    public void preListTablesInRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx,
521      String groupName) throws IOException {
522      preListTablesInRSGroupCalled = true;
523    }
524
525    @Override
526    public void postListTablesInRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx,
527      String groupName) throws IOException {
528      postListTablesInRSGroupCalled = true;
529    }
530
531    @Override
532    public void preGetConfiguredNamespacesAndTablesInRSGroup(
533      ObserverContext<MasterCoprocessorEnvironment> ctx, String groupName) throws IOException {
534      preGetConfiguredNamespacesAndTablesInRSGroupCalled = true;
535    }
536
537    @Override
538    public void postGetConfiguredNamespacesAndTablesInRSGroup(
539      ObserverContext<MasterCoprocessorEnvironment> ctx, String groupName) throws IOException {
540      postGetConfiguredNamespacesAndTablesInRSGroupCalled = true;
541    }
542
543    @Override
544    public void preRenameRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx, String oldName,
545      String newName) throws IOException {
546      preRenameRSGroup = true;
547    }
548
549    @Override
550    public void postRenameRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx, String oldName,
551      String newName) throws IOException {
552      postRenameRSGroup = true;
553    }
554
555    @Override
556    public void preUpdateRSGroupConfig(final ObserverContext<MasterCoprocessorEnvironment> ctx,
557      final String groupName, final Map<String, String> configuration) throws IOException {
558      preUpdateRSGroupConfig = true;
559    }
560
561    @Override
562    public void postUpdateRSGroupConfig(final ObserverContext<MasterCoprocessorEnvironment> ctx,
563      final String groupName, final Map<String, String> configuration) throws IOException {
564      postUpdateRSGroupConfig = true;
565    }
566  }
567}