001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.rsgroup;
019
020import static org.junit.Assert.assertTrue;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.EnumSet;
025import java.util.HashSet;
026import java.util.LinkedList;
027import java.util.List;
028import java.util.Map;
029import java.util.Optional;
030import java.util.Set;
031import java.util.TreeMap;
032import java.util.concurrent.ThreadLocalRandom;
033import java.util.regex.Pattern;
034import org.apache.hadoop.conf.Configuration;
035import org.apache.hadoop.hbase.ClusterMetrics;
036import org.apache.hadoop.hbase.ClusterMetrics.Option;
037import org.apache.hadoop.hbase.HBaseClusterInterface;
038import org.apache.hadoop.hbase.HBaseTestingUtil;
039import org.apache.hadoop.hbase.NamespaceDescriptor;
040import org.apache.hadoop.hbase.ServerName;
041import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
042import org.apache.hadoop.hbase.TableName;
043import org.apache.hadoop.hbase.Waiter;
044import org.apache.hadoop.hbase.client.AbstractTestUpdateConfiguration;
045import org.apache.hadoop.hbase.client.Admin;
046import org.apache.hadoop.hbase.client.BalanceRequest;
047import org.apache.hadoop.hbase.client.BalanceResponse;
048import org.apache.hadoop.hbase.client.RegionInfo;
049import org.apache.hadoop.hbase.client.TableDescriptor;
050import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
051import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
052import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
053import org.apache.hadoop.hbase.coprocessor.MasterObserver;
054import org.apache.hadoop.hbase.coprocessor.ObserverContext;
055import org.apache.hadoop.hbase.master.HMaster;
056import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
057import org.apache.hadoop.hbase.master.ServerManager;
058import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
059import org.apache.hadoop.hbase.net.Address;
060import org.apache.hadoop.hbase.quotas.QuotaUtil;
061import org.junit.Rule;
062import org.junit.rules.TestName;
063import org.slf4j.Logger;
064import org.slf4j.LoggerFactory;
065
066import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
067import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
068
069public abstract class TestRSGroupsBase extends AbstractTestUpdateConfiguration {
070  protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsBase.class);
071
072  // shared
073  protected static final String GROUP_PREFIX = "Group";
074  protected static final String TABLE_PREFIX = "Group";
075
076  // shared, cluster type specific
077  protected static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
078  protected static Admin ADMIN;
079  protected static HBaseClusterInterface CLUSTER;
080  protected static HMaster MASTER;
081  protected boolean INIT = false;
082  protected static CPMasterObserver OBSERVER;
083
084  public final static long WAIT_TIMEOUT = 60000;
085  public final static int NUM_SLAVES_BASE = 4; // number of slaves for the smallest cluster
086  public static int NUM_DEAD_SERVERS = 0;
087
088  // Per test variables
089  @Rule
090  public TestName name = new TestName();
091  protected TableName tableName;
092
093  public static String getNameWithoutIndex(String name) {
094    return name.split("\\[")[0];
095  }
096
097  public static void setUpTestBeforeClass() throws Exception {
098    Configuration conf = TEST_UTIL.getConfiguration();
099    conf.setFloat("hbase.master.balancer.stochastic.tableSkewCost", 6000);
100    if (conf.get(RSGroupUtil.RS_GROUP_ENABLED) == null) {
101      RSGroupUtil.enableRSGroup(conf);
102    }
103    if (conf.get(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY) != null) {
104      conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
105        conf.get(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY) + "," +
106          CPMasterObserver.class.getName());
107    } else {
108      conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, CPMasterObserver.class.getName());
109    }
110
111    conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
112      NUM_SLAVES_BASE - 1);
113    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
114    conf.setInt("hbase.rpc.timeout", 100000);
115
116    TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE - 1);
117    initialize();
118  }
119
120  protected static void initialize() throws Exception {
121    ADMIN = new VerifyingRSGroupAdmin(TEST_UTIL.getConfiguration());
122    CLUSTER = TEST_UTIL.getHBaseCluster();
123    MASTER = TEST_UTIL.getMiniHBaseCluster().getMaster();
124
125    // wait for balancer to come online
126    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
127      @Override
128      public boolean evaluate() throws Exception {
129        return MASTER.isInitialized() &&
130          ((RSGroupBasedLoadBalancer) MASTER.getLoadBalancer()).isOnline();
131      }
132    });
133    ADMIN.balancerSwitch(false, true);
134    MasterCoprocessorHost host = MASTER.getMasterCoprocessorHost();
135    OBSERVER = (CPMasterObserver) host.findCoprocessor(CPMasterObserver.class.getName());
136  }
137
138  public static void tearDownAfterClass() throws Exception {
139    TEST_UTIL.shutdownMiniCluster();
140  }
141
142  public void setUpBeforeMethod() throws Exception {
143    LOG.info(name.getMethodName());
144    tableName = TableName.valueOf(TABLE_PREFIX + "_" + name.getMethodName().split("\\[")[0]);
145    if (!INIT) {
146      INIT = true;
147      tearDownAfterMethod();
148    }
149    OBSERVER.resetFlags();
150  }
151
152  public void tearDownAfterMethod() throws Exception {
153    deleteTableIfNecessary();
154    deleteNamespaceIfNecessary();
155    deleteGroups();
156
157    for (ServerName sn : ADMIN.listDecommissionedRegionServers()) {
158      ADMIN.recommissionRegionServer(sn, null);
159    }
160    assertTrue(ADMIN.listDecommissionedRegionServers().isEmpty());
161
162    int missing = NUM_SLAVES_BASE - getNumServers();
163    LOG.info("Restoring servers: " + missing);
164    for (int i = 0; i < missing; i++) {
165      ((SingleProcessHBaseCluster) CLUSTER).startRegionServer();
166    }
167    ADMIN.addRSGroup("master");
168    ServerName masterServerName = ((SingleProcessHBaseCluster) CLUSTER).getMaster().getServerName();
169    try {
170      ADMIN.moveServersToRSGroup(Sets.newHashSet(masterServerName.getAddress()), "master");
171    } catch (Exception ex) {
172      LOG.warn("Got this on setup, FYI", ex);
173    }
174    assertTrue(OBSERVER.preMoveServersCalled);
175    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
176      @Override
177      public boolean evaluate() throws Exception {
178        LOG.info("Waiting for cleanup to finish " + ADMIN.listRSGroups());
179        // Might be greater since moving servers back to default
180        // is after starting a server
181
182        return ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().size() == NUM_SLAVES_BASE;
183      }
184    });
185  }
186
187  protected final RSGroupInfo addGroup(String groupName, int serverCount)
188    throws IOException, InterruptedException {
189    RSGroupInfo defaultInfo = ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP);
190    ADMIN.addRSGroup(groupName);
191    Set<Address> set = new HashSet<>();
192    for (Address server : defaultInfo.getServers()) {
193      if (set.size() == serverCount) {
194        break;
195      }
196      set.add(server);
197    }
198    ADMIN.moveServersToRSGroup(set, groupName);
199    RSGroupInfo result = ADMIN.getRSGroup(groupName);
200    return result;
201  }
202
203  protected final void removeGroup(String groupName) throws IOException {
204    Set<TableName> tables = new HashSet<>();
205    for (TableDescriptor td : ADMIN.listTableDescriptors(true)) {
206      RSGroupInfo groupInfo = ADMIN.getRSGroup(td.getTableName());
207      if (groupInfo != null && groupInfo.getName().equals(groupName)) {
208        tables.add(td.getTableName());
209      }
210    }
211    ADMIN.setRSGroup(tables, RSGroupInfo.DEFAULT_GROUP);
212    for (NamespaceDescriptor nd : ADMIN.listNamespaceDescriptors()) {
213      if (groupName.equals(nd.getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP))) {
214        nd.removeConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP);
215        ADMIN.modifyNamespace(nd);
216      }
217    }
218    RSGroupInfo groupInfo = ADMIN.getRSGroup(groupName);
219    ADMIN.moveServersToRSGroup(groupInfo.getServers(), RSGroupInfo.DEFAULT_GROUP);
220    ADMIN.removeRSGroup(groupName);
221  }
222
223  protected final void deleteTableIfNecessary() throws IOException {
224    for (TableDescriptor desc : TEST_UTIL.getAdmin()
225      .listTableDescriptors(Pattern.compile(TABLE_PREFIX + ".*"))) {
226      TEST_UTIL.deleteTable(desc.getTableName());
227    }
228  }
229
230  protected final void deleteNamespaceIfNecessary() throws IOException {
231    for (NamespaceDescriptor desc : TEST_UTIL.getAdmin().listNamespaceDescriptors()) {
232      if (desc.getName().startsWith(TABLE_PREFIX)) {
233        ADMIN.deleteNamespace(desc.getName());
234      }
235    }
236  }
237
238  protected final void deleteGroups() throws IOException {
239    for (RSGroupInfo groupInfo : ADMIN.listRSGroups()) {
240      if (!groupInfo.getName().equals(RSGroupInfo.DEFAULT_GROUP)) {
241        removeGroup(groupInfo.getName());
242      }
243    }
244  }
245
246  protected Map<TableName, List<String>> getTableRegionMap() throws IOException {
247    Map<TableName, List<String>> map = Maps.newTreeMap();
248    Map<TableName, Map<ServerName, List<String>>> tableServerRegionMap = getTableServerRegionMap();
249    for (TableName tableName : tableServerRegionMap.keySet()) {
250      if (!map.containsKey(tableName)) {
251        map.put(tableName, new LinkedList<>());
252      }
253      for (List<String> subset : tableServerRegionMap.get(tableName).values()) {
254        map.get(tableName).addAll(subset);
255      }
256    }
257    return map;
258  }
259
260  protected Map<TableName, Map<ServerName, List<String>>> getTableServerRegionMap()
261    throws IOException {
262    Map<TableName, Map<ServerName, List<String>>> map = Maps.newTreeMap();
263    Admin admin = TEST_UTIL.getAdmin();
264    ClusterMetrics metrics =
265      admin.getClusterMetrics(EnumSet.of(ClusterMetrics.Option.SERVERS_NAME));
266    for (ServerName serverName : metrics.getServersName()) {
267      for (RegionInfo region : admin.getRegions(serverName)) {
268        TableName tableName = region.getTable();
269        map.computeIfAbsent(tableName, k -> new TreeMap<>())
270          .computeIfAbsent(serverName, k -> new ArrayList<>()).add(region.getRegionNameAsString());
271      }
272    }
273    return map;
274  }
275
276  // return the real number of region servers, excluding the master embedded region server in 2.0+
277  protected int getNumServers() throws IOException {
278    ClusterMetrics status = ADMIN.getClusterMetrics(EnumSet.of(Option.MASTER, Option.LIVE_SERVERS));
279    ServerName masterName = status.getMasterName();
280    int count = 0;
281    for (ServerName sn : status.getLiveServerMetrics().keySet()) {
282      if (!sn.equals(masterName)) {
283        count++;
284      }
285    }
286    return count;
287  }
288
289  protected final String getGroupName(String baseName) {
290    return GROUP_PREFIX + "_" + getNameWithoutIndex(baseName) + "_" +
291      ThreadLocalRandom.current().nextInt(Integer.MAX_VALUE);
292  }
293
294  /**
295   * The server name in group does not contain the start code, this method will find out the start
296   * code and construct the ServerName object.
297   */
298  protected final ServerName getServerName(Address addr) {
299    return TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().stream()
300      .map(t -> t.getRegionServer().getServerName()).filter(sn -> sn.getAddress().equals(addr))
301      .findFirst().get();
302  }
303
304  protected final void toggleQuotaCheckAndRestartMiniCluster(boolean enable) throws Exception {
305    TEST_UTIL.shutdownMiniCluster();
306    TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, enable);
307    TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE - 1);
308    TEST_UTIL.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
309      NUM_SLAVES_BASE - 1);
310    TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
311    initialize();
312  }
313
314  public static class CPMasterObserver implements MasterCoprocessor, MasterObserver {
315    boolean preBalanceRSGroupCalled = false;
316    boolean postBalanceRSGroupCalled = false;
317    boolean preMoveServersCalled = false;
318    boolean postMoveServersCalled = false;
319    boolean preMoveTablesCalled = false;
320    boolean postMoveTablesCalled = false;
321    boolean preAddRSGroupCalled = false;
322    boolean postAddRSGroupCalled = false;
323    boolean preRemoveRSGroupCalled = false;
324    boolean postRemoveRSGroupCalled = false;
325    boolean preRemoveServersCalled = false;
326    boolean postRemoveServersCalled = false;
327    boolean preMoveServersAndTables = false;
328    boolean postMoveServersAndTables = false;
329    boolean preGetRSGroupInfoCalled = false;
330    boolean postGetRSGroupInfoCalled = false;
331    boolean preGetRSGroupInfoOfTableCalled = false;
332    boolean postGetRSGroupInfoOfTableCalled = false;
333    boolean preListRSGroupsCalled = false;
334    boolean postListRSGroupsCalled = false;
335    boolean preGetRSGroupInfoOfServerCalled = false;
336    boolean postGetRSGroupInfoOfServerCalled = false;
337    boolean preSetRSGroupForTablesCalled = false;
338    boolean postSetRSGroupForTablesCalled = false;
339    boolean preListTablesInRSGroupCalled = false;
340    boolean postListTablesInRSGroupCalled = false;
341    boolean preGetConfiguredNamespacesAndTablesInRSGroupCalled = false;
342    boolean postGetConfiguredNamespacesAndTablesInRSGroupCalled = false;
343    boolean preRenameRSGroup = false;
344    boolean postRenameRSGroup = false;
345    boolean preUpdateRSGroupConfig = false;
346    boolean postUpdateRSGroupConfig = false;
347
348    public void resetFlags() {
349      preBalanceRSGroupCalled = false;
350      postBalanceRSGroupCalled = false;
351      preMoveServersCalled = false;
352      postMoveServersCalled = false;
353      preMoveTablesCalled = false;
354      postMoveTablesCalled = false;
355      preAddRSGroupCalled = false;
356      postAddRSGroupCalled = false;
357      preRemoveRSGroupCalled = false;
358      postRemoveRSGroupCalled = false;
359      preRemoveServersCalled = false;
360      postRemoveServersCalled = false;
361      preMoveServersAndTables = false;
362      postMoveServersAndTables = false;
363      preGetRSGroupInfoCalled = false;
364      postGetRSGroupInfoCalled = false;
365      preGetRSGroupInfoOfTableCalled = false;
366      postGetRSGroupInfoOfTableCalled = false;
367      preListRSGroupsCalled = false;
368      postListRSGroupsCalled = false;
369      preGetRSGroupInfoOfServerCalled = false;
370      postGetRSGroupInfoOfServerCalled = false;
371      preSetRSGroupForTablesCalled = false;
372      postSetRSGroupForTablesCalled = false;
373      preListTablesInRSGroupCalled = false;
374      postListTablesInRSGroupCalled = false;
375      preGetConfiguredNamespacesAndTablesInRSGroupCalled = false;
376      postGetConfiguredNamespacesAndTablesInRSGroupCalled = false;
377      preRenameRSGroup = false;
378      postRenameRSGroup = false;
379      preUpdateRSGroupConfig = false;
380      postUpdateRSGroupConfig = false;
381    }
382
383    @Override
384    public Optional<MasterObserver> getMasterObserver() {
385      return Optional.of(this);
386    }
387
388    @Override
389    public void preMoveServersAndTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
390      Set<Address> servers, Set<TableName> tables, String targetGroup) throws IOException {
391      preMoveServersAndTables = true;
392    }
393
394    @Override
395    public void postMoveServersAndTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
396      Set<Address> servers, Set<TableName> tables, String targetGroup) throws IOException {
397      postMoveServersAndTables = true;
398    }
399
400    @Override
401    public void preRemoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx,
402      Set<Address> servers) throws IOException {
403      preRemoveServersCalled = true;
404    }
405
406    @Override
407    public void postRemoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx,
408      Set<Address> servers) throws IOException {
409      postRemoveServersCalled = true;
410    }
411
412    @Override
413    public void preRemoveRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
414      String name) throws IOException {
415      preRemoveRSGroupCalled = true;
416    }
417
418    @Override
419    public void postRemoveRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
420      String name) throws IOException {
421      postRemoveRSGroupCalled = true;
422    }
423
424    @Override
425    public void preAddRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx, String name)
426      throws IOException {
427      preAddRSGroupCalled = true;
428    }
429
430    @Override
431    public void postAddRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx, String name)
432      throws IOException {
433      postAddRSGroupCalled = true;
434    }
435
436    @Override
437    public void preMoveTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
438      Set<TableName> tables, String targetGroup) throws IOException {
439      preMoveTablesCalled = true;
440    }
441
442    @Override
443    public void postMoveTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
444      Set<TableName> tables, String targetGroup) throws IOException {
445      postMoveTablesCalled = true;
446    }
447
448    @Override
449    public void preMoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx,
450      Set<Address> servers, String targetGroup) throws IOException {
451      preMoveServersCalled = true;
452    }
453
454    @Override
455    public void postMoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx,
456      Set<Address> servers, String targetGroup) throws IOException {
457      postMoveServersCalled = true;
458    }
459
460    @Override
461    public void preBalanceRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
462        String groupName, BalanceRequest request) throws IOException {
463      preBalanceRSGroupCalled = true;
464    }
465
466    @Override
467    public void postBalanceRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
468        String groupName, BalanceRequest request, BalanceResponse response) throws IOException {
469      postBalanceRSGroupCalled = true;
470    }
471
472    @Override
473    public void preGetRSGroupInfo(final ObserverContext<MasterCoprocessorEnvironment> ctx,
474      final String groupName) throws IOException {
475      preGetRSGroupInfoCalled = true;
476    }
477
478    @Override
479    public void postGetRSGroupInfo(final ObserverContext<MasterCoprocessorEnvironment> ctx,
480      final String groupName) throws IOException {
481      postGetRSGroupInfoCalled = true;
482    }
483
484    @Override
485    public void preGetRSGroupInfoOfTable(final ObserverContext<MasterCoprocessorEnvironment> ctx,
486      final TableName tableName) throws IOException {
487      preGetRSGroupInfoOfTableCalled = true;
488    }
489
490    @Override
491    public void postGetRSGroupInfoOfTable(final ObserverContext<MasterCoprocessorEnvironment> ctx,
492      final TableName tableName) throws IOException {
493      postGetRSGroupInfoOfTableCalled = true;
494    }
495
496    @Override
497    public void preListRSGroups(final ObserverContext<MasterCoprocessorEnvironment> ctx)
498      throws IOException {
499      preListRSGroupsCalled = true;
500    }
501
502    @Override
503    public void postListRSGroups(final ObserverContext<MasterCoprocessorEnvironment> ctx)
504      throws IOException {
505      postListRSGroupsCalled = true;
506    }
507
508    @Override
509    public void preGetRSGroupInfoOfServer(final ObserverContext<MasterCoprocessorEnvironment> ctx,
510      final Address server) throws IOException {
511      preGetRSGroupInfoOfServerCalled = true;
512    }
513
514    @Override
515    public void postGetRSGroupInfoOfServer(final ObserverContext<MasterCoprocessorEnvironment> ctx,
516      final Address server) throws IOException {
517      postGetRSGroupInfoOfServerCalled = true;
518    }
519
520    @Override
521    public void preListTablesInRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx,
522      String groupName) throws IOException {
523      preListTablesInRSGroupCalled = true;
524    }
525
526    @Override
527    public void postListTablesInRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx,
528      String groupName) throws IOException {
529      postListTablesInRSGroupCalled = true;
530    }
531
532    @Override
533    public void preGetConfiguredNamespacesAndTablesInRSGroup(
534      ObserverContext<MasterCoprocessorEnvironment> ctx, String groupName) throws IOException {
535      preGetConfiguredNamespacesAndTablesInRSGroupCalled = true;
536    }
537
538    @Override
539    public void postGetConfiguredNamespacesAndTablesInRSGroup(
540      ObserverContext<MasterCoprocessorEnvironment> ctx, String groupName) throws IOException {
541      postGetConfiguredNamespacesAndTablesInRSGroupCalled = true;
542    }
543
544    @Override
545    public void preRenameRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx, String oldName,
546      String newName) throws IOException {
547      preRenameRSGroup = true;
548    }
549
550    @Override
551    public void postRenameRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx, String oldName,
552      String newName) throws IOException {
553      postRenameRSGroup = true;
554    }
555
556    @Override
557    public void preUpdateRSGroupConfig(final ObserverContext<MasterCoprocessorEnvironment> ctx,
558      final String groupName, final Map<String, String> configuration) throws IOException {
559      preUpdateRSGroupConfig = true;
560    }
561
562    @Override
563    public void postUpdateRSGroupConfig(final ObserverContext<MasterCoprocessorEnvironment> ctx,
564      final String groupName, final Map<String, String> configuration) throws IOException {
565      postUpdateRSGroupConfig = true;
566    }
567  }
568}