001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.rsgroup;
019
020import static org.junit.Assert.assertTrue;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.EnumSet;
025import java.util.HashSet;
026import java.util.LinkedList;
027import java.util.List;
028import java.util.Map;
029import java.util.Optional;
030import java.util.Random;
031import java.util.Set;
032import java.util.TreeMap;
033import java.util.regex.Pattern;
034import org.apache.hadoop.hbase.ClusterMetrics;
035import org.apache.hadoop.hbase.ClusterMetrics.Option;
036import org.apache.hadoop.hbase.HBaseCluster;
037import org.apache.hadoop.hbase.HBaseTestingUtility;
038import org.apache.hadoop.hbase.HConstants;
039import org.apache.hadoop.hbase.MiniHBaseCluster;
040import org.apache.hadoop.hbase.NamespaceDescriptor;
041import org.apache.hadoop.hbase.ServerName;
042import org.apache.hadoop.hbase.TableName;
043import org.apache.hadoop.hbase.Waiter;
044import org.apache.hadoop.hbase.client.Admin;
045import org.apache.hadoop.hbase.client.RegionInfo;
046import org.apache.hadoop.hbase.client.TableDescriptor;
047import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
048import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
049import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
050import org.apache.hadoop.hbase.coprocessor.MasterObserver;
051import org.apache.hadoop.hbase.coprocessor.ObserverContext;
052import org.apache.hadoop.hbase.master.HMaster;
053import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
054import org.apache.hadoop.hbase.master.ServerManager;
055import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
056import org.apache.hadoop.hbase.net.Address;
057import org.junit.Rule;
058import org.junit.rules.TestName;
059import org.slf4j.Logger;
060import org.slf4j.LoggerFactory;
061
062import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
063import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
064
065public abstract class TestRSGroupsBase {
066  protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsBase.class);
067
068  //shared
069  protected final static String groupPrefix = "Group";
070  protected final static String tablePrefix = "Group";
071  protected final static Random rand = new Random();
072
073  //shared, cluster type specific
074  protected static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
075  protected static Admin admin;
076  protected static HBaseCluster cluster;
077  protected static RSGroupAdmin rsGroupAdmin;
078  protected static HMaster master;
079  protected boolean INIT = false;
080  protected static RSGroupAdminEndpoint rsGroupAdminEndpoint;
081  protected static CPMasterObserver observer;
082
083  public final static long WAIT_TIMEOUT = 60000;
084  public final static int NUM_SLAVES_BASE = 4; //number of slaves for the smallest cluster
085  public static int NUM_DEAD_SERVERS = 0;
086
087  // Per test variables
088  @Rule
089  public TestName name = new TestName();
090  protected TableName tableName;
091
092  public static void setUpTestBeforeClass() throws Exception {
093    TEST_UTIL.getConfiguration().setFloat(
094            "hbase.master.balancer.stochastic.tableSkewCost", 6000);
095    TEST_UTIL.getConfiguration().set(
096        HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
097        RSGroupBasedLoadBalancer.class.getName());
098    TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
099        RSGroupAdminEndpoint.class.getName() + "," + CPMasterObserver.class.getName());
100    TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE - 1);
101    TEST_UTIL.getConfiguration().setInt(
102        ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
103        NUM_SLAVES_BASE - 1);
104    TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
105    initialize();
106  }
107
108  protected static void initialize() throws Exception {
109    admin = TEST_UTIL.getAdmin();
110    cluster = TEST_UTIL.getHBaseCluster();
111    master = TEST_UTIL.getMiniHBaseCluster().getMaster();
112
113    //wait for balancer to come online
114    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
115      @Override
116      public boolean evaluate() throws Exception {
117        return master.isInitialized() &&
118            ((RSGroupBasedLoadBalancer) master.getLoadBalancer()).isOnline();
119      }
120    });
121    admin.balancerSwitch(false, true);
122    rsGroupAdmin = new VerifyingRSGroupAdminClient(
123        new RSGroupAdminClient(TEST_UTIL.getConnection()), TEST_UTIL.getConfiguration());
124    MasterCoprocessorHost host = master.getMasterCoprocessorHost();
125    observer = (CPMasterObserver) host.findCoprocessor(CPMasterObserver.class.getName());
126    rsGroupAdminEndpoint = (RSGroupAdminEndpoint)
127        host.findCoprocessor(RSGroupAdminEndpoint.class.getName());
128  }
129
130  public static void tearDownAfterClass() throws Exception {
131    TEST_UTIL.shutdownMiniCluster();
132  }
133
134  public void setUpBeforeMethod() throws Exception {
135    LOG.info(name.getMethodName());
136    tableName = TableName.valueOf(tablePrefix + "_" + name.getMethodName());
137    if (!INIT) {
138      INIT = true;
139      tearDownAfterMethod();
140    }
141    observer.resetFlags();
142  }
143
144  public void tearDownAfterMethod() throws Exception {
145    deleteTableIfNecessary();
146    deleteNamespaceIfNecessary();
147    deleteGroups();
148
149    for(ServerName sn : admin.listDecommissionedRegionServers()){
150      admin.recommissionRegionServer(sn, null);
151    }
152    assertTrue(admin.listDecommissionedRegionServers().isEmpty());
153
154    int missing = NUM_SLAVES_BASE - getNumServers();
155    LOG.info("Restoring servers: "+missing);
156    for(int i=0; i<missing; i++) {
157      ((MiniHBaseCluster)cluster).startRegionServer();
158    }
159
160    rsGroupAdmin.addRSGroup("master");
161    ServerName masterServerName =
162        ((MiniHBaseCluster)cluster).getMaster().getServerName();
163
164    try {
165      rsGroupAdmin.moveServers(Sets.newHashSet(masterServerName.getAddress()), "master");
166    } catch (Exception ex) {
167      LOG.warn("Got this on setup, FYI", ex);
168    }
169    assertTrue(observer.preMoveServersCalled);
170    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
171      @Override
172      public boolean evaluate() throws Exception {
173        LOG.info("Waiting for cleanup to finish " + rsGroupAdmin.listRSGroups());
174        //Might be greater since moving servers back to default
175        //is after starting a server
176
177        return rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size()
178            == NUM_SLAVES_BASE;
179      }
180    });
181  }
182
183  protected RSGroupInfo addGroup(String groupName, int serverCount)
184      throws IOException, InterruptedException {
185    RSGroupInfo defaultInfo = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP);
186    rsGroupAdmin.addRSGroup(groupName);
187    Set<Address> set = new HashSet<>();
188    for(Address server: defaultInfo.getServers()) {
189      if(set.size() == serverCount) {
190        break;
191      }
192      set.add(server);
193    }
194    rsGroupAdmin.moveServers(set, groupName);
195    RSGroupInfo result = rsGroupAdmin.getRSGroupInfo(groupName);
196    return result;
197  }
198
199  protected void removeGroup(String groupName) throws IOException {
200    RSGroupInfo groupInfo = rsGroupAdmin.getRSGroupInfo(groupName);
201    rsGroupAdmin.moveTables(groupInfo.getTables(), RSGroupInfo.DEFAULT_GROUP);
202    rsGroupAdmin.moveServers(groupInfo.getServers(), RSGroupInfo.DEFAULT_GROUP);
203    rsGroupAdmin.removeRSGroup(groupName);
204  }
205
206  protected void deleteTableIfNecessary() throws IOException {
207    for (TableDescriptor desc : TEST_UTIL.getAdmin()
208      .listTableDescriptors(Pattern.compile(tablePrefix + ".*"))) {
209      TEST_UTIL.deleteTable(desc.getTableName());
210    }
211  }
212
213  protected void deleteNamespaceIfNecessary() throws IOException {
214    for (NamespaceDescriptor desc : TEST_UTIL.getAdmin().listNamespaceDescriptors()) {
215      if(desc.getName().startsWith(tablePrefix)) {
216        admin.deleteNamespace(desc.getName());
217      }
218    }
219  }
220
221  protected void deleteGroups() throws IOException {
222    RSGroupAdmin groupAdmin = new RSGroupAdminClient(TEST_UTIL.getConnection());
223    for(RSGroupInfo group: groupAdmin.listRSGroups()) {
224      if(!group.getName().equals(RSGroupInfo.DEFAULT_GROUP)) {
225        groupAdmin.moveTables(group.getTables(), RSGroupInfo.DEFAULT_GROUP);
226        groupAdmin.moveServers(group.getServers(), RSGroupInfo.DEFAULT_GROUP);
227        groupAdmin.removeRSGroup(group.getName());
228      }
229    }
230  }
231
232  protected Map<TableName, List<String>> getTableRegionMap() throws IOException {
233    Map<TableName, List<String>> map = Maps.newTreeMap();
234    Map<TableName, Map<ServerName, List<String>>> tableServerRegionMap
235        = getTableServerRegionMap();
236    for(TableName tableName : tableServerRegionMap.keySet()) {
237      if(!map.containsKey(tableName)) {
238        map.put(tableName, new LinkedList<>());
239      }
240      for(List<String> subset: tableServerRegionMap.get(tableName).values()) {
241        map.get(tableName).addAll(subset);
242      }
243    }
244    return map;
245  }
246
247  protected Map<TableName, Map<ServerName, List<String>>> getTableServerRegionMap()
248    throws IOException {
249    Map<TableName, Map<ServerName, List<String>>> map = Maps.newTreeMap();
250    Admin admin = TEST_UTIL.getAdmin();
251    ClusterMetrics metrics =
252      admin.getClusterMetrics(EnumSet.of(ClusterMetrics.Option.SERVERS_NAME));
253    for (ServerName serverName : metrics.getServersName()) {
254      for (RegionInfo region : admin.getRegions(serverName)) {
255        TableName tableName = region.getTable();
256        map.computeIfAbsent(tableName, k -> new TreeMap<>())
257          .computeIfAbsent(serverName, k -> new ArrayList<>()).add(region.getRegionNameAsString());
258      }
259    }
260    return map;
261  }
262
263  // return the real number of region servers, excluding the master embedded region server in 2.0+
264  protected int getNumServers() throws IOException {
265    ClusterMetrics status =
266        admin.getClusterMetrics(EnumSet.of(Option.MASTER, Option.LIVE_SERVERS));
267    ServerName masterName = status.getMasterName();
268    int count = 0;
269    for (ServerName sn : status.getLiveServerMetrics().keySet()) {
270      if (!sn.equals(masterName)) {
271        count++;
272      }
273    }
274    return count;
275  }
276
277  protected String getGroupName(String baseName) {
278    return groupPrefix + "_" + baseName + "_" + rand.nextInt(Integer.MAX_VALUE);
279  }
280
281  /**
282   * The server name in group does not contain the start code, this method will find out the start
283   * code and construct the ServerName object.
284   */
285  protected ServerName getServerName(Address addr) {
286    return TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().stream()
287      .map(t -> t.getRegionServer().getServerName()).filter(sn -> sn.getAddress().equals(addr))
288      .findFirst().get();
289  }
290
291  public static class CPMasterObserver implements MasterCoprocessor, MasterObserver {
292    boolean preBalanceRSGroupCalled = false;
293    boolean postBalanceRSGroupCalled = false;
294    boolean preMoveServersCalled = false;
295    boolean postMoveServersCalled = false;
296    boolean preMoveTablesCalled = false;
297    boolean postMoveTablesCalled = false;
298    boolean preAddRSGroupCalled = false;
299    boolean postAddRSGroupCalled = false;
300    boolean preRemoveRSGroupCalled = false;
301    boolean postRemoveRSGroupCalled = false;
302    boolean preRemoveServersCalled = false;
303    boolean postRemoveServersCalled = false;
304    boolean preMoveServersAndTables = false;
305    boolean postMoveServersAndTables = false;
306    boolean preRenameRSGroupCalled = false;
307    boolean postRenameRSGroupCalled = false;
308
309    public void resetFlags() {
310      preBalanceRSGroupCalled = false;
311      postBalanceRSGroupCalled = false;
312      preMoveServersCalled = false;
313      postMoveServersCalled = false;
314      preMoveTablesCalled = false;
315      postMoveTablesCalled = false;
316      preAddRSGroupCalled = false;
317      postAddRSGroupCalled = false;
318      preRemoveRSGroupCalled = false;
319      postRemoveRSGroupCalled = false;
320      preRemoveServersCalled = false;
321      postRemoveServersCalled = false;
322      preMoveServersAndTables = false;
323      postMoveServersAndTables = false;
324      preRenameRSGroupCalled = false;
325      postRenameRSGroupCalled = false;
326    }
327
328    @Override
329    public Optional<MasterObserver> getMasterObserver() {
330      return Optional.of(this);
331    }
332
333    @Override
334    public void preMoveServersAndTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
335        Set<Address> servers, Set<TableName> tables, String targetGroup) throws IOException {
336      preMoveServersAndTables = true;
337    }
338
339    @Override
340    public void postMoveServersAndTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
341        Set<Address> servers, Set<TableName> tables, String targetGroup) throws IOException {
342      postMoveServersAndTables = true;
343    }
344
345    @Override
346    public void preRemoveServers(
347        final ObserverContext<MasterCoprocessorEnvironment> ctx,
348        Set<Address> servers) throws IOException {
349      preRemoveServersCalled = true;
350    }
351
352    @Override
353    public void postRemoveServers(
354        final ObserverContext<MasterCoprocessorEnvironment> ctx,
355        Set<Address> servers) throws IOException {
356      postRemoveServersCalled = true;
357    }
358
359    @Override
360    public void preRemoveRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
361        String name) throws IOException {
362      preRemoveRSGroupCalled = true;
363    }
364
365    @Override
366    public void postRemoveRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
367        String name) throws IOException {
368      postRemoveRSGroupCalled = true;
369    }
370
371    @Override
372    public void preAddRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
373        String name) throws IOException {
374      preAddRSGroupCalled = true;
375    }
376
377    @Override
378    public void postAddRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
379        String name) throws IOException {
380      postAddRSGroupCalled = true;
381    }
382
383    @Override
384    public void preMoveTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
385        Set<TableName> tables, String targetGroup) throws IOException {
386      preMoveTablesCalled = true;
387    }
388
389    @Override
390    public void postMoveTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
391        Set<TableName> tables, String targetGroup) throws IOException {
392      postMoveTablesCalled = true;
393    }
394
395    @Override
396    public void preMoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx,
397        Set<Address> servers, String targetGroup) throws IOException {
398      preMoveServersCalled = true;
399    }
400
401    @Override
402    public void postMoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx,
403        Set<Address> servers, String targetGroup) throws IOException {
404      postMoveServersCalled = true;
405    }
406
407    @Override
408    public void preBalanceRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
409        String groupName) throws IOException {
410      preBalanceRSGroupCalled = true;
411    }
412
413    @Override
414    public void postBalanceRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
415        String groupName, boolean balancerRan) throws IOException {
416      postBalanceRSGroupCalled = true;
417    }
418
419    @Override
420    public void preRenameRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx,
421        String oldName, String newName) throws IOException {
422      preRenameRSGroupCalled = true;
423    }
424
425    @Override
426    public void postRenameRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx,
427        String oldName, String newName) throws IOException {
428      postRenameRSGroupCalled = true;
429    }
430  }
431
432}