001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.rsgroup;
019
020import static org.junit.Assert.assertTrue;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.EnumSet;
025import java.util.HashSet;
026import java.util.LinkedList;
027import java.util.List;
028import java.util.Map;
029import java.util.Optional;
030import java.util.Random;
031import java.util.Set;
032import java.util.TreeMap;
033import java.util.regex.Pattern;
034import org.apache.hadoop.hbase.ClusterMetrics;
035import org.apache.hadoop.hbase.ClusterMetrics.Option;
036import org.apache.hadoop.hbase.HBaseCluster;
037import org.apache.hadoop.hbase.HBaseTestingUtility;
038import org.apache.hadoop.hbase.HConstants;
039import org.apache.hadoop.hbase.MiniHBaseCluster;
040import org.apache.hadoop.hbase.NamespaceDescriptor;
041import org.apache.hadoop.hbase.ServerName;
042import org.apache.hadoop.hbase.TableName;
043import org.apache.hadoop.hbase.Waiter;
044import org.apache.hadoop.hbase.client.Admin;
045import org.apache.hadoop.hbase.client.RegionInfo;
046import org.apache.hadoop.hbase.client.TableDescriptor;
047import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
048import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
049import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
050import org.apache.hadoop.hbase.coprocessor.MasterObserver;
051import org.apache.hadoop.hbase.coprocessor.ObserverContext;
052import org.apache.hadoop.hbase.master.HMaster;
053import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
054import org.apache.hadoop.hbase.master.ServerManager;
055import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
056import org.apache.hadoop.hbase.net.Address;
057import org.junit.Rule;
058import org.junit.rules.TestName;
059import org.slf4j.Logger;
060import org.slf4j.LoggerFactory;
061
062import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
063import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
064
065public abstract class TestRSGroupsBase {
066  protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsBase.class);
067
068  //shared
069  protected final static String groupPrefix = "Group";
070  protected final static String tablePrefix = "Group";
071  protected final static Random rand = new Random();
072
073  //shared, cluster type specific
074  protected static HBaseTestingUtility TEST_UTIL;
075  protected static Admin admin;
076  protected static HBaseCluster cluster;
077  protected static RSGroupAdmin rsGroupAdmin;
078  protected static HMaster master;
079  protected boolean INIT = false;
080  protected static RSGroupAdminEndpoint rsGroupAdminEndpoint;
081  protected static CPMasterObserver observer;
082
083  public final static long WAIT_TIMEOUT = 60000;
084  public final static int NUM_SLAVES_BASE = 4; //number of slaves for the smallest cluster
085  public static int NUM_DEAD_SERVERS = 0;
086
087  // Per test variables
088  @Rule
089  public TestName name = new TestName();
090  protected TableName tableName;
091
092  public static void setUpTestBeforeClass() throws Exception {
093    TEST_UTIL = new HBaseTestingUtility();
094    TEST_UTIL.getConfiguration().setFloat(
095            "hbase.master.balancer.stochastic.tableSkewCost", 6000);
096    TEST_UTIL.getConfiguration().set(
097        HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
098        RSGroupBasedLoadBalancer.class.getName());
099    TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
100        RSGroupAdminEndpoint.class.getName() + "," + CPMasterObserver.class.getName());
101    TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE - 1);
102    TEST_UTIL.getConfiguration().setInt(
103        ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
104        NUM_SLAVES_BASE - 1);
105    TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
106    initialize();
107  }
108
109  protected static void initialize() throws Exception {
110    admin = TEST_UTIL.getAdmin();
111    cluster = TEST_UTIL.getHBaseCluster();
112    master = TEST_UTIL.getMiniHBaseCluster().getMaster();
113
114    //wait for balancer to come online
115    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
116      @Override
117      public boolean evaluate() throws Exception {
118        return master.isInitialized() &&
119            ((RSGroupBasedLoadBalancer) master.getLoadBalancer()).isOnline();
120      }
121    });
122    admin.balancerSwitch(false, true);
123    rsGroupAdmin = new VerifyingRSGroupAdminClient(
124        new RSGroupAdminClient(TEST_UTIL.getConnection()), TEST_UTIL.getConfiguration());
125    MasterCoprocessorHost host = master.getMasterCoprocessorHost();
126    observer = (CPMasterObserver) host.findCoprocessor(CPMasterObserver.class.getName());
127    rsGroupAdminEndpoint = (RSGroupAdminEndpoint)
128        host.findCoprocessor(RSGroupAdminEndpoint.class.getName());
129  }
130
131  public static void tearDownAfterClass() throws Exception {
132    TEST_UTIL.shutdownMiniCluster();
133  }
134
135  public void setUpBeforeMethod() throws Exception {
136    LOG.info(name.getMethodName());
137    tableName = TableName.valueOf(tablePrefix + "_" + name.getMethodName());
138    if (!INIT) {
139      INIT = true;
140      tearDownAfterMethod();
141    }
142    observer.resetFlags();
143  }
144
145  public void tearDownAfterMethod() throws Exception {
146    deleteTableIfNecessary();
147    deleteNamespaceIfNecessary();
148    deleteGroups();
149
150    for(ServerName sn : admin.listDecommissionedRegionServers()){
151      admin.recommissionRegionServer(sn, null);
152    }
153    assertTrue(admin.listDecommissionedRegionServers().isEmpty());
154
155    int missing = NUM_SLAVES_BASE - getNumServers();
156    LOG.info("Restoring servers: "+missing);
157    for(int i=0; i<missing; i++) {
158      ((MiniHBaseCluster)cluster).startRegionServer();
159    }
160
161    rsGroupAdmin.addRSGroup("master");
162    ServerName masterServerName =
163        ((MiniHBaseCluster)cluster).getMaster().getServerName();
164
165    try {
166      rsGroupAdmin.moveServers(Sets.newHashSet(masterServerName.getAddress()), "master");
167    } catch (Exception ex) {
168      LOG.warn("Got this on setup, FYI", ex);
169    }
170    assertTrue(observer.preMoveServersCalled);
171    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
172      @Override
173      public boolean evaluate() throws Exception {
174        LOG.info("Waiting for cleanup to finish " + rsGroupAdmin.listRSGroups());
175        //Might be greater since moving servers back to default
176        //is after starting a server
177
178        return rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size()
179            == NUM_SLAVES_BASE;
180      }
181    });
182  }
183
184  protected RSGroupInfo addGroup(String groupName, int serverCount)
185      throws IOException, InterruptedException {
186    RSGroupInfo defaultInfo = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP);
187    rsGroupAdmin.addRSGroup(groupName);
188    Set<Address> set = new HashSet<>();
189    for(Address server: defaultInfo.getServers()) {
190      if(set.size() == serverCount) {
191        break;
192      }
193      set.add(server);
194    }
195    rsGroupAdmin.moveServers(set, groupName);
196    RSGroupInfo result = rsGroupAdmin.getRSGroupInfo(groupName);
197    return result;
198  }
199
200  protected void removeGroup(String groupName) throws IOException {
201    RSGroupInfo groupInfo = rsGroupAdmin.getRSGroupInfo(groupName);
202    rsGroupAdmin.moveTables(groupInfo.getTables(), RSGroupInfo.DEFAULT_GROUP);
203    rsGroupAdmin.moveServers(groupInfo.getServers(), RSGroupInfo.DEFAULT_GROUP);
204    rsGroupAdmin.removeRSGroup(groupName);
205  }
206
207  protected void deleteTableIfNecessary() throws IOException {
208    for (TableDescriptor desc : TEST_UTIL.getAdmin()
209      .listTableDescriptors(Pattern.compile(tablePrefix + ".*"))) {
210      TEST_UTIL.deleteTable(desc.getTableName());
211    }
212  }
213
214  protected void deleteNamespaceIfNecessary() throws IOException {
215    for (NamespaceDescriptor desc : TEST_UTIL.getAdmin().listNamespaceDescriptors()) {
216      if(desc.getName().startsWith(tablePrefix)) {
217        admin.deleteNamespace(desc.getName());
218      }
219    }
220  }
221
222  protected void deleteGroups() throws IOException {
223    RSGroupAdmin groupAdmin = new RSGroupAdminClient(TEST_UTIL.getConnection());
224    for(RSGroupInfo group: groupAdmin.listRSGroups()) {
225      if(!group.getName().equals(RSGroupInfo.DEFAULT_GROUP)) {
226        groupAdmin.moveTables(group.getTables(), RSGroupInfo.DEFAULT_GROUP);
227        groupAdmin.moveServers(group.getServers(), RSGroupInfo.DEFAULT_GROUP);
228        groupAdmin.removeRSGroup(group.getName());
229      }
230    }
231  }
232
233  protected Map<TableName, List<String>> getTableRegionMap() throws IOException {
234    Map<TableName, List<String>> map = Maps.newTreeMap();
235    Map<TableName, Map<ServerName, List<String>>> tableServerRegionMap
236        = getTableServerRegionMap();
237    for(TableName tableName : tableServerRegionMap.keySet()) {
238      if(!map.containsKey(tableName)) {
239        map.put(tableName, new LinkedList<>());
240      }
241      for(List<String> subset: tableServerRegionMap.get(tableName).values()) {
242        map.get(tableName).addAll(subset);
243      }
244    }
245    return map;
246  }
247
248  protected Map<TableName, Map<ServerName, List<String>>> getTableServerRegionMap()
249    throws IOException {
250    Map<TableName, Map<ServerName, List<String>>> map = Maps.newTreeMap();
251    Admin admin = TEST_UTIL.getAdmin();
252    ClusterMetrics metrics =
253      admin.getClusterMetrics(EnumSet.of(ClusterMetrics.Option.SERVERS_NAME));
254    for (ServerName serverName : metrics.getServersName()) {
255      for (RegionInfo region : admin.getRegions(serverName)) {
256        TableName tableName = region.getTable();
257        map.computeIfAbsent(tableName, k -> new TreeMap<>())
258          .computeIfAbsent(serverName, k -> new ArrayList<>()).add(region.getRegionNameAsString());
259      }
260    }
261    return map;
262  }
263
264  // return the real number of region servers, excluding the master embedded region server in 2.0+
265  protected int getNumServers() throws IOException {
266    ClusterMetrics status =
267        admin.getClusterMetrics(EnumSet.of(Option.MASTER, Option.LIVE_SERVERS));
268    ServerName masterName = status.getMasterName();
269    int count = 0;
270    for (ServerName sn : status.getLiveServerMetrics().keySet()) {
271      if (!sn.equals(masterName)) {
272        count++;
273      }
274    }
275    return count;
276  }
277
278  protected String getGroupName(String baseName) {
279    return groupPrefix + "_" + baseName + "_" + rand.nextInt(Integer.MAX_VALUE);
280  }
281
282  /**
283   * The server name in group does not contain the start code, this method will find out the start
284   * code and construct the ServerName object.
285   */
286  protected ServerName getServerName(Address addr) {
287    return TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().stream()
288      .map(t -> t.getRegionServer().getServerName()).filter(sn -> sn.getAddress().equals(addr))
289      .findFirst().get();
290  }
291
292  public static class CPMasterObserver implements MasterCoprocessor, MasterObserver {
293    boolean preBalanceRSGroupCalled = false;
294    boolean postBalanceRSGroupCalled = false;
295    boolean preMoveServersCalled = false;
296    boolean postMoveServersCalled = false;
297    boolean preMoveTablesCalled = false;
298    boolean postMoveTablesCalled = false;
299    boolean preAddRSGroupCalled = false;
300    boolean postAddRSGroupCalled = false;
301    boolean preRemoveRSGroupCalled = false;
302    boolean postRemoveRSGroupCalled = false;
303    boolean preRemoveServersCalled = false;
304    boolean postRemoveServersCalled = false;
305    boolean preMoveServersAndTables = false;
306    boolean postMoveServersAndTables = false;
307    boolean preRenameRSGroupCalled = false;
308    boolean postRenameRSGroupCalled = false;
309
310    public void resetFlags() {
311      preBalanceRSGroupCalled = false;
312      postBalanceRSGroupCalled = false;
313      preMoveServersCalled = false;
314      postMoveServersCalled = false;
315      preMoveTablesCalled = false;
316      postMoveTablesCalled = false;
317      preAddRSGroupCalled = false;
318      postAddRSGroupCalled = false;
319      preRemoveRSGroupCalled = false;
320      postRemoveRSGroupCalled = false;
321      preRemoveServersCalled = false;
322      postRemoveServersCalled = false;
323      preMoveServersAndTables = false;
324      postMoveServersAndTables = false;
325      preRenameRSGroupCalled = false;
326      postRenameRSGroupCalled = false;
327    }
328
329    @Override
330    public Optional<MasterObserver> getMasterObserver() {
331      return Optional.of(this);
332    }
333
334    @Override
335    public void preMoveServersAndTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
336        Set<Address> servers, Set<TableName> tables, String targetGroup) throws IOException {
337      preMoveServersAndTables = true;
338    }
339
340    @Override
341    public void postMoveServersAndTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
342        Set<Address> servers, Set<TableName> tables, String targetGroup) throws IOException {
343      postMoveServersAndTables = true;
344    }
345
346    @Override
347    public void preRemoveServers(
348        final ObserverContext<MasterCoprocessorEnvironment> ctx,
349        Set<Address> servers) throws IOException {
350      preRemoveServersCalled = true;
351    }
352
353    @Override
354    public void postRemoveServers(
355        final ObserverContext<MasterCoprocessorEnvironment> ctx,
356        Set<Address> servers) throws IOException {
357      postRemoveServersCalled = true;
358    }
359
360    @Override
361    public void preRemoveRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
362        String name) throws IOException {
363      preRemoveRSGroupCalled = true;
364    }
365
366    @Override
367    public void postRemoveRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
368        String name) throws IOException {
369      postRemoveRSGroupCalled = true;
370    }
371
372    @Override
373    public void preAddRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
374        String name) throws IOException {
375      preAddRSGroupCalled = true;
376    }
377
378    @Override
379    public void postAddRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
380        String name) throws IOException {
381      postAddRSGroupCalled = true;
382    }
383
384    @Override
385    public void preMoveTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
386        Set<TableName> tables, String targetGroup) throws IOException {
387      preMoveTablesCalled = true;
388    }
389
390    @Override
391    public void postMoveTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
392        Set<TableName> tables, String targetGroup) throws IOException {
393      postMoveTablesCalled = true;
394    }
395
396    @Override
397    public void preMoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx,
398        Set<Address> servers, String targetGroup) throws IOException {
399      preMoveServersCalled = true;
400    }
401
402    @Override
403    public void postMoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx,
404        Set<Address> servers, String targetGroup) throws IOException {
405      postMoveServersCalled = true;
406    }
407
408    @Override
409    public void preBalanceRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
410        String groupName) throws IOException {
411      preBalanceRSGroupCalled = true;
412    }
413
414    @Override
415    public void postBalanceRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
416        String groupName, boolean balancerRan) throws IOException {
417      postBalanceRSGroupCalled = true;
418    }
419
420    @Override
421    public void preRenameRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx,
422        String oldName, String newName) throws IOException {
423      preRenameRSGroupCalled = true;
424    }
425
426    @Override
427    public void postRenameRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx,
428        String oldName, String newName) throws IOException {
429      postRenameRSGroupCalled = true;
430    }
431  }
432
433}