Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.rsgroup;
019
020import static org.junit.Assert.assertTrue;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.EnumSet;
025import java.util.HashSet;
026import java.util.LinkedList;
027import java.util.List;
028import java.util.Map;
029import java.util.Optional;
030import java.util.Random;
031import java.util.Set;
032import java.util.TreeMap;
033import java.util.regex.Pattern;
034import org.apache.hadoop.hbase.ClusterMetrics;
035import org.apache.hadoop.hbase.ClusterMetrics.Option;
036import org.apache.hadoop.hbase.HBaseCluster;
037import org.apache.hadoop.hbase.HBaseTestingUtility;
038import org.apache.hadoop.hbase.HConstants;
039import org.apache.hadoop.hbase.MiniHBaseCluster;
040import org.apache.hadoop.hbase.NamespaceDescriptor;
041import org.apache.hadoop.hbase.ServerName;
042import org.apache.hadoop.hbase.TableName;
043import org.apache.hadoop.hbase.Waiter;
044import org.apache.hadoop.hbase.client.Admin;
045import org.apache.hadoop.hbase.client.RegionInfo;
046import org.apache.hadoop.hbase.client.TableDescriptor;
047import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
048import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
049import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
050import org.apache.hadoop.hbase.coprocessor.MasterObserver;
051import org.apache.hadoop.hbase.coprocessor.ObserverContext;
052import org.apache.hadoop.hbase.master.HMaster;
053import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
054import org.apache.hadoop.hbase.master.ServerManager;
055import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
056import org.apache.hadoop.hbase.net.Address;
057import org.junit.Rule;
058import org.junit.rules.TestName;
059import org.slf4j.Logger;
060import org.slf4j.LoggerFactory;
061
062import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
063import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
064
065public abstract class TestRSGroupsBase {
066  protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsBase.class);
067
068  //shared
069  protected final static String groupPrefix = "Group";
070  protected final static String tablePrefix = "Group";
071  protected final static Random rand = new Random();
072
073  //shared, cluster type specific
074  protected static HBaseTestingUtility TEST_UTIL;
075  protected static Admin admin;
076  protected static HBaseCluster cluster;
077  protected static RSGroupAdmin rsGroupAdmin;
078  protected static HMaster master;
079  protected boolean INIT = false;
080  protected static RSGroupAdminEndpoint rsGroupAdminEndpoint;
081  protected static CPMasterObserver observer;
082
083  public final static long WAIT_TIMEOUT = 60000;
084  public final static int NUM_SLAVES_BASE = 4; //number of slaves for the smallest cluster
085  public static int NUM_DEAD_SERVERS = 0;
086
087  // Per test variables
088  @Rule
089  public TestName name = new TestName();
090  protected TableName tableName;
091
092  public static void setUpTestBeforeClass() throws Exception {
093    TEST_UTIL = new HBaseTestingUtility();
094    TEST_UTIL.getConfiguration().setFloat(
095            "hbase.master.balancer.stochastic.tableSkewCost", 6000);
096    TEST_UTIL.getConfiguration().set(
097        HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
098        RSGroupBasedLoadBalancer.class.getName());
099    TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
100        RSGroupAdminEndpoint.class.getName() + "," + CPMasterObserver.class.getName());
101    TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE - 1);
102    TEST_UTIL.getConfiguration().setInt(
103        ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
104        NUM_SLAVES_BASE - 1);
105    TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
106    initialize();
107  }
108
109  protected static void initialize() throws Exception {
110    admin = TEST_UTIL.getAdmin();
111    cluster = TEST_UTIL.getHBaseCluster();
112    master = TEST_UTIL.getMiniHBaseCluster().getMaster();
113
114    //wait for balancer to come online
115    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
116      @Override
117      public boolean evaluate() throws Exception {
118        return master.isInitialized() &&
119            ((RSGroupBasedLoadBalancer) master.getLoadBalancer()).isOnline();
120      }
121    });
122    admin.balancerSwitch(false, true);
123    rsGroupAdmin = new VerifyingRSGroupAdminClient(
124        new RSGroupAdminClient(TEST_UTIL.getConnection()), TEST_UTIL.getConfiguration());
125    MasterCoprocessorHost host = master.getMasterCoprocessorHost();
126    observer = (CPMasterObserver) host.findCoprocessor(CPMasterObserver.class.getName());
127    rsGroupAdminEndpoint = (RSGroupAdminEndpoint)
128        host.findCoprocessor(RSGroupAdminEndpoint.class.getName());
129  }
130
131  public static void tearDownAfterClass() throws Exception {
132    TEST_UTIL.shutdownMiniCluster();
133  }
134
135  public void setUpBeforeMethod() throws Exception {
136    LOG.info(name.getMethodName());
137    tableName = TableName.valueOf(tablePrefix + "_" + name.getMethodName());
138    if (!INIT) {
139      INIT = true;
140      tearDownAfterMethod();
141    }
142    observer.resetFlags();
143  }
144
145  public void tearDownAfterMethod() throws Exception {
146    deleteTableIfNecessary();
147    deleteNamespaceIfNecessary();
148    deleteGroups();
149
150    for(ServerName sn : admin.listDecommissionedRegionServers()){
151      admin.recommissionRegionServer(sn, null);
152    }
153    assertTrue(admin.listDecommissionedRegionServers().isEmpty());
154
155    int missing = NUM_SLAVES_BASE - getNumServers();
156    LOG.info("Restoring servers: "+missing);
157    for(int i=0; i<missing; i++) {
158      ((MiniHBaseCluster)cluster).startRegionServer();
159    }
160
161    rsGroupAdmin.addRSGroup("master");
162    ServerName masterServerName =
163        ((MiniHBaseCluster)cluster).getMaster().getServerName();
164
165    try {
166      rsGroupAdmin.moveServers(Sets.newHashSet(masterServerName.getAddress()), "master");
167    } catch (Exception ex) {
168      LOG.warn("Got this on setup, FYI", ex);
169    }
170    assertTrue(observer.preMoveServersCalled);
171    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
172      @Override
173      public boolean evaluate() throws Exception {
174        LOG.info("Waiting for cleanup to finish " + rsGroupAdmin.listRSGroups());
175        //Might be greater since moving servers back to default
176        //is after starting a server
177
178        return rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size()
179            == NUM_SLAVES_BASE;
180      }
181    });
182  }
183
184  protected RSGroupInfo addGroup(String groupName, int serverCount)
185      throws IOException, InterruptedException {
186    RSGroupInfo defaultInfo = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP);
187    rsGroupAdmin.addRSGroup(groupName);
188    Set<Address> set = new HashSet<>();
189    for(Address server: defaultInfo.getServers()) {
190      if(set.size() == serverCount) {
191        break;
192      }
193      set.add(server);
194    }
195    rsGroupAdmin.moveServers(set, groupName);
196    RSGroupInfo result = rsGroupAdmin.getRSGroupInfo(groupName);
197    return result;
198  }
199
200  protected void removeGroup(String groupName) throws IOException {
201    RSGroupInfo groupInfo = rsGroupAdmin.getRSGroupInfo(groupName);
202    rsGroupAdmin.moveTables(groupInfo.getTables(), RSGroupInfo.DEFAULT_GROUP);
203    rsGroupAdmin.moveServers(groupInfo.getServers(), RSGroupInfo.DEFAULT_GROUP);
204    rsGroupAdmin.removeRSGroup(groupName);
205  }
206
207  protected void deleteTableIfNecessary() throws IOException {
208    for (TableDescriptor desc : TEST_UTIL.getAdmin()
209      .listTableDescriptors(Pattern.compile(tablePrefix + ".*"))) {
210      TEST_UTIL.deleteTable(desc.getTableName());
211    }
212  }
213
214  protected void deleteNamespaceIfNecessary() throws IOException {
215    for (NamespaceDescriptor desc : TEST_UTIL.getAdmin().listNamespaceDescriptors()) {
216      if(desc.getName().startsWith(tablePrefix)) {
217        admin.deleteNamespace(desc.getName());
218      }
219    }
220  }
221
222  protected void deleteGroups() throws IOException {
223    RSGroupAdmin groupAdmin = new RSGroupAdminClient(TEST_UTIL.getConnection());
224    for(RSGroupInfo group: groupAdmin.listRSGroups()) {
225      if(!group.getName().equals(RSGroupInfo.DEFAULT_GROUP)) {
226        groupAdmin.moveTables(group.getTables(), RSGroupInfo.DEFAULT_GROUP);
227        groupAdmin.moveServers(group.getServers(), RSGroupInfo.DEFAULT_GROUP);
228        groupAdmin.removeRSGroup(group.getName());
229      }
230    }
231  }
232
233  protected Map<TableName, List<String>> getTableRegionMap() throws IOException {
234    Map<TableName, List<String>> map = Maps.newTreeMap();
235    Map<TableName, Map<ServerName, List<String>>> tableServerRegionMap
236        = getTableServerRegionMap();
237    for(TableName tableName : tableServerRegionMap.keySet()) {
238      if(!map.containsKey(tableName)) {
239        map.put(tableName, new LinkedList<>());
240      }
241      for(List<String> subset: tableServerRegionMap.get(tableName).values()) {
242        map.get(tableName).addAll(subset);
243      }
244    }
245    return map;
246  }
247
248  protected Map<TableName, Map<ServerName, List<String>>> getTableServerRegionMap()
249    throws IOException {
250    Map<TableName, Map<ServerName, List<String>>> map = Maps.newTreeMap();
251    Admin admin = TEST_UTIL.getAdmin();
252    ClusterMetrics metrics =
253      admin.getClusterMetrics(EnumSet.of(ClusterMetrics.Option.SERVERS_NAME));
254    for (ServerName serverName : metrics.getServersName()) {
255      for (RegionInfo region : admin.getRegions(serverName)) {
256        TableName tableName = region.getTable();
257        map.computeIfAbsent(tableName, k -> new TreeMap<>())
258          .computeIfAbsent(serverName, k -> new ArrayList<>()).add(region.getRegionNameAsString());
259      }
260    }
261    return map;
262  }
263
264  // return the real number of region servers, excluding the master embedded region server in 2.0+
265  protected int getNumServers() throws IOException {
266    ClusterMetrics status =
267        admin.getClusterMetrics(EnumSet.of(Option.MASTER, Option.LIVE_SERVERS));
268    ServerName masterName = status.getMasterName();
269    int count = 0;
270    for (ServerName sn : status.getLiveServerMetrics().keySet()) {
271      if (!sn.equals(masterName)) {
272        count++;
273      }
274    }
275    return count;
276  }
277
278  protected String getGroupName(String baseName) {
279    return groupPrefix + "_" + baseName + "_" + rand.nextInt(Integer.MAX_VALUE);
280  }
281
282  /**
283   * The server name in group does not contain the start code, this method will find out the start
284   * code and construct the ServerName object.
285   */
286  protected ServerName getServerName(Address addr) {
287    return TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().stream()
288      .map(t -> t.getRegionServer().getServerName()).filter(sn -> sn.getAddress().equals(addr))
289      .findFirst().get();
290  }
291
292  public static class CPMasterObserver implements MasterCoprocessor, MasterObserver {
293    boolean preBalanceRSGroupCalled = false;
294    boolean postBalanceRSGroupCalled = false;
295    boolean preMoveServersCalled = false;
296    boolean postMoveServersCalled = false;
297    boolean preMoveTablesCalled = false;
298    boolean postMoveTablesCalled = false;
299    boolean preAddRSGroupCalled = false;
300    boolean postAddRSGroupCalled = false;
301    boolean preRemoveRSGroupCalled = false;
302    boolean postRemoveRSGroupCalled = false;
303    boolean preRemoveServersCalled = false;
304    boolean postRemoveServersCalled = false;
305    boolean preMoveServersAndTables = false;
306    boolean postMoveServersAndTables = false;
307
308    public void resetFlags() {
309      preBalanceRSGroupCalled = false;
310      postBalanceRSGroupCalled = false;
311      preMoveServersCalled = false;
312      postMoveServersCalled = false;
313      preMoveTablesCalled = false;
314      postMoveTablesCalled = false;
315      preAddRSGroupCalled = false;
316      postAddRSGroupCalled = false;
317      preRemoveRSGroupCalled = false;
318      postRemoveRSGroupCalled = false;
319      preRemoveServersCalled = false;
320      postRemoveServersCalled = false;
321      preMoveServersAndTables = false;
322      postMoveServersAndTables = false;
323    }
324
325    @Override
326    public Optional<MasterObserver> getMasterObserver() {
327      return Optional.of(this);
328    }
329
330    @Override
331    public void preMoveServersAndTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
332        Set<Address> servers, Set<TableName> tables, String targetGroup) throws IOException {
333      preMoveServersAndTables = true;
334    }
335
336    @Override
337    public void postMoveServersAndTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
338        Set<Address> servers, Set<TableName> tables, String targetGroup) throws IOException {
339      postMoveServersAndTables = true;
340    }
341
342    @Override
343    public void preRemoveServers(
344        final ObserverContext<MasterCoprocessorEnvironment> ctx,
345        Set<Address> servers) throws IOException {
346      preRemoveServersCalled = true;
347    }
348
349    @Override
350    public void postRemoveServers(
351        final ObserverContext<MasterCoprocessorEnvironment> ctx,
352        Set<Address> servers) throws IOException {
353      postRemoveServersCalled = true;
354    }
355
356    @Override
357    public void preRemoveRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
358        String name) throws IOException {
359      preRemoveRSGroupCalled = true;
360    }
361
362    @Override
363    public void postRemoveRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
364        String name) throws IOException {
365      postRemoveRSGroupCalled = true;
366    }
367
368    @Override
369    public void preAddRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
370        String name) throws IOException {
371      preAddRSGroupCalled = true;
372    }
373
374    @Override
375    public void postAddRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
376        String name) throws IOException {
377      postAddRSGroupCalled = true;
378    }
379
380    @Override
381    public void preMoveTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
382        Set<TableName> tables, String targetGroup) throws IOException {
383      preMoveTablesCalled = true;
384    }
385
386    @Override
387    public void postMoveTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
388        Set<TableName> tables, String targetGroup) throws IOException {
389      postMoveTablesCalled = true;
390    }
391
392    @Override
393    public void preMoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx,
394        Set<Address> servers, String targetGroup) throws IOException {
395      preMoveServersCalled = true;
396    }
397
398    @Override
399    public void postMoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx,
400        Set<Address> servers, String targetGroup) throws IOException {
401      postMoveServersCalled = true;
402    }
403
404    @Override
405    public void preBalanceRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
406        String groupName) throws IOException {
407      preBalanceRSGroupCalled = true;
408    }
409
410    @Override
411    public void postBalanceRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
412        String groupName, boolean balancerRan) throws IOException {
413      postBalanceRSGroupCalled = true;
414    }
415  }
416
417}