001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.rsgroup;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022import static org.junit.Assert.fail;
023
024import java.io.IOException;
025import java.util.Iterator;
026import java.util.Optional;
027import java.util.Set;
028
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseTestingUtility;
031import org.apache.hadoop.hbase.HColumnDescriptor;
032import org.apache.hadoop.hbase.HConstants;
033import org.apache.hadoop.hbase.HTableDescriptor;
034import org.apache.hadoop.hbase.MiniHBaseCluster;
035import org.apache.hadoop.hbase.NamespaceDescriptor;
036import org.apache.hadoop.hbase.ServerName;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.Waiter;
039import org.apache.hadoop.hbase.Waiter.Predicate;
040import org.apache.hadoop.hbase.client.ClusterConnection;
041import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
042import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
043import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
044import org.apache.hadoop.hbase.coprocessor.MasterObserver;
045import org.apache.hadoop.hbase.coprocessor.ObserverContext;
046import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
047import org.apache.hadoop.hbase.master.ServerManager;
048import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
049import org.apache.hadoop.hbase.net.Address;
050import org.apache.hadoop.hbase.quotas.QuotaTableUtil;
051import org.apache.hadoop.hbase.quotas.QuotaUtil;
052import org.apache.hadoop.hbase.testclassification.MediumTests;
053import org.apache.hadoop.hbase.util.Bytes;
054import org.junit.After;
055import org.junit.AfterClass;
056import org.junit.Assert;
057import org.junit.Before;
058import org.junit.BeforeClass;
059import org.junit.ClassRule;
060import org.junit.Test;
061import org.junit.experimental.categories.Category;
062import org.slf4j.Logger;
063import org.slf4j.LoggerFactory;
064
065import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
066
067import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
068import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
069
070@Category({MediumTests.class})
071public class TestRSGroups extends TestRSGroupsBase {
072
073  @ClassRule
074  public static final HBaseClassTestRule CLASS_RULE =
075      HBaseClassTestRule.forClass(TestRSGroups.class);
076
077  protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroups.class);
078  private static boolean INIT = false;
079  private static RSGroupAdminEndpoint rsGroupAdminEndpoint;
080  private static CPMasterObserver observer;
081
082  @BeforeClass
083  public static void setUp() throws Exception {
084    TEST_UTIL = new HBaseTestingUtility();
085    TEST_UTIL.getConfiguration().setFloat(
086            "hbase.master.balancer.stochastic.tableSkewCost", 6000);
087    TEST_UTIL.getConfiguration().set(
088        HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
089        RSGroupBasedLoadBalancer.class.getName());
090    TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
091        RSGroupAdminEndpoint.class.getName() + "," + CPMasterObserver.class.getName());
092    TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE - 1);
093    TEST_UTIL.getConfiguration().setInt(
094        ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
095        NUM_SLAVES_BASE - 1);
096    TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
097
098    initialize();
099  }
100
101  private static void initialize() throws Exception {
102    admin = TEST_UTIL.getAdmin();
103    cluster = TEST_UTIL.getHBaseCluster();
104    master = ((MiniHBaseCluster)cluster).getMaster();
105
106    //wait for balancer to come online
107    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
108      @Override
109      public boolean evaluate() throws Exception {
110        return master.isInitialized() &&
111            ((RSGroupBasedLoadBalancer) master.getLoadBalancer()).isOnline();
112      }
113    });
114    admin.setBalancerRunning(false,true);
115    rsGroupAdmin = new VerifyingRSGroupAdminClient(
116        new RSGroupAdminClient(TEST_UTIL.getConnection()), TEST_UTIL.getConfiguration());
117    MasterCoprocessorHost host = master.getMasterCoprocessorHost();
118    observer = (CPMasterObserver) host.findCoprocessor(CPMasterObserver.class.getName());
119    rsGroupAdminEndpoint = (RSGroupAdminEndpoint)
120        host.findCoprocessor(RSGroupAdminEndpoint.class.getName());
121  }
122
123  @AfterClass
124  public static void tearDown() throws Exception {
125    TEST_UTIL.shutdownMiniCluster();
126  }
127
128  @Before
129  public void beforeMethod() throws Exception {
130    if (!INIT) {
131      INIT = true;
132      afterMethod();
133    }
134
135  }
136
137  @After
138  public void afterMethod() throws Exception {
139    deleteTableIfNecessary();
140    deleteNamespaceIfNecessary();
141    deleteGroups();
142
143    for(ServerName sn : admin.listDecommissionedRegionServers()){
144      admin.recommissionRegionServer(sn, null);
145    }
146    assertTrue(admin.listDecommissionedRegionServers().isEmpty());
147
148    int missing = NUM_SLAVES_BASE - getNumServers();
149    LOG.info("Restoring servers: "+missing);
150    for(int i=0; i<missing; i++) {
151      ((MiniHBaseCluster)cluster).startRegionServer();
152    }
153
154    rsGroupAdmin.addRSGroup("master");
155    ServerName masterServerName =
156        ((MiniHBaseCluster)cluster).getMaster().getServerName();
157
158    try {
159      rsGroupAdmin.moveServers(Sets.newHashSet(masterServerName.getAddress()), "master");
160    } catch (Exception ex) {
161      LOG.warn("Got this on setup, FYI", ex);
162    }
163    assertTrue(observer.preMoveServersCalled);
164    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
165      @Override
166      public boolean evaluate() throws Exception {
167        LOG.info("Waiting for cleanup to finish " + rsGroupAdmin.listRSGroups());
168        //Might be greater since moving servers back to default
169        //is after starting a server
170
171        return rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size()
172            == NUM_SLAVES_BASE;
173      }
174    });
175  }
176
177  @Test
178  public void testBasicStartUp() throws IOException {
179    RSGroupInfo defaultInfo = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP);
180    assertEquals(4, defaultInfo.getServers().size());
181    // Assignment of root and meta regions.
182    int count = master.getAssignmentManager().getRegionStates().getRegionAssignments().size();
183    //3 meta,namespace, group
184    assertEquals(3, count);
185  }
186
187  @Test
188  public void testNamespaceCreateAndAssign() throws Exception {
189    LOG.info("testNamespaceCreateAndAssign");
190    String nsName = tablePrefix+"_foo";
191    final TableName tableName = TableName.valueOf(nsName, tablePrefix + "_testCreateAndAssign");
192    RSGroupInfo appInfo = addGroup("appInfo", 1);
193    admin.createNamespace(NamespaceDescriptor.create(nsName)
194        .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, "appInfo").build());
195    final HTableDescriptor desc = new HTableDescriptor(tableName);
196    desc.addFamily(new HColumnDescriptor("f"));
197    admin.createTable(desc);
198    //wait for created table to be assigned
199    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
200      @Override
201      public boolean evaluate() throws Exception {
202        return getTableRegionMap().get(desc.getTableName()) != null;
203      }
204    });
205    ServerName targetServer =
206        ServerName.parseServerName(appInfo.getServers().iterator().next().toString());
207    AdminProtos.AdminService.BlockingInterface rs =
208      ((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
209    //verify it was assigned to the right group
210    Assert.assertEquals(1, ProtobufUtil.getOnlineRegions(rs).size());
211  }
212
213  @Test
214  public void testDefaultNamespaceCreateAndAssign() throws Exception {
215    LOG.info("testDefaultNamespaceCreateAndAssign");
216    String tableName = tablePrefix + "_testCreateAndAssign";
217    admin.modifyNamespace(NamespaceDescriptor.create("default")
218        .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, "default").build());
219    final HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
220    desc.addFamily(new HColumnDescriptor("f"));
221    admin.createTable(desc);
222    //wait for created table to be assigned
223    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
224      @Override
225      public boolean evaluate() throws Exception {
226        return getTableRegionMap().get(desc.getTableName()) != null;
227      }
228    });
229  }
230
231  @Test
232  public void testNamespaceConstraint() throws Exception {
233    String nsName = tablePrefix+"_foo";
234    String groupName = tablePrefix+"_foo";
235    LOG.info("testNamespaceConstraint");
236    rsGroupAdmin.addRSGroup(groupName);
237    assertTrue(observer.preAddRSGroupCalled);
238    assertTrue(observer.postAddRSGroupCalled);
239
240    admin.createNamespace(NamespaceDescriptor.create(nsName)
241        .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, groupName)
242        .build());
243    //test removing a referenced group
244    try {
245      rsGroupAdmin.removeRSGroup(groupName);
246      fail("Expected a constraint exception");
247    } catch (IOException ex) {
248    }
249    //test modify group
250    //changing with the same name is fine
251    admin.modifyNamespace(
252        NamespaceDescriptor.create(nsName)
253          .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, groupName)
254          .build());
255    String anotherGroup = tablePrefix+"_anotherGroup";
256    rsGroupAdmin.addRSGroup(anotherGroup);
257    //test add non-existent group
258    admin.deleteNamespace(nsName);
259    rsGroupAdmin.removeRSGroup(groupName);
260    assertTrue(observer.preRemoveRSGroupCalled);
261    assertTrue(observer.postRemoveRSGroupCalled);
262    try {
263      admin.createNamespace(NamespaceDescriptor.create(nsName)
264          .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, "foo")
265          .build());
266      fail("Expected a constraint exception");
267    } catch (IOException ex) {
268    }
269  }
270
271  @Test
272  public void testGroupInfoMultiAccessing() throws Exception {
273    RSGroupInfoManager manager = rsGroupAdminEndpoint.getGroupInfoManager();
274    RSGroupInfo defaultGroup = manager.getRSGroup("default");
275    // getRSGroup updates default group's server list
276    // this process must not affect other threads iterating the list
277    Iterator<Address> it = defaultGroup.getServers().iterator();
278    manager.getRSGroup("default");
279    it.next();
280  }
281
282  public static class CPMasterObserver implements MasterCoprocessor, MasterObserver {
283    boolean preBalanceRSGroupCalled = false;
284    boolean postBalanceRSGroupCalled = false;
285    boolean preMoveServersCalled = false;
286    boolean postMoveServersCalled = false;
287    boolean preMoveTablesCalled = false;
288    boolean postMoveTablesCalled = false;
289    boolean preAddRSGroupCalled = false;
290    boolean postAddRSGroupCalled = false;
291    boolean preRemoveRSGroupCalled = false;
292    boolean postRemoveRSGroupCalled = false;
293    boolean preRemoveServersCalled = false;
294    boolean postRemoveServersCalled = false;
295    boolean preMoveServersAndTables = false;
296    boolean postMoveServersAndTables = false;
297
298    @Override
299    public Optional<MasterObserver> getMasterObserver() {
300      return Optional.of(this);
301    }
302    @Override
303    public void preMoveServersAndTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
304        Set<Address> servers, Set<TableName> tables, String targetGroup) throws IOException {
305      preMoveServersAndTables = true;
306    }
307    @Override
308    public void postMoveServersAndTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
309        Set<Address> servers, Set<TableName> tables, String targetGroup) throws IOException {
310      postMoveServersAndTables = true;
311    }
312    @Override
313    public void preRemoveServers(
314        final ObserverContext<MasterCoprocessorEnvironment> ctx,
315        Set<Address> servers) throws IOException {
316      preRemoveServersCalled = true;
317    }
318    @Override
319    public void postRemoveServers(
320        final ObserverContext<MasterCoprocessorEnvironment> ctx,
321        Set<Address> servers) throws IOException {
322      postRemoveServersCalled = true;
323    }
324    @Override
325    public void preRemoveRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
326        String name) throws IOException {
327      preRemoveRSGroupCalled = true;
328    }
329    @Override
330    public void postRemoveRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
331        String name) throws IOException {
332      postRemoveRSGroupCalled = true;
333    }
334    @Override
335    public void preAddRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
336        String name) throws IOException {
337      preAddRSGroupCalled = true;
338    }
339    @Override
340    public void postAddRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
341        String name) throws IOException {
342      postAddRSGroupCalled = true;
343    }
344    @Override
345    public void preMoveTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
346        Set<TableName> tables, String targetGroup) throws IOException {
347      preMoveTablesCalled = true;
348    }
349    @Override
350    public void postMoveTables(final ObserverContext<MasterCoprocessorEnvironment> ctx,
351        Set<TableName> tables, String targetGroup) throws IOException {
352      postMoveTablesCalled = true;
353    }
354    @Override
355    public void preMoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx,
356        Set<Address> servers, String targetGroup) throws IOException {
357      preMoveServersCalled = true;
358    }
359
360    @Override
361    public void postMoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx,
362        Set<Address> servers, String targetGroup) throws IOException {
363      postMoveServersCalled = true;
364    }
365    @Override
366    public void preBalanceRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
367        String groupName) throws IOException {
368      preBalanceRSGroupCalled = true;
369    }
370    @Override
371    public void postBalanceRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
372        String groupName, boolean balancerRan) throws IOException {
373      postBalanceRSGroupCalled = true;
374    }
375  }
376  @Test
377  public void testMoveServersAndTables() throws Exception {
378    super.testMoveServersAndTables();
379    assertTrue(observer.preMoveServersAndTables);
380    assertTrue(observer.postMoveServersAndTables);
381  }
382  @Test
383  public void testTableMoveTruncateAndDrop() throws Exception {
384    super.testTableMoveTruncateAndDrop();
385    assertTrue(observer.preMoveTablesCalled);
386    assertTrue(observer.postMoveTablesCalled);
387  }
388
389  @Test
390  public void testRemoveServers() throws Exception {
391    super.testRemoveServers();
392    assertTrue(observer.preRemoveServersCalled);
393    assertTrue(observer.postRemoveServersCalled);
394  }
395
396  @Test
397  public void testMisplacedRegions() throws Exception {
398    final TableName tableName = TableName.valueOf(tablePrefix+"_testMisplacedRegions");
399    LOG.info("testMisplacedRegions");
400
401    final RSGroupInfo RSGroupInfo = addGroup("testMisplacedRegions", 1);
402
403    TEST_UTIL.createMultiRegionTable(tableName, new byte[]{'f'}, 15);
404    TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
405
406    rsGroupAdminEndpoint.getGroupInfoManager()
407        .moveTables(Sets.newHashSet(tableName), RSGroupInfo.getName());
408
409    admin.setBalancerRunning(true,true);
410    assertTrue(rsGroupAdmin.balanceRSGroup(RSGroupInfo.getName()));
411    admin.setBalancerRunning(false,true);
412    assertTrue(observer.preBalanceRSGroupCalled);
413    assertTrue(observer.postBalanceRSGroupCalled);
414
415    TEST_UTIL.waitFor(60000, new Predicate<Exception>() {
416      @Override
417      public boolean evaluate() throws Exception {
418        ServerName serverName =
419            ServerName.valueOf(RSGroupInfo.getServers().iterator().next().toString(), 1);
420        return admin.getConnection().getAdmin()
421            .getOnlineRegions(serverName).size() == 15;
422      }
423    });
424  }
425
426  @Test
427  public void testCloneSnapshot() throws Exception {
428    byte[] FAMILY = Bytes.toBytes("test");
429    String snapshotName = tableName.getNameAsString() + "_snap";
430    TableName clonedTableName = TableName.valueOf(tableName.getNameAsString() + "_clone");
431
432    // create base table
433    TEST_UTIL.createTable(tableName, FAMILY);
434
435    // create snapshot
436    admin.snapshot(snapshotName, tableName);
437
438    // clone
439    admin.cloneSnapshot(snapshotName, clonedTableName);
440  }
441
442  @Test
443  public void testRSGroupsWithHBaseQuota() throws Exception {
444    TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
445    restartHBaseCluster();
446    try {
447      TEST_UTIL.waitFor(90000, new Waiter.Predicate<Exception>() {
448        @Override
449        public boolean evaluate() throws Exception {
450          return admin.isTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME);
451        }
452      });
453    } finally {
454      TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, false);
455      restartHBaseCluster();
456    }
457  }
458
459  private void restartHBaseCluster() throws Exception {
460    LOG.info("\n\nShutting down cluster");
461    TEST_UTIL.shutdownMiniHBaseCluster();
462    LOG.info("\n\nSleeping a bit");
463    Thread.sleep(2000);
464    TEST_UTIL.restartHBaseCluster(NUM_SLAVES_BASE - 1);
465    initialize();
466  }
467}