001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertThrows;
023import static org.junit.Assert.assertTrue;
024import static org.junit.Assert.fail;
025
026import java.io.IOException;
027import java.util.ArrayList;
028import java.util.Collection;
029import java.util.Collections;
030import java.util.EnumSet;
031import java.util.HashMap;
032import java.util.HashSet;
033import java.util.List;
034import java.util.concurrent.ThreadLocalRandom;
035import java.util.concurrent.atomic.AtomicInteger;
036import java.util.stream.Collectors;
037import org.apache.hadoop.hbase.ClusterMetrics.Option;
038import org.apache.hadoop.hbase.HBaseClassTestRule;
039import org.apache.hadoop.hbase.HConstants;
040import org.apache.hadoop.hbase.ServerName;
041import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
042import org.apache.hadoop.hbase.TableExistsException;
043import org.apache.hadoop.hbase.TableName;
044import org.apache.hadoop.hbase.TableNotDisabledException;
045import org.apache.hadoop.hbase.TableNotEnabledException;
046import org.apache.hadoop.hbase.TableNotFoundException;
047import org.apache.hadoop.hbase.UnknownRegionException;
048import org.apache.hadoop.hbase.Waiter.Predicate;
049import org.apache.hadoop.hbase.constraint.ConstraintException;
050import org.apache.hadoop.hbase.master.HMaster;
051import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
052import org.apache.hadoop.hbase.regionserver.HRegion;
053import org.apache.hadoop.hbase.regionserver.HRegionServer;
054import org.apache.hadoop.hbase.regionserver.HStore;
055import org.apache.hadoop.hbase.testclassification.ClientTests;
056import org.apache.hadoop.hbase.testclassification.LargeTests;
057import org.apache.hadoop.hbase.util.Bytes;
058import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
059import org.apache.hadoop.hbase.util.FutureUtils;
060import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
061import org.junit.Assert;
062import org.junit.ClassRule;
063import org.junit.Test;
064import org.junit.experimental.categories.Category;
065import org.slf4j.Logger;
066import org.slf4j.LoggerFactory;
067
068import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
069import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
070
071/**
072 * Class to test HBaseAdmin. Spins up the minicluster once at test start and then takes it down
073 * afterward. Add any testing of HBaseAdmin functionality here.
074 */
075@Category({ LargeTests.class, ClientTests.class })
076public class TestAdmin2 extends TestAdminBase {
077
078  @ClassRule
079  public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestAdmin2.class);
080
081  private static final Logger LOG = LoggerFactory.getLogger(TestAdmin2.class);
082
083  @Test
084  public void testCreateBadTables() throws IOException {
085    String msg = null;
086    try {
087      ADMIN.createTable(TableDescriptorBuilder.newBuilder(TableName.META_TABLE_NAME).build());
088    } catch (TableExistsException e) {
089      msg = e.toString();
090    }
091    assertTrue("Unexcepted exception message " + msg,
092      msg != null && msg.startsWith(TableExistsException.class.getName())
093        && msg.contains(TableName.META_TABLE_NAME.getNameAsString()));
094
095    // Now try and do concurrent creation with a bunch of threads.
096    TableDescriptor tableDescriptor =
097      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
098        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build();
099    int count = 10;
100    Thread[] threads = new Thread[count];
101    final AtomicInteger successes = new AtomicInteger(0);
102    final AtomicInteger failures = new AtomicInteger(0);
103    final Admin localAdmin = ADMIN;
104    for (int i = 0; i < count; i++) {
105      threads[i] = new Thread(Integer.toString(i)) {
106        @Override
107        public void run() {
108          try {
109            localAdmin.createTable(tableDescriptor);
110            successes.incrementAndGet();
111          } catch (TableExistsException e) {
112            failures.incrementAndGet();
113          } catch (IOException e) {
114            throw new RuntimeException("Failed threaded create" + getName(), e);
115          }
116        }
117      };
118    }
119    for (int i = 0; i < count; i++) {
120      threads[i].start();
121    }
122    for (int i = 0; i < count; i++) {
123      while (threads[i].isAlive()) {
124        try {
125          Thread.sleep(100);
126        } catch (InterruptedException e) {
127          // continue
128        }
129      }
130    }
131    // All threads are now dead. Count up how many tables were created and
132    // how many failed w/ appropriate exception.
133    assertEquals(1, successes.get());
134    assertEquals(count - 1, failures.get());
135  }
136
137  /**
138   * Test for hadoop-1581 'HBASE: Unopenable tablename bug'.
139   */
140  @Test
141  public void testTableNameClash() throws Exception {
142    final String name = this.name.getMethodName();
143    TableDescriptor tableDescriptor1 =
144      TableDescriptorBuilder.newBuilder(TableName.valueOf(name + "SOMEUPPERCASE"))
145        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build();
146    TableDescriptor tableDescriptor2 = TableDescriptorBuilder.newBuilder(TableName.valueOf(name))
147      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build();
148    ADMIN.createTable(tableDescriptor1);
149    ADMIN.createTable(tableDescriptor2);
150    // Before fix, below would fail throwing a NoServerForRegionException.
151    TEST_UTIL.getConnection().getTable(tableDescriptor2.getTableName()).close();
152  }
153
154  /***
155   * HMaster.createTable used to be kind of synchronous call Thus creating of table with lots of
156   * regions can cause RPC timeout After the fix to make createTable truly async, RPC timeout
157   * shouldn't be an issue anymore
158   */
159  @Test
160  public void testCreateTableRPCTimeOut() throws Exception {
161    final String name = this.name.getMethodName();
162    int oldTimeout = TEST_UTIL.getConfiguration().getInt(HConstants.HBASE_RPC_TIMEOUT_KEY,
163      HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
164    TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 1500);
165    try {
166      int expectedRegions = 100;
167      // Use 80 bit numbers to make sure we aren't limited
168      byte[] startKey = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
169      byte[] endKey = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 };
170      Admin hbaseadmin = TEST_UTIL.getAdmin();
171      TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(TableName.valueOf(name))
172        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build();
173      hbaseadmin.createTable(tableDescriptor, startKey, endKey, expectedRegions);
174    } finally {
175      TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, oldTimeout);
176    }
177  }
178
179  /**
180   * Test read only tables
181   */
182  @Test
183  public void testReadOnlyTable() throws Exception {
184    final TableName name = TableName.valueOf(this.name.getMethodName());
185    Table table = TEST_UTIL.createTable(name, HConstants.CATALOG_FAMILY);
186    byte[] value = Bytes.toBytes("somedata");
187    // This used to use an empty row... That must have been a bug
188    Put put = new Put(value);
189    put.addColumn(HConstants.CATALOG_FAMILY, HConstants.CATALOG_FAMILY, value);
190    table.put(put);
191    table.close();
192  }
193
194  /**
195   * Test that user table names can contain '-' and '.' so long as they do not start with same.
196   * HBASE-771
197   */
198  @Test
199  public void testTableNames() throws IOException {
200    byte[][] illegalNames = new byte[][] { Bytes.toBytes("-bad"), Bytes.toBytes(".bad") };
201    for (byte[] illegalName : illegalNames) {
202      assertThrows(
203        "Did not detect '" + Bytes.toString(illegalName) + "' as an illegal user table name",
204        IllegalArgumentException.class, () -> TableName.valueOf(illegalName));
205    }
206    byte[] legalName = Bytes.toBytes("g-oo.d");
207    try {
208      TableName.valueOf(legalName);
209    } catch (IllegalArgumentException e) {
210      fail("Legal user table name: '" + Bytes.toString(legalName)
211        + "' caused IllegalArgumentException: " + e.getMessage());
212    }
213  }
214
215  /**
216   * For HADOOP-2579
217   */
218  @Test(expected = TableExistsException.class)
219  public void testTableExistsExceptionWithATable() throws IOException {
220    final TableName name = TableName.valueOf(this.name.getMethodName());
221    TEST_UTIL.createTable(name, HConstants.CATALOG_FAMILY).close();
222    TEST_UTIL.createTable(name, HConstants.CATALOG_FAMILY);
223  }
224
225  /**
226   * Can't disable a table if the table isn't in enabled state
227   */
228  @Test(expected = TableNotEnabledException.class)
229  public void testTableNotEnabledExceptionWithATable() throws IOException {
230    final TableName name = TableName.valueOf(this.name.getMethodName());
231    TEST_UTIL.createTable(name, HConstants.CATALOG_FAMILY).close();
232    ADMIN.disableTable(name);
233    ADMIN.disableTable(name);
234  }
235
236  /**
237   * Can't enable a table if the table isn't in disabled state
238   */
239  @Test(expected = TableNotDisabledException.class)
240  public void testTableNotDisabledExceptionWithATable() throws IOException {
241    final TableName name = TableName.valueOf(this.name.getMethodName());
242    try (Table t = TEST_UTIL.createTable(name, HConstants.CATALOG_FAMILY)) {
243      ADMIN.enableTable(name);
244    }
245  }
246
247  /**
248   * For HADOOP-2579
249   */
250  @Test(expected = TableNotFoundException.class)
251  public void testTableNotFoundExceptionWithoutAnyTables() throws IOException {
252    TableName tableName = TableName.valueOf("testTableNotFoundExceptionWithoutAnyTables");
253    try (Table ht = TEST_UTIL.getConnection().getTable(tableName)) {
254      ht.get(new Get(Bytes.toBytes("e")));
255    }
256  }
257
258  @Test
259  public void testShouldUnassignTheRegion() throws Exception {
260    final TableName tableName = TableName.valueOf(name.getMethodName());
261    createTableWithDefaultConf(tableName);
262
263    RegionInfo info = null;
264    HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(tableName);
265    List<RegionInfo> onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices());
266    for (RegionInfo regionInfo : onlineRegions) {
267      if (!regionInfo.getTable().isSystemTable()) {
268        info = regionInfo;
269        ADMIN.unassign(regionInfo.getRegionName(), true);
270      }
271    }
272    boolean isInList = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()).contains(info);
273    long timeout = EnvironmentEdgeManager.currentTime() + 10000;
274    while ((EnvironmentEdgeManager.currentTime() < timeout) && (isInList)) {
275      Thread.sleep(100);
276      isInList = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()).contains(info);
277    }
278
279    assertFalse("The region should not be present in online regions list.", isInList);
280  }
281
282  @Test
283  public void testCloseRegionIfInvalidRegionNameIsPassed() throws Exception {
284    final String name = this.name.getMethodName();
285    byte[] tableName = Bytes.toBytes(name);
286    createTableWithDefaultConf(tableName);
287
288    RegionInfo info = null;
289    HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TableName.valueOf(tableName));
290    List<RegionInfo> onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices());
291    for (RegionInfo regionInfo : onlineRegions) {
292      if (!regionInfo.isMetaRegion()) {
293        if (regionInfo.getRegionNameAsString().contains(name)) {
294          info = regionInfo;
295          assertThrows(UnknownRegionException.class, () -> ADMIN.unassign(
296            Bytes.toBytes("test,,1358563771069.acc1ad1b7962564fc3a43e5907e8db33."), true));
297        }
298      }
299    }
300    onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices());
301    assertTrue("The region should be present in online regions list.",
302      onlineRegions.contains(info));
303  }
304
305  @Test
306  public void testCloseRegionThatFetchesTheHRIFromMeta() throws Exception {
307    final TableName tableName = TableName.valueOf(name.getMethodName());
308    createTableWithDefaultConf(tableName);
309
310    RegionInfo info = null;
311    HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(tableName);
312    List<RegionInfo> onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices());
313    for (RegionInfo regionInfo : onlineRegions) {
314      if (!regionInfo.isMetaRegion()) {
315        if (regionInfo.getRegionNameAsString().contains("TestHBACloseRegion2")) {
316          info = regionInfo;
317          ADMIN.unassign(regionInfo.getRegionName(), true);
318        }
319      }
320    }
321
322    boolean isInList = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()).contains(info);
323    long timeout = EnvironmentEdgeManager.currentTime() + 10000;
324    while ((EnvironmentEdgeManager.currentTime() < timeout) && (isInList)) {
325      Thread.sleep(100);
326      isInList = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()).contains(info);
327    }
328
329    assertFalse("The region should not be present in online regions list.", isInList);
330  }
331
332  private Admin createTable(TableName tableName) throws IOException {
333    Admin admin = TEST_UTIL.getAdmin();
334
335    TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(tableName);
336    ColumnFamilyDescriptor columnFamilyDescriptor =
337      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("value")).build();
338
339    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
340    admin.createTable(tableDescriptorBuilder.build());
341    return admin;
342  }
343
344  private void createTableWithDefaultConf(byte[] TABLENAME) throws IOException {
345    createTableWithDefaultConf(TableName.valueOf(TABLENAME));
346  }
347
348  private void createTableWithDefaultConf(TableName TABLENAME) throws IOException {
349    TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(TABLENAME);
350    ColumnFamilyDescriptor columnFamilyDescriptor =
351      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("value")).build();
352    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
353
354    ADMIN.createTable(tableDescriptorBuilder.build());
355  }
356
357  /**
358   * For HBASE-2556
359   */
360  @Test
361  public void testGetTableRegions() throws IOException {
362    final TableName tableName = TableName.valueOf(name.getMethodName());
363
364    int expectedRegions = 10;
365
366    // Use 80 bit numbers to make sure we aren't limited
367    byte[] startKey = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
368    byte[] endKey = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 };
369
370    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
371      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build();
372    ADMIN.createTable(tableDescriptor, startKey, endKey, expectedRegions);
373
374    List<RegionInfo> RegionInfos = ADMIN.getRegions(tableName);
375
376    assertEquals(
377      "Tried to create " + expectedRegions + " regions " + "but only found " + RegionInfos.size(),
378      expectedRegions, RegionInfos.size());
379  }
380
381  @Test
382  public void testMoveToPreviouslyAssignedRS() throws IOException, InterruptedException {
383    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
384    HMaster master = cluster.getMaster();
385    final TableName tableName = TableName.valueOf(name.getMethodName());
386    Admin localAdmin = createTable(tableName);
387    List<RegionInfo> tableRegions = localAdmin.getRegions(tableName);
388    RegionInfo hri = tableRegions.get(0);
389    AssignmentManager am = master.getAssignmentManager();
390    ServerName server = am.getRegionStates().getRegionServerOfRegion(hri);
391    localAdmin.move(hri.getEncodedNameAsBytes(), server);
392    assertEquals("Current region server and region server before move should be same.", server,
393      am.getRegionStates().getRegionServerOfRegion(hri));
394  }
395
396  @Test
397  public void testWALRollWriting() throws Exception {
398    setUpforLogRolling();
399    String className = this.getClass().getName();
400    StringBuilder v = new StringBuilder(className);
401    while (v.length() < 1000) {
402      v.append(className);
403    }
404    byte[] value = Bytes.toBytes(v.toString());
405    HRegionServer regionServer = startAndWriteData(TableName.valueOf(name.getMethodName()), value);
406    LOG.info("after writing there are "
407      + AbstractFSWALProvider.getNumRolledLogFiles(regionServer.getWAL(null)) + " log files");
408
409    // flush all regions
410    for (HRegion r : regionServer.getOnlineRegionsLocalContext()) {
411      r.flush(true);
412    }
413    ADMIN.rollWALWriter(regionServer.getServerName());
414    TEST_UTIL.waitFor(5000, () -> {
415      int count = AbstractFSWALProvider.getNumRolledLogFiles(regionServer.getWAL(null));
416      LOG.info("after flushing all regions and rolling logs there are " + count + " log files");
417      return count <= 2;
418    });
419  }
420
421  private void setUpforLogRolling() {
422    // Force a region split after every 768KB
423    TEST_UTIL.getConfiguration().setLong(HConstants.HREGION_MAX_FILESIZE, 768L * 1024L);
424
425    // We roll the log after every 32 writes
426    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.maxlogentries", 32);
427
428    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.logroll.errors.tolerated", 2);
429    TEST_UTIL.getConfiguration().setInt("hbase.rpc.timeout", 10 * 1000);
430
431    // For less frequently updated regions flush after every 2 flushes
432    TEST_UTIL.getConfiguration().setInt("hbase.hregion.memstore.optionalflushcount", 2);
433
434    // We flush the cache after every 8192 bytes
435    TEST_UTIL.getConfiguration().setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 8192);
436
437    // Increase the amount of time between client retries
438    TEST_UTIL.getConfiguration().setLong("hbase.client.pause", 10 * 1000);
439
440    // Reduce thread wake frequency so that other threads can get
441    // a chance to run.
442    TEST_UTIL.getConfiguration().setInt(HConstants.THREAD_WAKE_FREQUENCY, 2 * 1000);
443
444    /**** configuration for testLogRollOnDatanodeDeath ****/
445    // lower the namenode & datanode heartbeat so the namenode
446    // quickly detects datanode failures
447    TEST_UTIL.getConfiguration().setInt("dfs.namenode.heartbeat.recheck-interval", 5000);
448    TEST_UTIL.getConfiguration().setInt("dfs.heartbeat.interval", 1);
449    // the namenode might still try to choose the recently-dead datanode
450    // for a pipeline, so try to a new pipeline multiple times
451    TEST_UTIL.getConfiguration().setInt("dfs.client.block.write.retries", 30);
452    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.hlog.tolerable.lowreplication", 2);
453    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.hlog.lowreplication.rolllimit", 3);
454  }
455
456  private HRegionServer startAndWriteData(TableName tableName, byte[] value)
457    throws IOException, InterruptedException {
458    // When the hbase:meta table can be opened, the region servers are running
459    TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME).close();
460
461    // Create the test table and open it
462    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
463      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build();
464    ADMIN.createTable(tableDescriptor);
465    Table table = TEST_UTIL.getConnection().getTable(tableName);
466
467    HRegionServer regionServer = TEST_UTIL.getRSForFirstRegionInTable(tableName);
468    for (int i = 1; i <= 256; i++) { // 256 writes should cause 8 log rolls
469      Put put = new Put(Bytes.toBytes("row" + String.format("%1$04d", i)));
470      put.addColumn(HConstants.CATALOG_FAMILY, null, value);
471      table.put(put);
472      if (i % 32 == 0) {
473        // After every 32 writes sleep to let the log roller run
474        try {
475          Thread.sleep(2000);
476        } catch (InterruptedException e) {
477          // continue
478        }
479      }
480    }
481
482    table.close();
483    return regionServer;
484  }
485
486  @Test
487  public void testDisableCatalogTable() throws Exception {
488    try {
489      ADMIN.disableTable(TableName.META_TABLE_NAME);
490      fail("Expected to throw ConstraintException");
491    } catch (ConstraintException e) {
492    }
493    // Before the fix for HBASE-6146, the below table creation was failing as the hbase:meta table
494    // actually getting disabled by the disableTable() call.
495    TableDescriptor tableDescriptor =
496      TableDescriptorBuilder.newBuilder(TableName.valueOf(Bytes.toBytes(name.getMethodName())))
497        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(Bytes.toBytes("cf1"))).build();
498    TEST_UTIL.getAdmin().createTable(tableDescriptor);
499  }
500
501  @Test
502  public void testIsEnabledOrDisabledOnUnknownTable() throws Exception {
503    try {
504      ADMIN.isTableEnabled(TableName.valueOf(name.getMethodName()));
505      fail("Test should fail if isTableEnabled called on unknown table.");
506    } catch (IOException e) {
507    }
508
509    try {
510      ADMIN.isTableDisabled(TableName.valueOf(name.getMethodName()));
511      fail("Test should fail if isTableDisabled called on unknown table.");
512    } catch (IOException e) {
513    }
514  }
515
516  @Test
517  public void testBalancer() throws Exception {
518    boolean initialState = ADMIN.isBalancerEnabled();
519
520    // Start the balancer, wait for it.
521    boolean prevState = ADMIN.balancerSwitch(!initialState, true);
522
523    // The previous state should be the original state we observed
524    assertEquals(initialState, prevState);
525
526    // Current state should be opposite of the original
527    assertEquals(!initialState, ADMIN.isBalancerEnabled());
528
529    // Reset it back to what it was
530    prevState = ADMIN.balancerSwitch(initialState, true);
531
532    // The previous state should be the opposite of the initial state
533    assertEquals(!initialState, prevState);
534    // Current state should be the original state again
535    assertEquals(initialState, ADMIN.isBalancerEnabled());
536  }
537
538  @Test
539  public void testRegionNormalizer() throws Exception {
540    boolean initialState = ADMIN.isNormalizerEnabled();
541
542    // flip state
543    boolean prevState = ADMIN.normalizerSwitch(!initialState);
544
545    // The previous state should be the original state we observed
546    assertEquals(initialState, prevState);
547
548    // Current state should be opposite of the original
549    assertEquals(!initialState, ADMIN.isNormalizerEnabled());
550
551    // Reset it back to what it was
552    prevState = ADMIN.normalizerSwitch(initialState);
553
554    // The previous state should be the opposite of the initial state
555    assertEquals(!initialState, prevState);
556    // Current state should be the original state again
557    assertEquals(initialState, ADMIN.isNormalizerEnabled());
558  }
559
560  @Test
561  public void testAbortProcedureFail() throws Exception {
562    long procId = ThreadLocalRandom.current().nextLong();
563    boolean abortResult = ADMIN.abortProcedure(procId, true);
564    assertFalse(abortResult);
565  }
566
567  @Test
568  public void testGetProcedures() throws Exception {
569    String procList = ADMIN.getProcedures();
570    assertTrue(procList.startsWith("["));
571  }
572
573  @Test
574  public void testGetLocks() throws Exception {
575    String lockList = ADMIN.getLocks();
576    assertTrue(lockList.startsWith("["));
577  }
578
579  @Test
580  public void testDecommissionRegionServers() throws Exception {
581    List<ServerName> decommissionedRegionServers = ADMIN.listDecommissionedRegionServers();
582    assertTrue(decommissionedRegionServers.isEmpty());
583
584    final TableName tableName = TableName.valueOf(name.getMethodName());
585    TEST_UTIL.createMultiRegionTable(tableName, Bytes.toBytes("f"), 6);
586
587    ArrayList<ServerName> clusterRegionServers = new ArrayList<>(
588      ADMIN.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().keySet());
589
590    assertEquals(3, clusterRegionServers.size());
591
592    HashMap<ServerName, List<RegionInfo>> serversToDecommssion = new HashMap<>();
593    // Get a server that has meta online. We will decommission two of the servers,
594    // leaving one online.
595    int i;
596    for (i = 0; i < clusterRegionServers.size(); i++) {
597      List<RegionInfo> regionsOnServer = ADMIN.getRegions(clusterRegionServers.get(i));
598      if (ADMIN.getRegions(clusterRegionServers.get(i)).stream().anyMatch(p -> p.isMetaRegion())) {
599        serversToDecommssion.put(clusterRegionServers.get(i), regionsOnServer);
600        break;
601      }
602    }
603
604    clusterRegionServers.remove(i);
605    // Get another server to decommission.
606    serversToDecommssion.put(clusterRegionServers.get(0),
607      ADMIN.getRegions(clusterRegionServers.get(0)));
608
609    ServerName remainingServer = clusterRegionServers.get(1);
610
611    // Decommission
612    ADMIN.decommissionRegionServers(new ArrayList<ServerName>(serversToDecommssion.keySet()), true);
613    assertEquals(2, ADMIN.listDecommissionedRegionServers().size());
614
615    // Verify the regions have been off the decommissioned servers, all on the one
616    // remaining server.
617    for (ServerName server : serversToDecommssion.keySet()) {
618      for (RegionInfo region : serversToDecommssion.get(server)) {
619        TEST_UTIL.assertRegionOnServer(region, remainingServer, 10000);
620      }
621    }
622
623    // Recommission and load the regions.
624    for (ServerName server : serversToDecommssion.keySet()) {
625      List<byte[]> encodedRegionNames = serversToDecommssion.get(server).stream()
626        .map(region -> region.getEncodedNameAsBytes()).collect(Collectors.toList());
627      ADMIN.recommissionRegionServer(server, encodedRegionNames);
628    }
629    assertTrue(ADMIN.listDecommissionedRegionServers().isEmpty());
630    // Verify the regions have been moved to the recommissioned servers
631    for (ServerName server : serversToDecommssion.keySet()) {
632      for (RegionInfo region : serversToDecommssion.get(server)) {
633        TEST_UTIL.assertRegionOnServer(region, server, 10000);
634      }
635    }
636  }
637
638  /**
639   * TestCase for HBASE-21355
640   */
641  @Test
642  public void testGetRegionInfo() throws Exception {
643    final TableName tableName = TableName.valueOf(name.getMethodName());
644    Table table = TEST_UTIL.createTable(tableName, Bytes.toBytes("f"));
645    for (int i = 0; i < 100; i++) {
646      table.put(new Put(Bytes.toBytes(i)).addColumn(Bytes.toBytes("f"), Bytes.toBytes("q"),
647        Bytes.toBytes(i)));
648    }
649    ADMIN.flush(tableName);
650
651    HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(table.getName());
652    List<HRegion> regions = rs.getRegions(tableName);
653    Assert.assertEquals(1, regions.size());
654
655    HRegion region = regions.get(0);
656    byte[] regionName = region.getRegionInfo().getRegionName();
657    HStore store = region.getStore(Bytes.toBytes("f"));
658    long expectedStoreFilesSize = store.getStorefilesSize();
659    Assert.assertNotNull(store);
660    Assert.assertEquals(expectedStoreFilesSize, store.getSize());
661    for (int i = 0; i < 10; i++) {
662      RegionInfo ri = ProtobufUtil
663        .toRegionInfo(TEST_UTIL.getAsyncConnection().getRegionServerAdmin(rs.getServerName())
664          .getRegionInfo(RequestConverter.buildGetRegionInfoRequest(regionName)).get()
665          .getRegionInfo());
666
667      Assert.assertEquals(region.getRegionInfo(), ri);
668
669      // Make sure that the store size is still the actual file system's store size.
670      Assert.assertEquals(expectedStoreFilesSize, store.getSize());
671    }
672
673    // Test querying using the encoded name only. When encoded name passed,
674    // and the target server is the Master, we return the full region name.
675    // Convenience.
676    ServerName sn = null;
677    try (Admin admin = TEST_UTIL.getConnection().getAdmin()) {
678      sn = admin.getMaster();
679    }
680    RegionInfo ri = region.getRegionInfo();
681    testGetWithRegionName(sn, ri, ri.getEncodedNameAsBytes());
682    testGetWithRegionName(sn, ri, ri.getRegionName());
683    // Try querying meta encoded name.
684    ri = RegionInfoBuilder.FIRST_META_REGIONINFO;
685    testGetWithRegionName(sn, ri, ri.getEncodedNameAsBytes());
686    testGetWithRegionName(sn, ri, ri.getRegionName());
687  }
688
689  /**
690   * Do get of RegionInfo from Master using encoded region name.
691   */
692  private void testGetWithRegionName(ServerName sn, RegionInfo inputRI, byte[] regionName)
693    throws IOException {
694    RegionInfo ri = ProtobufUtil
695      .toRegionInfo(FutureUtils.get(TEST_UTIL.getAsyncConnection().getRegionServerAdmin(sn)
696        .getRegionInfo(ProtobufUtil.getGetRegionInfoRequest(regionName))).getRegionInfo());
697    assertEquals(inputRI, ri);
698  }
699
700  @Test
701  public void testTableSplitFollowedByModify() throws Exception {
702    final TableName tableName = TableName.valueOf(name.getMethodName());
703    TEST_UTIL.createTable(tableName, Bytes.toBytes("f"));
704
705    // get the original table region count
706    List<RegionInfo> regions = ADMIN.getRegions(tableName);
707    int originalCount = regions.size();
708    assertEquals(1, originalCount);
709
710    // split the table and wait until region count increases
711    ADMIN.split(tableName, Bytes.toBytes(3));
712    TEST_UTIL.waitFor(30000, new Predicate<Exception>() {
713
714      @Override
715      public boolean evaluate() throws Exception {
716        return ADMIN.getRegions(tableName).size() > originalCount;
717      }
718    });
719
720    // do some table modification
721    TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(ADMIN.getDescriptor(tableName))
722      .setMaxFileSize(11111111).build();
723    ADMIN.modifyTable(tableDesc);
724    assertEquals(11111111, ADMIN.getDescriptor(tableName).getMaxFileSize());
725  }
726
727  @SuppressWarnings("FutureReturnValueIgnored")
728  @Test
729  public void testTableMergeFollowedByModify() throws Exception {
730    final TableName tableName = TableName.valueOf(name.getMethodName());
731    TEST_UTIL.createTable(tableName, new byte[][] { Bytes.toBytes("f") },
732      new byte[][] { Bytes.toBytes(3) });
733
734    // assert we have at least 2 regions in the table
735    List<RegionInfo> regions = ADMIN.getRegions(tableName);
736    int originalCount = regions.size();
737    assertTrue(originalCount >= 2);
738
739    byte[] nameOfRegionA = regions.get(0).getEncodedNameAsBytes();
740    byte[] nameOfRegionB = regions.get(1).getEncodedNameAsBytes();
741
742    // merge the table regions and wait until region count decreases
743    ADMIN.mergeRegionsAsync(nameOfRegionA, nameOfRegionB, true);
744    TEST_UTIL.waitFor(30000, new Predicate<Exception>() {
745
746      @Override
747      public boolean evaluate() throws Exception {
748        return ADMIN.getRegions(tableName).size() < originalCount;
749      }
750    });
751
752    // do some table modification
753    TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(ADMIN.getDescriptor(tableName))
754      .setMaxFileSize(11111111).build();
755    ADMIN.modifyTable(tableDesc);
756    assertEquals(11111111, ADMIN.getDescriptor(tableName).getMaxFileSize());
757  }
758
759  @Test
760  public void testSnapshotCleanupAsync() throws Exception {
761    testSnapshotCleanup(false);
762  }
763
764  @Test
765  public void testSnapshotCleanupSync() throws Exception {
766    testSnapshotCleanup(true);
767  }
768
769  private void testSnapshotCleanup(final boolean synchronous) throws IOException {
770    final boolean initialState = ADMIN.isSnapshotCleanupEnabled();
771    // Switch the snapshot auto cleanup state to opposite to initial state
772    boolean prevState = ADMIN.snapshotCleanupSwitch(!initialState, synchronous);
773    // The previous state should be the original state we observed
774    assertEquals(initialState, prevState);
775    // Current state should be opposite of the initial state
776    assertEquals(!initialState, ADMIN.isSnapshotCleanupEnabled());
777    // Reset the state back to what it was initially
778    prevState = ADMIN.snapshotCleanupSwitch(initialState, synchronous);
779    // The previous state should be the opposite of the initial state
780    assertEquals(!initialState, prevState);
781    // Current state should be the original state again
782    assertEquals(initialState, ADMIN.isSnapshotCleanupEnabled());
783  }
784
785  @Test
786  public void testSlowLogResponses() throws Exception {
787    // get all live server names
788    Collection<ServerName> serverNames = ADMIN.getRegionServers();
789    List<ServerName> serverNameList = new ArrayList<>(serverNames);
790
791    // clean up slowlog responses maintained in memory by RegionServers
792    List<Boolean> areSlowLogsCleared = ADMIN.clearSlowLogResponses(new HashSet<>(serverNameList));
793
794    int countFailedClearSlowResponse = 0;
795    for (Boolean isSlowLogCleared : areSlowLogsCleared) {
796      if (!isSlowLogCleared) {
797        ++countFailedClearSlowResponse;
798      }
799    }
800    Assert.assertEquals(countFailedClearSlowResponse, 0);
801
802    List<LogEntry> onlineLogRecords = ADMIN.getLogEntries(new HashSet<>(serverNames), "SLOW_LOG",
803      ServerType.REGION_SERVER, 100, null);
804    // after cleanup of slowlog responses, total count of slowlog payloads should be 0
805    Assert.assertEquals(onlineLogRecords.size(), 0);
806    List<LogEntry> balancerDecisionRecords =
807      ADMIN.getLogEntries(null, "BALANCER_DECISION", ServerType.MASTER, 100, null);
808    Assert.assertEquals(balancerDecisionRecords.size(), 0);
809  }
810
811  @Test
812  public void testGetRegionServers() throws Exception {
813    // get all live server names
814    List<ServerName> serverNames = new ArrayList<>(ADMIN.getRegionServers(true));
815    Assert.assertEquals(3, serverNames.size());
816
817    List<ServerName> serversToDecom = new ArrayList<>();
818    ServerName serverToDecommission = serverNames.get(0);
819
820    serversToDecom.add(serverToDecommission);
821    ADMIN.decommissionRegionServers(serversToDecom, false);
822    waitForServerCommissioned(serverToDecommission, true);
823
824    Assert.assertEquals(2, ADMIN.getRegionServers(true).size());
825    Assert.assertEquals(3, ADMIN.getRegionServers(false).size());
826
827    ADMIN.recommissionRegionServer(serverToDecommission, Collections.emptyList());
828    waitForServerCommissioned(null, false);
829
830    Assert.assertEquals(3, ADMIN.getRegionServers(true).size());
831    Assert.assertEquals(3, ADMIN.getRegionServers(false).size());
832  }
833
834  private static void waitForServerCommissioned(ServerName excludeServer,
835    boolean anyServerDecommissioned) {
836    TEST_UTIL.waitFor(3000, () -> {
837      try {
838        List<ServerName> decomServers = TEST_UTIL.getAdmin().listDecommissionedRegionServers();
839        if (anyServerDecommissioned) {
840          return decomServers.size() == 1 && decomServers.get(0).equals(excludeServer);
841        } else {
842          return decomServers.size() == 0;
843        }
844      } catch (IOException e) {
845        throw new RuntimeException(e);
846      }
847    });
848  }
849
850}