001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.namespace;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertNull;
024import static org.junit.Assert.assertTrue;
025import static org.junit.Assert.fail;
026
027import java.io.IOException;
028import java.util.Collections;
029import java.util.List;
030import java.util.Optional;
031import java.util.concurrent.CountDownLatch;
032import java.util.concurrent.ExecutionException;
033import java.util.concurrent.Future;
034import java.util.concurrent.TimeUnit;
035import org.apache.hadoop.conf.Configuration;
036import org.apache.hadoop.fs.FileSystem;
037import org.apache.hadoop.fs.Path;
038import org.apache.hadoop.hbase.Coprocessor;
039import org.apache.hadoop.hbase.CoprocessorEnvironment;
040import org.apache.hadoop.hbase.HBaseClassTestRule;
041import org.apache.hadoop.hbase.HBaseTestingUtil;
042import org.apache.hadoop.hbase.HConstants;
043import org.apache.hadoop.hbase.NamespaceDescriptor;
044import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
045import org.apache.hadoop.hbase.StartTestingClusterOption;
046import org.apache.hadoop.hbase.TableName;
047import org.apache.hadoop.hbase.Waiter;
048import org.apache.hadoop.hbase.client.Admin;
049import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
050import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
051import org.apache.hadoop.hbase.client.CompactionState;
052import org.apache.hadoop.hbase.client.Connection;
053import org.apache.hadoop.hbase.client.ConnectionFactory;
054import org.apache.hadoop.hbase.client.DoNotRetryRegionException;
055import org.apache.hadoop.hbase.client.RegionInfo;
056import org.apache.hadoop.hbase.client.RegionLocator;
057import org.apache.hadoop.hbase.client.Table;
058import org.apache.hadoop.hbase.client.TableDescriptor;
059import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
060import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
061import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
062import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
063import org.apache.hadoop.hbase.coprocessor.MasterObserver;
064import org.apache.hadoop.hbase.coprocessor.ObserverContext;
065import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
066import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
067import org.apache.hadoop.hbase.coprocessor.RegionObserver;
068import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessor;
069import org.apache.hadoop.hbase.coprocessor.RegionServerObserver;
070import org.apache.hadoop.hbase.master.HMaster;
071import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
072import org.apache.hadoop.hbase.master.TableNamespaceManager;
073import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
074import org.apache.hadoop.hbase.quotas.QuotaExceededException;
075import org.apache.hadoop.hbase.quotas.QuotaUtil;
076import org.apache.hadoop.hbase.regionserver.HRegion;
077import org.apache.hadoop.hbase.regionserver.Store;
078import org.apache.hadoop.hbase.regionserver.StoreFile;
079import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
080import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
081import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
082import org.apache.hadoop.hbase.testclassification.LargeTests;
083import org.apache.hadoop.hbase.util.Bytes;
084import org.apache.hadoop.hbase.util.CommonFSUtils;
085import org.apache.zookeeper.KeeperException;
086import org.junit.After;
087import org.junit.AfterClass;
088import org.junit.BeforeClass;
089import org.junit.ClassRule;
090import org.junit.Test;
091import org.junit.experimental.categories.Category;
092import org.slf4j.Logger;
093import org.slf4j.LoggerFactory;
094
095@Category(LargeTests.class)
096public class TestNamespaceAuditor {
097
098  @ClassRule
099  public static final HBaseClassTestRule CLASS_RULE =
100      HBaseClassTestRule.forClass(TestNamespaceAuditor.class);
101
102  private static final Logger LOG = LoggerFactory.getLogger(TestNamespaceAuditor.class);
103  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
104  private static Admin ADMIN;
105  private String prefix = "TestNamespaceAuditor";
106
107  @BeforeClass
108  public static void before() throws Exception {
109    Configuration conf = UTIL.getConfiguration();
110    conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, CustomObserver.class.getName());
111    conf.setStrings(
112      CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
113      MasterSyncObserver.class.getName(), CPMasterObserver.class.getName());
114    conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
115    conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
116    conf.setClass("hbase.coprocessor.regionserver.classes", CPRegionServerObserver.class,
117      RegionServerObserver.class);
118    StartTestingClusterOption option = StartTestingClusterOption.builder().numMasters(2).build();
119    UTIL.startMiniCluster(option);
120    waitForQuotaInitialize(UTIL);
121    ADMIN = UTIL.getAdmin();
122  }
123
124  @AfterClass
125  public static void tearDown() throws Exception {
126    UTIL.shutdownMiniCluster();
127  }
128
129  @After
130  public void cleanup() throws Exception, KeeperException {
131    for (TableDescriptor table : ADMIN.listTableDescriptors()) {
132      ADMIN.disableTable(table.getTableName());
133      deleteTable(table.getTableName());
134    }
135    for (NamespaceDescriptor ns : ADMIN.listNamespaceDescriptors()) {
136      if (ns.getName().startsWith(prefix)) {
137        ADMIN.deleteNamespace(ns.getName());
138      }
139    }
140    assertTrue("Quota manager not initialized", UTIL.getHBaseCluster().getMaster()
141        .getMasterQuotaManager().isQuotaInitialized());
142  }
143
144  @Test
145  public void testTableOperations() throws Exception {
146    String nsp = prefix + "_np2";
147    NamespaceDescriptor nspDesc =
148        NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5")
149            .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
150    ADMIN.createNamespace(nspDesc);
151    assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
152    assertEquals(3, ADMIN.listNamespaceDescriptors().length);
153    ColumnFamilyDescriptor columnFamilyDescriptor =
154      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("fam1")).build();
155
156    TableDescriptorBuilder tableDescOne = TableDescriptorBuilder
157      .newBuilder(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
158    tableDescOne.setColumnFamily(columnFamilyDescriptor);
159    TableDescriptorBuilder tableDescTwo = TableDescriptorBuilder
160      .newBuilder(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
161    tableDescTwo.setColumnFamily(columnFamilyDescriptor);
162    TableDescriptorBuilder tableDescThree = TableDescriptorBuilder
163      .newBuilder(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table3"));
164    tableDescThree.setColumnFamily(columnFamilyDescriptor);
165    ADMIN.createTable(tableDescOne.build());
166    boolean constraintViolated = false;
167    try {
168      ADMIN.createTable(tableDescTwo.build(), Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
169    } catch (Exception exp) {
170      assertTrue(exp instanceof IOException);
171      constraintViolated = true;
172    } finally {
173      assertTrue("Constraint not violated for table " + tableDescTwo.build().getTableName(),
174        constraintViolated);
175    }
176    ADMIN.createTable(tableDescTwo.build(), Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
177    NamespaceTableAndRegionInfo nspState = getQuotaManager().getState(nsp);
178    assertNotNull(nspState);
179    assertTrue(nspState.getTables().size() == 2);
180    assertTrue(nspState.getRegionCount() == 5);
181    constraintViolated = false;
182    try {
183      ADMIN.createTable(tableDescThree.build());
184    } catch (Exception exp) {
185      assertTrue(exp instanceof IOException);
186      constraintViolated = true;
187    } finally {
188      assertTrue("Constraint not violated for table " + tableDescThree.build().getTableName(),
189        constraintViolated);
190    }
191  }
192
193  @Test
194  public void testValidQuotas() throws Exception {
195    boolean exceptionCaught = false;
196    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
197    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
198    NamespaceDescriptor nspDesc =
199        NamespaceDescriptor.create(prefix + "vq1")
200            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "hihdufh")
201            .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
202    try {
203      ADMIN.createNamespace(nspDesc);
204    } catch (Exception exp) {
205      LOG.warn(exp.toString(), exp);
206      exceptionCaught = true;
207    } finally {
208      assertTrue(exceptionCaught);
209      assertFalse(fs.exists(CommonFSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
210    }
211    nspDesc =
212        NamespaceDescriptor.create(prefix + "vq2")
213            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "-456")
214            .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
215    try {
216      ADMIN.createNamespace(nspDesc);
217    } catch (Exception exp) {
218      LOG.warn(exp.toString(), exp);
219      exceptionCaught = true;
220    } finally {
221      assertTrue(exceptionCaught);
222      assertFalse(fs.exists(CommonFSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
223    }
224    nspDesc =
225        NamespaceDescriptor.create(prefix + "vq3")
226            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
227            .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "sciigd").build();
228    try {
229      ADMIN.createNamespace(nspDesc);
230    } catch (Exception exp) {
231      LOG.warn(exp.toString(), exp);
232      exceptionCaught = true;
233    } finally {
234      assertTrue(exceptionCaught);
235      assertFalse(fs.exists(CommonFSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
236    }
237    nspDesc =
238        NamespaceDescriptor.create(prefix + "vq4")
239            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
240            .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "-1500").build();
241    try {
242      ADMIN.createNamespace(nspDesc);
243    } catch (Exception exp) {
244      LOG.warn(exp.toString(), exp);
245      exceptionCaught = true;
246    } finally {
247      assertTrue(exceptionCaught);
248      assertFalse(fs.exists(CommonFSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
249    }
250  }
251
252  @Test
253  public void testDeleteTable() throws Exception {
254    String namespace = prefix + "_dummy";
255    NamespaceDescriptor nspDesc =
256        NamespaceDescriptor.create(namespace)
257            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "100")
258            .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "3").build();
259    ADMIN.createNamespace(nspDesc);
260    assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(namespace));
261    NamespaceTableAndRegionInfo stateInfo = getNamespaceState(nspDesc.getName());
262    assertNotNull("Namespace state found null for " + namespace, stateInfo);
263    ColumnFamilyDescriptor columnFamilyDescriptor =
264      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("fam1")).build();
265    TableDescriptorBuilder tableDescOne = TableDescriptorBuilder
266      .newBuilder(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table1"));
267    tableDescOne.setColumnFamily(columnFamilyDescriptor);
268    TableDescriptorBuilder tableDescTwo = TableDescriptorBuilder
269      .newBuilder(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table2"));
270    tableDescTwo.setColumnFamily(columnFamilyDescriptor);
271    ADMIN.createTable(tableDescOne.build());
272    ADMIN.createTable(tableDescTwo.build(), Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
273    stateInfo = getNamespaceState(nspDesc.getName());
274    assertNotNull("Namespace state found to be null.", stateInfo);
275    assertEquals(2, stateInfo.getTables().size());
276    assertEquals(5, stateInfo.getRegionCountOfTable(tableDescTwo.build().getTableName()));
277    assertEquals(6, stateInfo.getRegionCount());
278    ADMIN.disableTable(tableDescOne.build().getTableName());
279    deleteTable(tableDescOne.build().getTableName());
280    stateInfo = getNamespaceState(nspDesc.getName());
281    assertNotNull("Namespace state found to be null.", stateInfo);
282    assertEquals(5, stateInfo.getRegionCount());
283    assertEquals(1, stateInfo.getTables().size());
284    ADMIN.disableTable(tableDescTwo.build().getTableName());
285    deleteTable(tableDescTwo.build().getTableName());
286    ADMIN.deleteNamespace(namespace);
287    stateInfo = getNamespaceState(namespace);
288    assertNull("Namespace state not found to be null.", stateInfo);
289  }
290
291  public static class CPRegionServerObserver
292      implements RegionServerCoprocessor, RegionServerObserver {
293    private volatile boolean shouldFailMerge = false;
294
295    public void failMerge(boolean fail) {
296      shouldFailMerge = fail;
297    }
298
299    private boolean triggered = false;
300
301    public synchronized void waitUtilTriggered() throws InterruptedException {
302      while (!triggered) {
303        wait();
304      }
305    }
306
307    @Override
308    public Optional<RegionServerObserver> getRegionServerObserver() {
309      return Optional.of(this);
310    }
311  }
312
313  public static class CPMasterObserver implements MasterCoprocessor, MasterObserver {
314    private volatile boolean shouldFailMerge = false;
315
316    public void failMerge(boolean fail) {
317      shouldFailMerge = fail;
318    }
319
320    @Override
321    public Optional<MasterObserver> getMasterObserver() {
322      return Optional.of(this);
323    }
324
325    @Override
326    public synchronized void preMergeRegionsAction(
327        final ObserverContext<MasterCoprocessorEnvironment> ctx,
328        final RegionInfo[] regionsToMerge) throws IOException {
329      notifyAll();
330      if (shouldFailMerge) {
331        throw new IOException("fail merge");
332      }
333    }
334  }
335
336  @Test
337  public void testRegionMerge() throws Exception {
338    String nsp1 = prefix + "_regiontest";
339    final int initialRegions = 3;
340    NamespaceDescriptor nspDesc =
341        NamespaceDescriptor.create(nsp1)
342            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "" + initialRegions)
343            .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
344    ADMIN.createNamespace(nspDesc);
345    final TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
346    byte[] columnFamily = Bytes.toBytes("info");
347    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableTwo)
348      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(columnFamily)).build();
349    ADMIN.createTable(tableDescriptor, Bytes.toBytes("0"), Bytes.toBytes("9"), initialRegions);
350    Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
351    try (Table table = connection.getTable(tableTwo)) {
352      UTIL.loadNumericRows(table, Bytes.toBytes("info"), 1000, 1999);
353    }
354    ADMIN.flush(tableTwo);
355    List<RegionInfo> hris = ADMIN.getRegions(tableTwo);
356    assertEquals(initialRegions, hris.size());
357    Collections.sort(hris, RegionInfo.COMPARATOR);
358    Future<?> f = ADMIN.mergeRegionsAsync(
359      hris.get(0).getEncodedNameAsBytes(),
360      hris.get(1).getEncodedNameAsBytes(),
361      false);
362    f.get(10, TimeUnit.SECONDS);
363
364    hris = ADMIN.getRegions(tableTwo);
365    assertEquals(initialRegions - 1, hris.size());
366    Collections.sort(hris, RegionInfo.COMPARATOR);
367    byte[] splitKey = Bytes.toBytes("3");
368    HRegion regionToSplit = UTIL.getMiniHBaseCluster().getRegions(tableTwo).stream()
369      .filter(r -> r.getRegionInfo().containsRow(splitKey)).findFirst().get();
370    regionToSplit.compact(true);
371    // Waiting for compaction to finish
372    UTIL.waitFor(30000, new Waiter.Predicate<Exception>() {
373      @Override
374      public boolean evaluate() throws Exception {
375        return (CompactionState.NONE == ADMIN
376            .getCompactionStateForRegion(regionToSplit.getRegionInfo().getRegionName()));
377      }
378    });
379
380    // Cleaning compacted references for split to proceed
381    regionToSplit.getStores().stream().forEach(s -> {
382      try {
383        s.closeAndArchiveCompactedFiles();
384      } catch (IOException e1) {
385        LOG.error("Error whiling cleaning compacted file");
386      }
387    });
388    // the above compact may quit immediately if there is a compaction ongoing, so here we need to
389    // wait a while to let the ongoing compaction finish.
390    UTIL.waitFor(10000, regionToSplit::isSplittable);
391    ADMIN.splitRegionAsync(regionToSplit.getRegionInfo().getRegionName(), splitKey).get(10,
392      TimeUnit.SECONDS);
393    hris = ADMIN.getRegions(tableTwo);
394    assertEquals(initialRegions, hris.size());
395    Collections.sort(hris, RegionInfo.COMPARATOR);
396
397    // Fail region merge through Coprocessor hook
398    SingleProcessHBaseCluster cluster = UTIL.getHBaseCluster();
399    MasterCoprocessorHost cpHost = cluster.getMaster().getMasterCoprocessorHost();
400    Coprocessor coprocessor = cpHost.findCoprocessor(CPMasterObserver.class);
401    CPMasterObserver masterObserver = (CPMasterObserver) coprocessor;
402    masterObserver.failMerge(true);
403
404    f = ADMIN.mergeRegionsAsync(
405      hris.get(1).getEncodedNameAsBytes(),
406      hris.get(2).getEncodedNameAsBytes(),
407      false);
408    try {
409      f.get(10, TimeUnit.SECONDS);
410      fail("Merge was supposed to fail!");
411    } catch (ExecutionException ee) {
412      // Expected.
413    }
414    hris = ADMIN.getRegions(tableTwo);
415    assertEquals(initialRegions, hris.size());
416    Collections.sort(hris, RegionInfo.COMPARATOR);
417    // verify that we cannot split
418    try {
419      ADMIN.split(tableTwo, Bytes.toBytes("6"));
420      fail();
421    } catch (DoNotRetryRegionException e) {
422      // Expected
423    }
424    Thread.sleep(2000);
425    assertEquals(initialRegions, ADMIN.getRegions(tableTwo).size());
426  }
427
428  /*
429   * Create a table and make sure that the table creation fails after adding this table entry into
430   * namespace quota cache. Now correct the failure and recreate the table with same name.
431   * HBASE-13394
432   */
433  @Test
434  public void testRecreateTableWithSameNameAfterFirstTimeFailure() throws Exception {
435    String nsp1 = prefix + "_testRecreateTable";
436    NamespaceDescriptor nspDesc =
437        NamespaceDescriptor.create(nsp1)
438            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
439            .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1").build();
440    ADMIN.createNamespace(nspDesc);
441    final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
442    byte[] columnFamily = Bytes.toBytes("info");
443    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableOne)
444      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(columnFamily)).build();
445    MasterSyncObserver.throwExceptionInPreCreateTableAction = true;
446    try {
447      try {
448        ADMIN.createTable(tableDescriptor);
449        fail("Table " + tableOne.toString() + "creation should fail.");
450      } catch (Exception exp) {
451        LOG.error(exp.toString(), exp);
452      }
453      assertFalse(ADMIN.tableExists(tableOne));
454
455      NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp1);
456      assertEquals("First table creation failed in namespace so number of tables in namespace "
457          + "should be 0.", 0, nstate.getTables().size());
458
459      MasterSyncObserver.throwExceptionInPreCreateTableAction = false;
460      try {
461        ADMIN.createTable(tableDescriptor);
462      } catch (Exception e) {
463        fail("Table " + tableOne.toString() + "creation should succeed.");
464        LOG.error(e.toString(), e);
465      }
466      assertTrue(ADMIN.tableExists(tableOne));
467      nstate = getNamespaceState(nsp1);
468      assertEquals("First table was created successfully so table size in namespace should "
469          + "be one now.", 1, nstate.getTables().size());
470    } finally {
471      MasterSyncObserver.throwExceptionInPreCreateTableAction = false;
472      if (ADMIN.tableExists(tableOne)) {
473        ADMIN.disableTable(tableOne);
474        deleteTable(tableOne);
475      }
476      ADMIN.deleteNamespace(nsp1);
477    }
478  }
479
480  private NamespaceTableAndRegionInfo getNamespaceState(String namespace) throws KeeperException,
481      IOException {
482    return getQuotaManager().getState(namespace);
483  }
484
485  public static class CustomObserver implements RegionCoprocessor, RegionObserver {
486    volatile CountDownLatch postCompact;
487
488    @Override
489    public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store,
490        StoreFile resultFile, CompactionLifeCycleTracker tracker, CompactionRequest request)
491        throws IOException {
492      postCompact.countDown();
493    }
494
495    @Override
496    public void start(CoprocessorEnvironment e) throws IOException {
497      postCompact = new CountDownLatch(1);
498    }
499
500    @Override
501    public Optional<RegionObserver> getRegionObserver() {
502      return Optional.of(this);
503    }
504  }
505
506  @Test
507  public void testStatePreserve() throws Exception {
508    final String nsp1 = prefix + "_testStatePreserve";
509    NamespaceDescriptor nspDesc = NamespaceDescriptor.create(nsp1)
510        .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
511        .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "10").build();
512    ADMIN.createNamespace(nspDesc);
513    TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
514    TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
515    TableName tableThree = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table3");
516    ColumnFamilyDescriptor columnFamilyDescriptor =
517      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("fam1")).build();
518    TableDescriptorBuilder tableDescOne = TableDescriptorBuilder
519      .newBuilder(tableOne);
520    tableDescOne.setColumnFamily(columnFamilyDescriptor);
521    TableDescriptorBuilder tableDescTwo = TableDescriptorBuilder
522      .newBuilder(tableTwo);
523    tableDescTwo.setColumnFamily(columnFamilyDescriptor);
524    TableDescriptorBuilder tableDescThree = TableDescriptorBuilder
525      .newBuilder(tableThree);
526    tableDescThree.setColumnFamily(columnFamilyDescriptor);
527
528    ADMIN.createTable(tableDescOne.build(), Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
529    ADMIN.createTable(tableDescTwo.build(), Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
530    ADMIN.createTable(tableDescThree.build(), Bytes.toBytes("1"), Bytes.toBytes("1000"), 4);
531    ADMIN.disableTable(tableThree);
532    deleteTable(tableThree);
533    // wait for chore to complete
534    UTIL.waitFor(1000, new Waiter.Predicate<Exception>() {
535      @Override
536      public boolean evaluate() throws Exception {
537        return (getNamespaceState(nsp1).getTables().size() == 2);
538      }
539    });
540    NamespaceTableAndRegionInfo before = getNamespaceState(nsp1);
541    killActiveMaster();
542    NamespaceTableAndRegionInfo after = getNamespaceState(nsp1);
543    assertEquals("Expected: " + before.getTables() + " Found: " + after.getTables(), before
544        .getTables().size(), after.getTables().size());
545  }
546
547  public static void waitForQuotaInitialize(final HBaseTestingUtil util) throws Exception {
548    util.waitFor(60000, new Waiter.Predicate<Exception>() {
549      @Override
550      public boolean evaluate() throws Exception {
551        HMaster master = util.getHBaseCluster().getMaster();
552        if (master == null) {
553          return false;
554        }
555        MasterQuotaManager quotaManager = master.getMasterQuotaManager();
556        return quotaManager != null && quotaManager.isQuotaInitialized();
557      }
558    });
559  }
560
561  private void killActiveMaster() throws Exception {
562    UTIL.getHBaseCluster().getMaster(0).stop("Stopping to start again");
563    UTIL.getHBaseCluster().waitOnMaster(0);
564    waitForQuotaInitialize(UTIL);
565  }
566
567  private NamespaceAuditor getQuotaManager() {
568    return UTIL.getHBaseCluster().getMaster()
569        .getMasterQuotaManager().getNamespaceQuotaManager();
570  }
571
572  public static class MasterSyncObserver implements MasterCoprocessor, MasterObserver {
573    volatile CountDownLatch tableDeletionLatch;
574    static boolean throwExceptionInPreCreateTableAction;
575
576    @Override
577    public Optional<MasterObserver> getMasterObserver() {
578      return Optional.of(this);
579    }
580
581    @Override
582    public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
583        TableName tableName) throws IOException {
584      tableDeletionLatch = new CountDownLatch(1);
585    }
586
587    @Override
588    public void postCompletedDeleteTableAction(
589        final ObserverContext<MasterCoprocessorEnvironment> ctx,
590        final TableName tableName) throws IOException {
591      tableDeletionLatch.countDown();
592    }
593
594    @Override
595    public void preCreateTableAction(ObserverContext<MasterCoprocessorEnvironment> ctx,
596        TableDescriptor desc, RegionInfo[] regions) throws IOException {
597      if (throwExceptionInPreCreateTableAction) {
598        throw new IOException("Throw exception as it is demanded.");
599      }
600    }
601  }
602
603  private void deleteTable(final TableName tableName) throws Exception {
604    // NOTE: We need a latch because admin is not sync,
605    // so the postOp coprocessor method may be called after the admin operation returned.
606    MasterSyncObserver observer = UTIL.getHBaseCluster().getMaster()
607      .getMasterCoprocessorHost().findCoprocessor(MasterSyncObserver.class);
608    ADMIN.deleteTable(tableName);
609    observer.tableDeletionLatch.await();
610  }
611
612  @Test(expected = QuotaExceededException.class)
613  public void testExceedTableQuotaInNamespace() throws Exception {
614    String nsp = prefix + "_testExceedTableQuotaInNamespace";
615    NamespaceDescriptor nspDesc =
616        NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
617            .build();
618    ADMIN.createNamespace(nspDesc);
619    assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
620    assertEquals(3, ADMIN.listNamespaceDescriptors().length);
621    ColumnFamilyDescriptor columnFamilyDescriptor =
622      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("fam1")).build();
623    TableDescriptorBuilder tableDescOne = TableDescriptorBuilder
624      .newBuilder(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
625    tableDescOne.setColumnFamily(columnFamilyDescriptor);
626    TableDescriptorBuilder tableDescTwo = TableDescriptorBuilder
627      .newBuilder(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
628    tableDescTwo.setColumnFamily(columnFamilyDescriptor);
629    ADMIN.createTable(tableDescOne.build());
630    ADMIN.createTable(tableDescTwo.build(), Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
631  }
632
633  @Test(expected = QuotaExceededException.class)
634  public void testCloneSnapshotQuotaExceed() throws Exception {
635    String nsp = prefix + "_testTableQuotaExceedWithCloneSnapshot";
636    NamespaceDescriptor nspDesc =
637        NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
638            .build();
639    ADMIN.createNamespace(nspDesc);
640    assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
641    TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
642    TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
643    ColumnFamilyDescriptor columnFamilyDescriptor =
644      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("fam1")).build();
645    TableDescriptorBuilder tableDescOne = TableDescriptorBuilder
646      .newBuilder(tableName);
647    tableDescOne.setColumnFamily(columnFamilyDescriptor);
648    ADMIN.createTable(tableDescOne.build());
649    String snapshot = "snapshot_testTableQuotaExceedWithCloneSnapshot";
650    ADMIN.snapshot(snapshot, tableName);
651    ADMIN.cloneSnapshot(snapshot, cloneTableName);
652    ADMIN.deleteSnapshot(snapshot);
653  }
654
655  @Test
656  public void testCloneSnapshot() throws Exception {
657    String nsp = prefix + "_testCloneSnapshot";
658    NamespaceDescriptor nspDesc =
659        NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2")
660            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20").build();
661    ADMIN.createNamespace(nspDesc);
662    assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
663    TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
664    TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
665
666    ColumnFamilyDescriptor columnFamilyDescriptor =
667      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("fam1")).build();
668    TableDescriptorBuilder tableDescOne = TableDescriptorBuilder
669      .newBuilder(tableName);
670    tableDescOne.setColumnFamily(columnFamilyDescriptor);
671
672    ADMIN.createTable(tableDescOne.build(), Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
673    String snapshot = "snapshot_testCloneSnapshot";
674    ADMIN.snapshot(snapshot, tableName);
675    ADMIN.cloneSnapshot(snapshot, cloneTableName);
676
677    int tableLength;
678    try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(tableName)) {
679      tableLength = locator.getStartKeys().length;
680    }
681    assertEquals(tableName.getNameAsString() + " should have four regions.", 4, tableLength);
682
683    try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(cloneTableName)) {
684      tableLength = locator.getStartKeys().length;
685    }
686    assertEquals(cloneTableName.getNameAsString() + " should have four regions.", 4, tableLength);
687
688    NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
689    assertEquals("Total tables count should be 2.", 2, nstate.getTables().size());
690    assertEquals("Total regions count should be.", 8, nstate.getRegionCount());
691
692    ADMIN.deleteSnapshot(snapshot);
693  }
694
695  @Test
696  public void testRestoreSnapshot() throws Exception {
697    String nsp = prefix + "_testRestoreSnapshot";
698    NamespaceDescriptor nspDesc =
699        NamespaceDescriptor.create(nsp)
700            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
701    ADMIN.createNamespace(nspDesc);
702    assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
703    TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
704    ColumnFamilyDescriptor columnFamilyDescriptor =
705      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("fam1")).build();
706    TableDescriptorBuilder tableDescOne = TableDescriptorBuilder
707      .newBuilder(tableName1);
708    tableDescOne.setColumnFamily(columnFamilyDescriptor);
709    ADMIN.createTable(tableDescOne.build(), Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
710
711    NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
712    assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
713
714    String snapshot = "snapshot_testRestoreSnapshot";
715    ADMIN.snapshot(snapshot, tableName1);
716
717    List<RegionInfo> regions = ADMIN.getRegions(tableName1);
718    Collections.sort(regions, RegionInfo.COMPARATOR);
719
720    ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
721    Thread.sleep(2000);
722    assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());
723
724    ADMIN.disableTable(tableName1);
725    ADMIN.restoreSnapshot(snapshot);
726
727    assertEquals("Total regions count should be 4 after restore.", 4, nstate.getRegionCount());
728
729    ADMIN.enableTable(tableName1);
730    ADMIN.deleteSnapshot(snapshot);
731  }
732
733  @Test
734  public void testRestoreSnapshotQuotaExceed() throws Exception {
735    String nsp = prefix + "_testRestoreSnapshotQuotaExceed";
736    NamespaceDescriptor nspDesc =
737        NamespaceDescriptor.create(nsp)
738            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
739    ADMIN.createNamespace(nspDesc);
740    NamespaceDescriptor ndesc = ADMIN.getNamespaceDescriptor(nsp);
741    assertNotNull("Namespace descriptor found null.", ndesc);
742    TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
743    ColumnFamilyDescriptor columnFamilyDescriptor =
744      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("fam1")).build();
745    TableDescriptorBuilder tableDescOne = TableDescriptorBuilder
746      .newBuilder(tableName1);
747    tableDescOne.setColumnFamily(columnFamilyDescriptor);
748
749    ADMIN.createTable(tableDescOne.build(), Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
750
751    NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
752    assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
753
754    String snapshot = "snapshot_testRestoreSnapshotQuotaExceed";
755    // snapshot has 4 regions
756    ADMIN.snapshot(snapshot, tableName1);
757    // recreate table with 1 region and set max regions to 3 for namespace
758    ADMIN.disableTable(tableName1);
759    ADMIN.deleteTable(tableName1);
760    ADMIN.createTable(tableDescOne.build());
761    ndesc.setConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "3");
762    ADMIN.modifyNamespace(ndesc);
763
764    ADMIN.disableTable(tableName1);
765    try {
766      ADMIN.restoreSnapshot(snapshot);
767      fail("Region quota is exceeded so QuotaExceededException should be thrown but HBaseAdmin"
768          + " wraps IOException into RestoreSnapshotException");
769    } catch (RestoreSnapshotException ignore) {
770      assertTrue(ignore.getCause() instanceof QuotaExceededException);
771    }
772    assertEquals(1, getNamespaceState(nsp).getRegionCount());
773    ADMIN.enableTable(tableName1);
774    ADMIN.deleteSnapshot(snapshot);
775  }
776}