Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.namespace;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertNull;
024import static org.junit.Assert.assertTrue;
025import static org.junit.Assert.fail;
026
027import java.io.IOException;
028import java.util.Collections;
029import java.util.List;
030import java.util.Optional;
031import java.util.concurrent.CountDownLatch;
032import java.util.concurrent.ExecutionException;
033import java.util.concurrent.Future;
034import java.util.concurrent.TimeUnit;
035import org.apache.commons.lang3.StringUtils;
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.fs.FileSystem;
038import org.apache.hadoop.fs.Path;
039import org.apache.hadoop.hbase.Coprocessor;
040import org.apache.hadoop.hbase.CoprocessorEnvironment;
041import org.apache.hadoop.hbase.HBaseClassTestRule;
042import org.apache.hadoop.hbase.HBaseTestingUtility;
043import org.apache.hadoop.hbase.HColumnDescriptor;
044import org.apache.hadoop.hbase.HConstants;
045import org.apache.hadoop.hbase.HRegionInfo;
046import org.apache.hadoop.hbase.HTableDescriptor;
047import org.apache.hadoop.hbase.MiniHBaseCluster;
048import org.apache.hadoop.hbase.NamespaceDescriptor;
049import org.apache.hadoop.hbase.TableName;
050import org.apache.hadoop.hbase.Waiter;
051import org.apache.hadoop.hbase.client.Admin;
052import org.apache.hadoop.hbase.client.CompactionState;
053import org.apache.hadoop.hbase.client.Connection;
054import org.apache.hadoop.hbase.client.ConnectionFactory;
055import org.apache.hadoop.hbase.client.DoNotRetryRegionException;
056import org.apache.hadoop.hbase.client.RegionInfo;
057import org.apache.hadoop.hbase.client.RegionLocator;
058import org.apache.hadoop.hbase.client.Table;
059import org.apache.hadoop.hbase.client.TableDescriptor;
060import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
061import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
062import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
063import org.apache.hadoop.hbase.coprocessor.MasterObserver;
064import org.apache.hadoop.hbase.coprocessor.ObserverContext;
065import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
066import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
067import org.apache.hadoop.hbase.coprocessor.RegionObserver;
068import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessor;
069import org.apache.hadoop.hbase.coprocessor.RegionServerObserver;
070import org.apache.hadoop.hbase.master.HMaster;
071import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
072import org.apache.hadoop.hbase.master.TableNamespaceManager;
073import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
074import org.apache.hadoop.hbase.quotas.QuotaExceededException;
075import org.apache.hadoop.hbase.quotas.QuotaUtil;
076import org.apache.hadoop.hbase.regionserver.HRegion;
077import org.apache.hadoop.hbase.regionserver.Store;
078import org.apache.hadoop.hbase.regionserver.StoreFile;
079import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
080import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
081import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
082import org.apache.hadoop.hbase.testclassification.MediumTests;
083import org.apache.hadoop.hbase.util.Bytes;
084import org.apache.hadoop.hbase.util.FSUtils;
085import org.apache.zookeeper.KeeperException;
086import org.junit.After;
087import org.junit.AfterClass;
088import org.junit.BeforeClass;
089import org.junit.ClassRule;
090import org.junit.Test;
091import org.junit.experimental.categories.Category;
092import org.slf4j.Logger;
093import org.slf4j.LoggerFactory;
094
095@Category(MediumTests.class)
096public class TestNamespaceAuditor {
097
098  @ClassRule
099  public static final HBaseClassTestRule CLASS_RULE =
100      HBaseClassTestRule.forClass(TestNamespaceAuditor.class);
101
102  private static final Logger LOG = LoggerFactory.getLogger(TestNamespaceAuditor.class);
103  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
104  private static Admin ADMIN;
105  private String prefix = "TestNamespaceAuditor";
106
107  @BeforeClass
108  public static void before() throws Exception {
109    Configuration conf = UTIL.getConfiguration();
110    conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, CustomObserver.class.getName());
111    conf.setStrings(
112      CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
113      MasterSyncObserver.class.getName(), CPMasterObserver.class.getName());
114    conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
115    conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
116    conf.setClass("hbase.coprocessor.regionserver.classes", CPRegionServerObserver.class,
117      RegionServerObserver.class);
118    UTIL.startMiniCluster(1, 1);
119    waitForQuotaInitialize(UTIL);
120    ADMIN = UTIL.getAdmin();
121  }
122
123  @AfterClass
124  public static void tearDown() throws Exception {
125    UTIL.shutdownMiniCluster();
126  }
127
128  @After
129  public void cleanup() throws Exception, KeeperException {
130    for (HTableDescriptor table : ADMIN.listTables()) {
131      ADMIN.disableTable(table.getTableName());
132      deleteTable(table.getTableName());
133    }
134    for (NamespaceDescriptor ns : ADMIN.listNamespaceDescriptors()) {
135      if (ns.getName().startsWith(prefix)) {
136        ADMIN.deleteNamespace(ns.getName());
137      }
138    }
139    assertTrue("Quota manager not initialized", UTIL.getHBaseCluster().getMaster()
140        .getMasterQuotaManager().isQuotaInitialized());
141  }
142
143  @Test
144  public void testTableOperations() throws Exception {
145    String nsp = prefix + "_np2";
146    NamespaceDescriptor nspDesc =
147        NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5")
148            .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
149    ADMIN.createNamespace(nspDesc);
150    assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
151    assertEquals(3, ADMIN.listNamespaceDescriptors().length);
152    HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
153
154    HTableDescriptor tableDescOne =
155        new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
156    tableDescOne.addFamily(fam1);
157    HTableDescriptor tableDescTwo =
158        new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
159    tableDescTwo.addFamily(fam1);
160    HTableDescriptor tableDescThree =
161        new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table3"));
162    tableDescThree.addFamily(fam1);
163    ADMIN.createTable(tableDescOne);
164    boolean constraintViolated = false;
165    try {
166      ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
167    } catch (Exception exp) {
168      assertTrue(exp instanceof IOException);
169      constraintViolated = true;
170    } finally {
171      assertTrue("Constraint not violated for table " + tableDescTwo.getTableName(),
172        constraintViolated);
173    }
174    ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
175    NamespaceTableAndRegionInfo nspState = getQuotaManager().getState(nsp);
176    assertNotNull(nspState);
177    assertTrue(nspState.getTables().size() == 2);
178    assertTrue(nspState.getRegionCount() == 5);
179    constraintViolated = false;
180    try {
181      ADMIN.createTable(tableDescThree);
182    } catch (Exception exp) {
183      assertTrue(exp instanceof IOException);
184      constraintViolated = true;
185    } finally {
186      assertTrue("Constraint not violated for table " + tableDescThree.getTableName(),
187        constraintViolated);
188    }
189  }
190
191  @Test
192  public void testValidQuotas() throws Exception {
193    boolean exceptionCaught = false;
194    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
195    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
196    NamespaceDescriptor nspDesc =
197        NamespaceDescriptor.create(prefix + "vq1")
198            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "hihdufh")
199            .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
200    try {
201      ADMIN.createNamespace(nspDesc);
202    } catch (Exception exp) {
203      LOG.warn(exp.toString(), exp);
204      exceptionCaught = true;
205    } finally {
206      assertTrue(exceptionCaught);
207      assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
208    }
209    nspDesc =
210        NamespaceDescriptor.create(prefix + "vq2")
211            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "-456")
212            .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
213    try {
214      ADMIN.createNamespace(nspDesc);
215    } catch (Exception exp) {
216      LOG.warn(exp.toString(), exp);
217      exceptionCaught = true;
218    } finally {
219      assertTrue(exceptionCaught);
220      assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
221    }
222    nspDesc =
223        NamespaceDescriptor.create(prefix + "vq3")
224            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
225            .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "sciigd").build();
226    try {
227      ADMIN.createNamespace(nspDesc);
228    } catch (Exception exp) {
229      LOG.warn(exp.toString(), exp);
230      exceptionCaught = true;
231    } finally {
232      assertTrue(exceptionCaught);
233      assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
234    }
235    nspDesc =
236        NamespaceDescriptor.create(prefix + "vq4")
237            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
238            .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "-1500").build();
239    try {
240      ADMIN.createNamespace(nspDesc);
241    } catch (Exception exp) {
242      LOG.warn(exp.toString(), exp);
243      exceptionCaught = true;
244    } finally {
245      assertTrue(exceptionCaught);
246      assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
247    }
248  }
249
250  @Test
251  public void testDeleteTable() throws Exception {
252    String namespace = prefix + "_dummy";
253    NamespaceDescriptor nspDesc =
254        NamespaceDescriptor.create(namespace)
255            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "100")
256            .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "3").build();
257    ADMIN.createNamespace(nspDesc);
258    assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(namespace));
259    NamespaceTableAndRegionInfo stateInfo = getNamespaceState(nspDesc.getName());
260    assertNotNull("Namespace state found null for " + namespace, stateInfo);
261    HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
262    HTableDescriptor tableDescOne =
263        new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table1"));
264    tableDescOne.addFamily(fam1);
265    HTableDescriptor tableDescTwo =
266        new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table2"));
267    tableDescTwo.addFamily(fam1);
268    ADMIN.createTable(tableDescOne);
269    ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
270    stateInfo = getNamespaceState(nspDesc.getName());
271    assertNotNull("Namespace state found to be null.", stateInfo);
272    assertEquals(2, stateInfo.getTables().size());
273    assertEquals(5, stateInfo.getRegionCountOfTable(tableDescTwo.getTableName()));
274    assertEquals(6, stateInfo.getRegionCount());
275    ADMIN.disableTable(tableDescOne.getTableName());
276    deleteTable(tableDescOne.getTableName());
277    stateInfo = getNamespaceState(nspDesc.getName());
278    assertNotNull("Namespace state found to be null.", stateInfo);
279    assertEquals(5, stateInfo.getRegionCount());
280    assertEquals(1, stateInfo.getTables().size());
281    ADMIN.disableTable(tableDescTwo.getTableName());
282    deleteTable(tableDescTwo.getTableName());
283    ADMIN.deleteNamespace(namespace);
284    stateInfo = getNamespaceState(namespace);
285    assertNull("Namespace state not found to be null.", stateInfo);
286  }
287
288  public static class CPRegionServerObserver
289      implements RegionServerCoprocessor, RegionServerObserver {
290    private volatile boolean shouldFailMerge = false;
291
292    public void failMerge(boolean fail) {
293      shouldFailMerge = fail;
294    }
295
296    private boolean triggered = false;
297
298    public synchronized void waitUtilTriggered() throws InterruptedException {
299      while (!triggered) {
300        wait();
301      }
302    }
303
304    @Override
305    public Optional<RegionServerObserver> getRegionServerObserver() {
306      return Optional.of(this);
307    }
308  }
309
310  public static class CPMasterObserver implements MasterCoprocessor, MasterObserver {
311    private volatile boolean shouldFailMerge = false;
312
313    public void failMerge(boolean fail) {
314      shouldFailMerge = fail;
315    }
316
317    @Override
318    public Optional<MasterObserver> getMasterObserver() {
319      return Optional.of(this);
320    }
321
322    @Override
323    public synchronized void preMergeRegionsAction(
324        final ObserverContext<MasterCoprocessorEnvironment> ctx,
325        final RegionInfo[] regionsToMerge) throws IOException {
326      notifyAll();
327      if (shouldFailMerge) {
328        throw new IOException("fail merge");
329      }
330    }
331  }
332
333  @Test
334  public void testRegionMerge() throws Exception {
335    String nsp1 = prefix + "_regiontest";
336    final int initialRegions = 3;
337    NamespaceDescriptor nspDesc =
338        NamespaceDescriptor.create(nsp1)
339            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "" + initialRegions)
340            .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
341    ADMIN.createNamespace(nspDesc);
342    final TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
343    byte[] columnFamily = Bytes.toBytes("info");
344    HTableDescriptor tableDescOne = new HTableDescriptor(tableTwo);
345    tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
346    ADMIN.createTable(tableDescOne, Bytes.toBytes("0"), Bytes.toBytes("9"), initialRegions);
347    Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
348    try (Table table = connection.getTable(tableTwo)) {
349      UTIL.loadNumericRows(table, Bytes.toBytes("info"), 1000, 1999);
350    }
351    ADMIN.flush(tableTwo);
352    List<RegionInfo> hris = ADMIN.getRegions(tableTwo);
353    assertEquals(initialRegions, hris.size());
354    Collections.sort(hris, RegionInfo.COMPARATOR);
355    Future<?> f = ADMIN.mergeRegionsAsync(
356      hris.get(0).getEncodedNameAsBytes(),
357      hris.get(1).getEncodedNameAsBytes(),
358      false);
359    f.get(10, TimeUnit.SECONDS);
360
361    hris = ADMIN.getRegions(tableTwo);
362    assertEquals(initialRegions - 1, hris.size());
363    Collections.sort(hris, RegionInfo.COMPARATOR);
364    byte[] splitKey = Bytes.toBytes("3");
365    HRegion regionToSplit = UTIL.getMiniHBaseCluster().getRegions(tableTwo).stream()
366      .filter(r -> r.getRegionInfo().containsRow(splitKey)).findFirst().get();
367    regionToSplit.compact(true);
368    // Waiting for compaction to finish
369    UTIL.waitFor(30000, new Waiter.Predicate<Exception>() {
370      @Override
371      public boolean evaluate() throws Exception {
372        return (CompactionState.NONE == ADMIN
373            .getCompactionStateForRegion(regionToSplit.getRegionInfo().getRegionName()));
374      }
375    });
376
377    // Cleaning compacted references for split to proceed
378    regionToSplit.getStores().stream().forEach(s -> {
379      try {
380        s.closeAndArchiveCompactedFiles();
381      } catch (IOException e1) {
382        LOG.error("Error whiling cleaning compacted file");
383      }
384    });
385    // the above compact may quit immediately if there is a compaction ongoing, so here we need to
386    // wait a while to let the ongoing compaction finish.
387    UTIL.waitFor(10000, regionToSplit::isSplittable);
388    ADMIN.splitRegionAsync(regionToSplit.getRegionInfo().getRegionName(), splitKey).get(10,
389      TimeUnit.SECONDS);
390    hris = ADMIN.getRegions(tableTwo);
391    assertEquals(initialRegions, hris.size());
392    Collections.sort(hris, RegionInfo.COMPARATOR);
393
394    // Fail region merge through Coprocessor hook
395    MiniHBaseCluster cluster = UTIL.getHBaseCluster();
396    MasterCoprocessorHost cpHost = cluster.getMaster().getMasterCoprocessorHost();
397    Coprocessor coprocessor = cpHost.findCoprocessor(CPMasterObserver.class);
398    CPMasterObserver masterObserver = (CPMasterObserver) coprocessor;
399    masterObserver.failMerge(true);
400
401    f = ADMIN.mergeRegionsAsync(
402      hris.get(1).getEncodedNameAsBytes(),
403      hris.get(2).getEncodedNameAsBytes(),
404      false);
405    try {
406      f.get(10, TimeUnit.SECONDS);
407      fail("Merge was supposed to fail!");
408    } catch (ExecutionException ee) {
409      // Expected.
410    }
411    hris = ADMIN.getRegions(tableTwo);
412    assertEquals(initialRegions, hris.size());
413    Collections.sort(hris, RegionInfo.COMPARATOR);
414    // verify that we cannot split
415    try {
416      ADMIN.split(tableTwo, Bytes.toBytes("6"));
417      fail();
418    } catch (DoNotRetryRegionException e) {
419      // Expected
420    }
421    Thread.sleep(2000);
422    assertEquals(initialRegions, ADMIN.getRegions(tableTwo).size());
423  }
424
425  /*
426   * Create a table and make sure that the table creation fails after adding this table entry into
427   * namespace quota cache. Now correct the failure and recreate the table with same name.
428   * HBASE-13394
429   */
430  @Test
431  public void testRecreateTableWithSameNameAfterFirstTimeFailure() throws Exception {
432    String nsp1 = prefix + "_testRecreateTable";
433    NamespaceDescriptor nspDesc =
434        NamespaceDescriptor.create(nsp1)
435            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
436            .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1").build();
437    ADMIN.createNamespace(nspDesc);
438    final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
439    byte[] columnFamily = Bytes.toBytes("info");
440    HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
441    tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
442    MasterSyncObserver.throwExceptionInPreCreateTableAction = true;
443    try {
444      try {
445        ADMIN.createTable(tableDescOne);
446        fail("Table " + tableOne.toString() + "creation should fail.");
447      } catch (Exception exp) {
448        LOG.error(exp.toString(), exp);
449      }
450      assertFalse(ADMIN.tableExists(tableOne));
451
452      NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp1);
453      assertEquals("First table creation failed in namespace so number of tables in namespace "
454          + "should be 0.", 0, nstate.getTables().size());
455
456      MasterSyncObserver.throwExceptionInPreCreateTableAction = false;
457      try {
458        ADMIN.createTable(tableDescOne);
459      } catch (Exception e) {
460        fail("Table " + tableOne.toString() + "creation should succeed.");
461        LOG.error(e.toString(), e);
462      }
463      assertTrue(ADMIN.tableExists(tableOne));
464      nstate = getNamespaceState(nsp1);
465      assertEquals("First table was created successfully so table size in namespace should "
466          + "be one now.", 1, nstate.getTables().size());
467    } finally {
468      MasterSyncObserver.throwExceptionInPreCreateTableAction = false;
469      if (ADMIN.tableExists(tableOne)) {
470        ADMIN.disableTable(tableOne);
471        deleteTable(tableOne);
472      }
473      ADMIN.deleteNamespace(nsp1);
474    }
475  }
476
477  private NamespaceTableAndRegionInfo getNamespaceState(String namespace) throws KeeperException,
478      IOException {
479    return getQuotaManager().getState(namespace);
480  }
481
482  byte[] getSplitKey(byte[] startKey, byte[] endKey) {
483    String skey = Bytes.toString(startKey);
484    int key;
485    if (StringUtils.isBlank(skey)) {
486      key = Integer.parseInt(Bytes.toString(endKey))/2 ;
487    } else {
488      key = (int) (Integer.parseInt(skey) * 1.5);
489    }
490    return Bytes.toBytes("" + key);
491  }
492
493  public static class CustomObserver implements RegionCoprocessor, RegionObserver {
494    volatile CountDownLatch postCompact;
495
496    @Override
497    public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store,
498        StoreFile resultFile, CompactionLifeCycleTracker tracker, CompactionRequest request)
499        throws IOException {
500      postCompact.countDown();
501    }
502
503    @Override
504    public void start(CoprocessorEnvironment e) throws IOException {
505      postCompact = new CountDownLatch(1);
506    }
507
508    @Override
509    public Optional<RegionObserver> getRegionObserver() {
510      return Optional.of(this);
511    }
512  }
513
514  @Test
515  public void testStatePreserve() throws Exception {
516    final String nsp1 = prefix + "_testStatePreserve";
517    NamespaceDescriptor nspDesc = NamespaceDescriptor.create(nsp1)
518        .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
519        .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "10").build();
520    ADMIN.createNamespace(nspDesc);
521    TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
522    TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
523    TableName tableThree = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table3");
524    HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
525    HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
526    tableDescOne.addFamily(fam1);
527    HTableDescriptor tableDescTwo = new HTableDescriptor(tableTwo);
528    tableDescTwo.addFamily(fam1);
529    HTableDescriptor tableDescThree = new HTableDescriptor(tableThree);
530    tableDescThree.addFamily(fam1);
531    ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
532    ADMIN.createTable(tableDescTwo, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
533    ADMIN.createTable(tableDescThree, Bytes.toBytes("1"), Bytes.toBytes("1000"), 4);
534    ADMIN.disableTable(tableThree);
535    deleteTable(tableThree);
536    // wait for chore to complete
537    UTIL.waitFor(1000, new Waiter.Predicate<Exception>() {
538      @Override
539      public boolean evaluate() throws Exception {
540       return (getNamespaceState(nsp1).getTables().size() == 2);
541      }
542    });
543    NamespaceTableAndRegionInfo before = getNamespaceState(nsp1);
544    restartMaster();
545    NamespaceTableAndRegionInfo after = getNamespaceState(nsp1);
546    assertEquals("Expected: " + before.getTables() + " Found: " + after.getTables(), before
547        .getTables().size(), after.getTables().size());
548  }
549
550  public static void waitForQuotaInitialize(final HBaseTestingUtility util) throws Exception {
551    util.waitFor(60000, new Waiter.Predicate<Exception>() {
552      @Override
553      public boolean evaluate() throws Exception {
554        HMaster master = util.getHBaseCluster().getMaster();
555        if (master == null) {
556          return false;
557        }
558        MasterQuotaManager quotaManager = master.getMasterQuotaManager();
559        return quotaManager != null && quotaManager.isQuotaInitialized();
560      }
561    });
562  }
563
564  private void restartMaster() throws Exception {
565    UTIL.getHBaseCluster().getMaster(0).stop("Stopping to start again");
566    UTIL.getHBaseCluster().waitOnMaster(0);
567    UTIL.getHBaseCluster().startMaster();
568    waitForQuotaInitialize(UTIL);
569  }
570
571  private NamespaceAuditor getQuotaManager() {
572    return UTIL.getHBaseCluster().getMaster()
573        .getMasterQuotaManager().getNamespaceQuotaManager();
574  }
575
576  public static class MasterSyncObserver implements MasterCoprocessor, MasterObserver {
577    volatile CountDownLatch tableDeletionLatch;
578    static boolean throwExceptionInPreCreateTableAction;
579
580    @Override
581    public Optional<MasterObserver> getMasterObserver() {
582      return Optional.of(this);
583    }
584
585    @Override
586    public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
587        TableName tableName) throws IOException {
588      tableDeletionLatch = new CountDownLatch(1);
589    }
590
591    @Override
592    public void postCompletedDeleteTableAction(
593        final ObserverContext<MasterCoprocessorEnvironment> ctx,
594        final TableName tableName) throws IOException {
595      tableDeletionLatch.countDown();
596    }
597
598    @Override
599    public void preCreateTableAction(ObserverContext<MasterCoprocessorEnvironment> ctx,
600        TableDescriptor desc, RegionInfo[] regions) throws IOException {
601      if (throwExceptionInPreCreateTableAction) {
602        throw new IOException("Throw exception as it is demanded.");
603      }
604    }
605  }
606
607  private void deleteTable(final TableName tableName) throws Exception {
608    // NOTE: We need a latch because admin is not sync,
609    // so the postOp coprocessor method may be called after the admin operation returned.
610    MasterSyncObserver observer = UTIL.getHBaseCluster().getMaster()
611      .getMasterCoprocessorHost().findCoprocessor(MasterSyncObserver.class);
612    ADMIN.deleteTable(tableName);
613    observer.tableDeletionLatch.await();
614  }
615
616  @Test(expected = QuotaExceededException.class)
617  public void testExceedTableQuotaInNamespace() throws Exception {
618    String nsp = prefix + "_testExceedTableQuotaInNamespace";
619    NamespaceDescriptor nspDesc =
620        NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
621            .build();
622    ADMIN.createNamespace(nspDesc);
623    assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
624    assertEquals(3, ADMIN.listNamespaceDescriptors().length);
625    HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
626    HTableDescriptor tableDescOne =
627        new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
628    tableDescOne.addFamily(fam1);
629    HTableDescriptor tableDescTwo =
630        new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
631    tableDescTwo.addFamily(fam1);
632    ADMIN.createTable(tableDescOne);
633    ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
634  }
635
636  @Test(expected = QuotaExceededException.class)
637  public void testCloneSnapshotQuotaExceed() throws Exception {
638    String nsp = prefix + "_testTableQuotaExceedWithCloneSnapshot";
639    NamespaceDescriptor nspDesc =
640        NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
641            .build();
642    ADMIN.createNamespace(nspDesc);
643    assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
644    TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
645    TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
646    HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
647    HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
648    tableDescOne.addFamily(fam1);
649    ADMIN.createTable(tableDescOne);
650    String snapshot = "snapshot_testTableQuotaExceedWithCloneSnapshot";
651    ADMIN.snapshot(snapshot, tableName);
652    ADMIN.cloneSnapshot(snapshot, cloneTableName);
653    ADMIN.deleteSnapshot(snapshot);
654  }
655
656  @Test
657  public void testCloneSnapshot() throws Exception {
658    String nsp = prefix + "_testCloneSnapshot";
659    NamespaceDescriptor nspDesc =
660        NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2")
661            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20").build();
662    ADMIN.createNamespace(nspDesc);
663    assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
664    TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
665    TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
666
667    HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
668    HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
669    tableDescOne.addFamily(fam1);
670
671    ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
672    String snapshot = "snapshot_testCloneSnapshot";
673    ADMIN.snapshot(snapshot, tableName);
674    ADMIN.cloneSnapshot(snapshot, cloneTableName);
675
676    int tableLength;
677    try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(tableName)) {
678      tableLength = locator.getStartKeys().length;
679    }
680    assertEquals(tableName.getNameAsString() + " should have four regions.", 4, tableLength);
681
682    try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(cloneTableName)) {
683      tableLength = locator.getStartKeys().length;
684    }
685    assertEquals(cloneTableName.getNameAsString() + " should have four regions.", 4, tableLength);
686
687    NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
688    assertEquals("Total tables count should be 2.", 2, nstate.getTables().size());
689    assertEquals("Total regions count should be.", 8, nstate.getRegionCount());
690
691    ADMIN.deleteSnapshot(snapshot);
692  }
693
694  @Test
695  public void testRestoreSnapshot() throws Exception {
696    String nsp = prefix + "_testRestoreSnapshot";
697    NamespaceDescriptor nspDesc =
698        NamespaceDescriptor.create(nsp)
699            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
700    ADMIN.createNamespace(nspDesc);
701    assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
702    TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
703    HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
704    HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
705    tableDescOne.addFamily(fam1);
706    ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
707
708    NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
709    assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
710
711    String snapshot = "snapshot_testRestoreSnapshot";
712    ADMIN.snapshot(snapshot, tableName1);
713
714    List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1);
715    Collections.sort(regions);
716
717    ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
718    Thread.sleep(2000);
719    assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());
720
721    ADMIN.disableTable(tableName1);
722    ADMIN.restoreSnapshot(snapshot);
723
724    assertEquals("Total regions count should be 4 after restore.", 4, nstate.getRegionCount());
725
726    ADMIN.enableTable(tableName1);
727    ADMIN.deleteSnapshot(snapshot);
728  }
729
730  @Test
731  public void testRestoreSnapshotQuotaExceed() throws Exception {
732    String nsp = prefix + "_testRestoreSnapshotQuotaExceed";
733    NamespaceDescriptor nspDesc =
734        NamespaceDescriptor.create(nsp)
735            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
736    ADMIN.createNamespace(nspDesc);
737    NamespaceDescriptor ndesc = ADMIN.getNamespaceDescriptor(nsp);
738    assertNotNull("Namespace descriptor found null.", ndesc);
739    TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
740    HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
741    HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
742    tableDescOne.addFamily(fam1);
743
744    ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
745
746    NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
747    assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
748
749    String snapshot = "snapshot_testRestoreSnapshotQuotaExceed";
750    // snapshot has 4 regions
751    ADMIN.snapshot(snapshot, tableName1);
752    // recreate table with 1 region and set max regions to 3 for namespace
753    ADMIN.disableTable(tableName1);
754    ADMIN.deleteTable(tableName1);
755    ADMIN.createTable(tableDescOne);
756    ndesc.setConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "3");
757    ADMIN.modifyNamespace(ndesc);
758
759    ADMIN.disableTable(tableName1);
760    try {
761      ADMIN.restoreSnapshot(snapshot);
762      fail("Region quota is exceeded so QuotaExceededException should be thrown but HBaseAdmin"
763          + " wraps IOException into RestoreSnapshotException");
764    } catch (RestoreSnapshotException ignore) {
765      assertTrue(ignore.getCause() instanceof QuotaExceededException);
766    }
767    assertEquals(1, getNamespaceState(nsp).getRegionCount());
768    ADMIN.enableTable(tableName1);
769    ADMIN.deleteSnapshot(snapshot);
770  }
771}