Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.quotas;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023
024import java.io.IOException;
025import java.util.ArrayList;
026import java.util.HashMap;
027import java.util.HashSet;
028import java.util.List;
029import java.util.Map;
030import java.util.Set;
031import java.util.concurrent.TimeUnit;
032
033import org.apache.hadoop.hbase.Cell;
034import org.apache.hadoop.hbase.CellScanner;
035import org.apache.hadoop.hbase.HBaseClassTestRule;
036import org.apache.hadoop.hbase.HBaseTestingUtility;
037import org.apache.hadoop.hbase.HConstants;
038import org.apache.hadoop.hbase.NamespaceDescriptor;
039import org.apache.hadoop.hbase.TableName;
040import org.apache.hadoop.hbase.client.Connection;
041import org.apache.hadoop.hbase.client.ConnectionFactory;
042import org.apache.hadoop.hbase.client.Delete;
043import org.apache.hadoop.hbase.client.Put;
044import org.apache.hadoop.hbase.client.Result;
045import org.apache.hadoop.hbase.client.ResultScanner;
046import org.apache.hadoop.hbase.client.Table;
047import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
048import org.apache.hadoop.hbase.testclassification.MasterTests;
049import org.apache.hadoop.hbase.testclassification.MediumTests;
050import org.junit.After;
051import org.junit.AfterClass;
052import org.junit.Before;
053import org.junit.BeforeClass;
054import org.junit.ClassRule;
055import org.junit.Rule;
056import org.junit.Test;
057import org.junit.experimental.categories.Category;
058import org.junit.rules.TestName;
059
060import org.apache.hbase.thirdparty.com.google.common.collect.HashMultimap;
061import org.apache.hbase.thirdparty.com.google.common.collect.Multimap;
062import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
063
064import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
065import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos;
066import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
067import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Throttle;
068
069/**
070 * Test the quota table helpers (e.g. CRUD operations)
071 */
072@Category({MasterTests.class, MediumTests.class})
073public class TestQuotaTableUtil {
074
075  @ClassRule
076  public static final HBaseClassTestRule CLASS_RULE =
077      HBaseClassTestRule.forClass(TestQuotaTableUtil.class);
078
079  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
080  private Connection connection;
081  private int tableNameCounter;
082
083  @Rule
084  public TestName testName = new TestName();
085
086  @Rule
087  public TestName name = new TestName();
088
089  @BeforeClass
090  public static void setUpBeforeClass() throws Exception {
091    TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
092    TEST_UTIL.getConfiguration().setInt(QuotaCache.REFRESH_CONF_KEY, 2000);
093    TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
094    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
095    TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
096    TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
097    TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
098    TEST_UTIL.startMiniCluster(1);
099    TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME);
100  }
101
102  @AfterClass
103  public static void tearDownAfterClass() throws Exception {
104    TEST_UTIL.shutdownMiniCluster();
105  }
106
107  @Before
108  public void before() throws IOException {
109    this.connection = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
110    this.tableNameCounter = 0;
111  }
112
113  @After
114  public void after() throws IOException {
115    this.connection.close();
116  }
117
118  @Test
119  public void testDeleteSnapshots() throws Exception {
120    TableName tn = TableName.valueOf(name.getMethodName());
121    try (Table t = connection.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
122      Quotas quota = Quotas.newBuilder().setSpace(
123          QuotaProtos.SpaceQuota.newBuilder().setSoftLimit(7L)
124              .setViolationPolicy(QuotaProtos.SpaceViolationPolicy.NO_WRITES).build()).build();
125      QuotaUtil.addTableQuota(connection, tn, quota);
126
127      String snapshotName = name.getMethodName() + "_snapshot";
128      t.put(QuotaTableUtil.createPutForSnapshotSize(tn, snapshotName, 3L));
129      t.put(QuotaTableUtil.createPutForSnapshotSize(tn, snapshotName, 5L));
130      assertEquals(1, QuotaTableUtil.getObservedSnapshotSizes(connection).size());
131
132      List<Delete> deletes = QuotaTableUtil.createDeletesForExistingTableSnapshotSizes(connection);
133      assertEquals(1, deletes.size());
134
135      t.delete(deletes);
136      assertEquals(0, QuotaTableUtil.getObservedSnapshotSizes(connection).size());
137
138      String ns = name.getMethodName();
139      t.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns, 5L));
140      t.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns, 3L));
141      assertEquals(3L, QuotaTableUtil.getNamespaceSnapshotSize(connection, ns));
142
143      deletes = QuotaTableUtil.createDeletesForExistingNamespaceSnapshotSizes(connection);
144      assertEquals(1, deletes.size());
145
146      t.delete(deletes);
147      assertEquals(0L, QuotaTableUtil.getNamespaceSnapshotSize(connection, ns));
148
149      t.put(QuotaTableUtil.createPutForSnapshotSize(TableName.valueOf("t1"), "s1", 3L));
150      t.put(QuotaTableUtil.createPutForSnapshotSize(TableName.valueOf("t2"), "s2", 3L));
151      t.put(QuotaTableUtil.createPutForSnapshotSize(TableName.valueOf("t3"), "s3", 3L));
152      t.put(QuotaTableUtil.createPutForSnapshotSize(TableName.valueOf("t4"), "s4", 3L));
153      t.put(QuotaTableUtil.createPutForSnapshotSize(TableName.valueOf("t1"), "s5", 3L));
154
155      t.put(QuotaTableUtil.createPutForNamespaceSnapshotSize("ns1", 3L));
156      t.put(QuotaTableUtil.createPutForNamespaceSnapshotSize("ns2", 3L));
157      t.put(QuotaTableUtil.createPutForNamespaceSnapshotSize("ns3", 3L));
158
159      assertEquals(5,QuotaTableUtil.getTableSnapshots(connection).size());
160      assertEquals(3,QuotaTableUtil.getNamespaceSnapshots(connection).size());
161
162      Multimap<TableName, String> tableSnapshotEntriesToRemove = HashMultimap.create();
163      tableSnapshotEntriesToRemove.put(TableName.valueOf("t1"), "s1");
164      tableSnapshotEntriesToRemove.put(TableName.valueOf("t3"), "s3");
165      tableSnapshotEntriesToRemove.put(TableName.valueOf("t4"), "s4");
166
167      Set<String> namespaceSnapshotEntriesToRemove = new HashSet<>();
168      namespaceSnapshotEntriesToRemove.add("ns2");
169      namespaceSnapshotEntriesToRemove.add("ns1");
170
171      deletes =
172          QuotaTableUtil.createDeletesForExistingTableSnapshotSizes(tableSnapshotEntriesToRemove);
173      assertEquals(3, deletes.size());
174      deletes = QuotaTableUtil
175          .createDeletesForExistingNamespaceSnapshotSizes(namespaceSnapshotEntriesToRemove);
176      assertEquals(2, deletes.size());
177    }
178  }
179
180  @Test
181  public void testTableQuotaUtil() throws Exception {
182    final TableName tableName = TableName.valueOf(name.getMethodName());
183
184    Quotas quota = Quotas.newBuilder()
185              .setThrottle(Throttle.newBuilder()
186                .setReqNum(ProtobufUtil.toTimedQuota(1000, TimeUnit.SECONDS, QuotaScope.MACHINE))
187                .setWriteNum(ProtobufUtil.toTimedQuota(600, TimeUnit.SECONDS, QuotaScope.MACHINE))
188                .setReadSize(ProtobufUtil.toTimedQuota(8192, TimeUnit.SECONDS, QuotaScope.MACHINE))
189                .build())
190              .build();
191
192    // Add user quota and verify it
193    QuotaUtil.addTableQuota(this.connection, tableName, quota);
194    Quotas resQuota = QuotaUtil.getTableQuota(this.connection, tableName);
195    assertEquals(quota, resQuota);
196
197    // Remove user quota and verify it
198    QuotaUtil.deleteTableQuota(this.connection, tableName);
199    resQuota = QuotaUtil.getTableQuota(this.connection, tableName);
200    assertEquals(null, resQuota);
201  }
202
203  @Test
204  public void testNamespaceQuotaUtil() throws Exception {
205    final String namespace = "testNamespaceQuotaUtilNS";
206
207    Quotas quota = Quotas.newBuilder()
208              .setThrottle(Throttle.newBuilder()
209                .setReqNum(ProtobufUtil.toTimedQuota(1000, TimeUnit.SECONDS, QuotaScope.MACHINE))
210                .setWriteNum(ProtobufUtil.toTimedQuota(600, TimeUnit.SECONDS, QuotaScope.MACHINE))
211                .setReadSize(ProtobufUtil.toTimedQuota(8192, TimeUnit.SECONDS, QuotaScope.MACHINE))
212                .build())
213              .build();
214
215    // Add user quota and verify it
216    QuotaUtil.addNamespaceQuota(this.connection, namespace, quota);
217    Quotas resQuota = QuotaUtil.getNamespaceQuota(this.connection, namespace);
218    assertEquals(quota, resQuota);
219
220    // Remove user quota and verify it
221    QuotaUtil.deleteNamespaceQuota(this.connection, namespace);
222    resQuota = QuotaUtil.getNamespaceQuota(this.connection, namespace);
223    assertEquals(null, resQuota);
224  }
225
226  @Test
227  public void testUserQuotaUtil() throws Exception {
228    final TableName tableName = TableName.valueOf(name.getMethodName());
229    final String namespace = "testNS";
230    final String user = "testUser";
231
232    Quotas quotaNamespace = Quotas.newBuilder()
233              .setThrottle(Throttle.newBuilder()
234                .setReqNum(ProtobufUtil.toTimedQuota(50000, TimeUnit.SECONDS, QuotaScope.MACHINE))
235                .build())
236              .build();
237    Quotas quotaTable = Quotas.newBuilder()
238              .setThrottle(Throttle.newBuilder()
239                .setReqNum(ProtobufUtil.toTimedQuota(1000, TimeUnit.SECONDS, QuotaScope.MACHINE))
240                .setWriteNum(ProtobufUtil.toTimedQuota(600, TimeUnit.SECONDS, QuotaScope.MACHINE))
241                .setReadSize(ProtobufUtil.toTimedQuota(10000, TimeUnit.SECONDS, QuotaScope.MACHINE))
242                .build())
243              .build();
244    Quotas quota = Quotas.newBuilder()
245              .setThrottle(Throttle.newBuilder()
246                .setReqSize(ProtobufUtil.toTimedQuota(8192, TimeUnit.SECONDS, QuotaScope.MACHINE))
247                .setWriteSize(ProtobufUtil.toTimedQuota(4096, TimeUnit.SECONDS, QuotaScope.MACHINE))
248                .setReadNum(ProtobufUtil.toTimedQuota(1000, TimeUnit.SECONDS, QuotaScope.MACHINE))
249                .build())
250              .build();
251
252    // Add user global quota
253    QuotaUtil.addUserQuota(this.connection, user, quota);
254    Quotas resQuota = QuotaUtil.getUserQuota(this.connection, user);
255    assertEquals(quota, resQuota);
256
257    // Add user quota for table
258    QuotaUtil.addUserQuota(this.connection, user, tableName, quotaTable);
259    Quotas resQuotaTable = QuotaUtil.getUserQuota(this.connection, user, tableName);
260    assertEquals(quotaTable, resQuotaTable);
261
262    // Add user quota for namespace
263    QuotaUtil.addUserQuota(this.connection, user, namespace, quotaNamespace);
264    Quotas resQuotaNS = QuotaUtil.getUserQuota(this.connection, user, namespace);
265    assertEquals(quotaNamespace, resQuotaNS);
266
267    // Delete user global quota
268    QuotaUtil.deleteUserQuota(this.connection, user);
269    resQuota = QuotaUtil.getUserQuota(this.connection, user);
270    assertEquals(null, resQuota);
271
272    // Delete user quota for table
273    QuotaUtil.deleteUserQuota(this.connection, user, tableName);
274    resQuotaTable = QuotaUtil.getUserQuota(this.connection, user, tableName);
275    assertEquals(null, resQuotaTable);
276
277    // Delete user quota for namespace
278    QuotaUtil.deleteUserQuota(this.connection, user, namespace);
279    resQuotaNS = QuotaUtil.getUserQuota(this.connection, user, namespace);
280    assertEquals(null, resQuotaNS);
281  }
282
283  @Test
284  public void testSerDeViolationPolicies() throws Exception {
285    final TableName tn1 = getUniqueTableName();
286    final SpaceQuotaSnapshot snapshot1 = new SpaceQuotaSnapshot(
287        new SpaceQuotaStatus(SpaceViolationPolicy.DISABLE), 512L, 1024L);
288    final TableName tn2 = getUniqueTableName();
289    final SpaceQuotaSnapshot snapshot2 = new SpaceQuotaSnapshot(
290        new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 512L, 1024L);
291    final TableName tn3 = getUniqueTableName();
292    final SpaceQuotaSnapshot snapshot3 = new SpaceQuotaSnapshot(
293        new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES), 512L, 1024L);
294    List<Put> puts = new ArrayList<>();
295    puts.add(QuotaTableUtil.createPutForSpaceSnapshot(tn1, snapshot1));
296    puts.add(QuotaTableUtil.createPutForSpaceSnapshot(tn2, snapshot2));
297    puts.add(QuotaTableUtil.createPutForSpaceSnapshot(tn3, snapshot3));
298    final Map<TableName,SpaceQuotaSnapshot> expectedPolicies = new HashMap<>();
299    expectedPolicies.put(tn1, snapshot1);
300    expectedPolicies.put(tn2, snapshot2);
301    expectedPolicies.put(tn3, snapshot3);
302
303    final Map<TableName,SpaceQuotaSnapshot> actualPolicies = new HashMap<>();
304    try (Table quotaTable = connection.getTable(QuotaUtil.QUOTA_TABLE_NAME)) {
305      quotaTable.put(puts);
306      ResultScanner scanner = quotaTable.getScanner(QuotaTableUtil.makeQuotaSnapshotScan());
307      for (Result r : scanner) {
308        QuotaTableUtil.extractQuotaSnapshot(r, actualPolicies);
309      }
310      scanner.close();
311    }
312
313    assertEquals(expectedPolicies, actualPolicies);
314  }
315
316  @Test
317  public void testSerdeTableSnapshotSizes() throws Exception {
318    TableName tn1 = TableName.valueOf("tn1");
319    TableName tn2 = TableName.valueOf("tn2");
320    try (Table quotaTable = connection.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
321      for (int i = 0; i < 3; i++) {
322        Put p = QuotaTableUtil.createPutForSnapshotSize(tn1, "tn1snap" + i, 1024L * (1+i));
323        quotaTable.put(p);
324      }
325      for (int i = 0; i < 3; i++) {
326        Put p = QuotaTableUtil.createPutForSnapshotSize(tn2, "tn2snap" + i, 2048L * (1+i));
327        quotaTable.put(p);
328      }
329
330      verifyTableSnapshotSize(quotaTable, tn1, "tn1snap0", 1024L);
331      verifyTableSnapshotSize(quotaTable, tn1, "tn1snap1", 2048L);
332      verifyTableSnapshotSize(quotaTable, tn1, "tn1snap2", 3072L);
333
334      verifyTableSnapshotSize(quotaTable, tn2, "tn2snap0", 2048L);
335      verifyTableSnapshotSize(quotaTable, tn2, "tn2snap1", 4096L);
336      verifyTableSnapshotSize(quotaTable, tn2, "tn2snap2", 6144L);
337
338      cleanUpSnapshotSizes();
339    }
340  }
341
342  @Test
343  public void testReadNamespaceSnapshotSizes() throws Exception {
344    String ns1 = "ns1";
345    String ns2 = "ns2";
346    String defaultNs = NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR;
347    try (Table quotaTable = connection.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
348      quotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns1, 1024L));
349      quotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns2, 2048L));
350      quotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(defaultNs, 8192L));
351
352      assertEquals(1024L, QuotaTableUtil.getNamespaceSnapshotSize(connection, ns1));
353      assertEquals(2048L, QuotaTableUtil.getNamespaceSnapshotSize(connection, ns2));
354      assertEquals(8192L, QuotaTableUtil.getNamespaceSnapshotSize(connection, defaultNs));
355
356      cleanUpSnapshotSizes();
357    }
358  }
359
360  private TableName getUniqueTableName() {
361    return TableName.valueOf(testName.getMethodName() + "_" + tableNameCounter++);
362  }
363
364  private void verifyTableSnapshotSize(
365      Table quotaTable, TableName tn, String snapshotName, long expectedSize) throws IOException {
366    Result r = quotaTable.get(QuotaTableUtil.makeGetForSnapshotSize(tn, snapshotName));
367    CellScanner cs = r.cellScanner();
368    assertTrue(cs.advance());
369    Cell c = cs.current();
370    assertEquals(expectedSize, QuotaProtos.SpaceQuotaSnapshot.parseFrom(
371        UnsafeByteOperations.unsafeWrap(
372            c.getValueArray(), c.getValueOffset(), c.getValueLength())).getQuotaUsage());
373    assertFalse(cs.advance());
374  }
375
376  private void cleanUpSnapshotSizes() throws IOException {
377    try (Table t = connection.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
378      QuotaTableUtil.createDeletesForExistingTableSnapshotSizes(connection);
379      List<Delete> deletes =
380          QuotaTableUtil.createDeletesForExistingNamespaceSnapshotSizes(connection);
381      deletes.addAll(QuotaTableUtil.createDeletesForExistingTableSnapshotSizes(connection));
382      t.delete(deletes);
383    }
384  }
385}