Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.quotas;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022import static org.junit.jupiter.api.Assertions.assertTrue;
023
024import java.io.IOException;
025import java.util.ArrayList;
026import java.util.HashMap;
027import java.util.HashSet;
028import java.util.List;
029import java.util.Map;
030import java.util.Set;
031import java.util.concurrent.TimeUnit;
032import org.apache.hadoop.hbase.Cell;
033import org.apache.hadoop.hbase.CellScanner;
034import org.apache.hadoop.hbase.HBaseTestingUtil;
035import org.apache.hadoop.hbase.HConstants;
036import org.apache.hadoop.hbase.NamespaceDescriptor;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.client.Connection;
039import org.apache.hadoop.hbase.client.ConnectionFactory;
040import org.apache.hadoop.hbase.client.Delete;
041import org.apache.hadoop.hbase.client.Put;
042import org.apache.hadoop.hbase.client.Result;
043import org.apache.hadoop.hbase.client.ResultScanner;
044import org.apache.hadoop.hbase.client.Table;
045import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
046import org.apache.hadoop.hbase.testclassification.MasterTests;
047import org.apache.hadoop.hbase.testclassification.MediumTests;
048import org.junit.jupiter.api.AfterAll;
049import org.junit.jupiter.api.AfterEach;
050import org.junit.jupiter.api.BeforeAll;
051import org.junit.jupiter.api.BeforeEach;
052import org.junit.jupiter.api.Tag;
053import org.junit.jupiter.api.Test;
054import org.junit.jupiter.api.TestInfo;
055
056import org.apache.hbase.thirdparty.com.google.common.collect.HashMultimap;
057import org.apache.hbase.thirdparty.com.google.common.collect.Multimap;
058import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
059
060import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
061import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos;
062import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
063import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Throttle;
064
065/**
066 * Test the quota table helpers (e.g. CRUD operations)
067 */
068@Tag(MasterTests.TAG)
069@Tag(MediumTests.TAG)
070public class TestQuotaTableUtil {
071
072  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
073  private Connection connection;
074  private int tableNameCounter;
075
076  @BeforeAll
077  public static void setUpBeforeClass() throws Exception {
078    TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
079    TEST_UTIL.getConfiguration().setInt(QuotaCache.REFRESH_CONF_KEY, 2000);
080    TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
081    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
082    TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
083    TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
084    TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
085    TEST_UTIL.startMiniCluster(1);
086    TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME);
087  }
088
089  @AfterAll
090  public static void tearDownAfterClass() throws Exception {
091    TEST_UTIL.shutdownMiniCluster();
092  }
093
094  @BeforeEach
095  public void before() throws IOException {
096    this.connection = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
097    this.tableNameCounter = 0;
098  }
099
100  @AfterEach
101  public void after() throws IOException {
102    this.connection.close();
103  }
104
105  @Test
106  public void testDeleteSnapshots(TestInfo testInfo) throws Exception {
107    String methodName = testInfo.getTestMethod().get().getName();
108    TableName tn = TableName.valueOf(methodName);
109    try (Table t = connection.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
110      Quotas quota =
111        Quotas.newBuilder().setSpace(QuotaProtos.SpaceQuota.newBuilder().setSoftLimit(7L)
112          .setViolationPolicy(QuotaProtos.SpaceViolationPolicy.NO_WRITES).build()).build();
113      QuotaUtil.addTableQuota(connection, tn, quota);
114
115      String snapshotName = methodName + "_snapshot";
116      t.put(QuotaTableUtil.createPutForSnapshotSize(tn, snapshotName, 3L));
117      t.put(QuotaTableUtil.createPutForSnapshotSize(tn, snapshotName, 5L));
118      assertEquals(1, QuotaTableUtil.getObservedSnapshotSizes(connection).size());
119
120      List<Delete> deletes = QuotaTableUtil.createDeletesForExistingTableSnapshotSizes(connection);
121      assertEquals(1, deletes.size());
122
123      t.delete(deletes);
124      assertEquals(0, QuotaTableUtil.getObservedSnapshotSizes(connection).size());
125
126      String ns = methodName;
127      t.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns, 5L));
128      t.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns, 3L));
129      assertEquals(3L, QuotaTableUtil.getNamespaceSnapshotSize(connection, ns));
130
131      deletes = QuotaTableUtil.createDeletesForExistingNamespaceSnapshotSizes(connection);
132      assertEquals(1, deletes.size());
133
134      t.delete(deletes);
135      assertEquals(0L, QuotaTableUtil.getNamespaceSnapshotSize(connection, ns));
136
137      t.put(QuotaTableUtil.createPutForSnapshotSize(TableName.valueOf("t1"), "s1", 3L));
138      t.put(QuotaTableUtil.createPutForSnapshotSize(TableName.valueOf("t2"), "s2", 3L));
139      t.put(QuotaTableUtil.createPutForSnapshotSize(TableName.valueOf("t3"), "s3", 3L));
140      t.put(QuotaTableUtil.createPutForSnapshotSize(TableName.valueOf("t4"), "s4", 3L));
141      t.put(QuotaTableUtil.createPutForSnapshotSize(TableName.valueOf("t1"), "s5", 3L));
142
143      t.put(QuotaTableUtil.createPutForNamespaceSnapshotSize("ns1", 3L));
144      t.put(QuotaTableUtil.createPutForNamespaceSnapshotSize("ns2", 3L));
145      t.put(QuotaTableUtil.createPutForNamespaceSnapshotSize("ns3", 3L));
146
147      assertEquals(5, QuotaTableUtil.getTableSnapshots(connection).size());
148      assertEquals(3, QuotaTableUtil.getNamespaceSnapshots(connection).size());
149
150      Multimap<TableName, String> tableSnapshotEntriesToRemove = HashMultimap.create();
151      tableSnapshotEntriesToRemove.put(TableName.valueOf("t1"), "s1");
152      tableSnapshotEntriesToRemove.put(TableName.valueOf("t3"), "s3");
153      tableSnapshotEntriesToRemove.put(TableName.valueOf("t4"), "s4");
154
155      Set<String> namespaceSnapshotEntriesToRemove = new HashSet<>();
156      namespaceSnapshotEntriesToRemove.add("ns2");
157      namespaceSnapshotEntriesToRemove.add("ns1");
158
159      deletes =
160        QuotaTableUtil.createDeletesForExistingTableSnapshotSizes(tableSnapshotEntriesToRemove);
161      assertEquals(3, deletes.size());
162      deletes = QuotaTableUtil
163        .createDeletesForExistingNamespaceSnapshotSizes(namespaceSnapshotEntriesToRemove);
164      assertEquals(2, deletes.size());
165    }
166  }
167
168  @Test
169  public void testTableQuotaUtil(TestInfo testInfo) throws Exception {
170    final TableName tableName = TableName.valueOf(testInfo.getTestMethod().get().getName());
171
172    Quotas quota = Quotas.newBuilder()
173      .setThrottle(Throttle.newBuilder()
174        .setReqNum(ProtobufUtil.toTimedQuota(1000, TimeUnit.SECONDS, QuotaScope.MACHINE))
175        .setWriteNum(ProtobufUtil.toTimedQuota(600, TimeUnit.SECONDS, QuotaScope.MACHINE))
176        .setReadSize(ProtobufUtil.toTimedQuota(8192, TimeUnit.SECONDS, QuotaScope.MACHINE)).build())
177      .build();
178
179    // Add user quota and verify it
180    QuotaUtil.addTableQuota(this.connection, tableName, quota);
181    Quotas resQuota = QuotaUtil.getTableQuota(this.connection, tableName);
182    assertEquals(quota, resQuota);
183
184    // Remove user quota and verify it
185    QuotaUtil.deleteTableQuota(this.connection, tableName);
186    resQuota = QuotaUtil.getTableQuota(this.connection, tableName);
187    assertEquals(null, resQuota);
188  }
189
190  @Test
191  public void testNamespaceQuotaUtil() throws Exception {
192    final String namespace = "testNamespaceQuotaUtilNS";
193
194    Quotas quota = Quotas.newBuilder()
195      .setThrottle(Throttle.newBuilder()
196        .setReqNum(ProtobufUtil.toTimedQuota(1000, TimeUnit.SECONDS, QuotaScope.MACHINE))
197        .setWriteNum(ProtobufUtil.toTimedQuota(600, TimeUnit.SECONDS, QuotaScope.MACHINE))
198        .setReadSize(ProtobufUtil.toTimedQuota(8192, TimeUnit.SECONDS, QuotaScope.MACHINE)).build())
199      .build();
200
201    // Add user quota and verify it
202    QuotaUtil.addNamespaceQuota(this.connection, namespace, quota);
203    Quotas resQuota = QuotaUtil.getNamespaceQuota(this.connection, namespace);
204    assertEquals(quota, resQuota);
205
206    // Remove user quota and verify it
207    QuotaUtil.deleteNamespaceQuota(this.connection, namespace);
208    resQuota = QuotaUtil.getNamespaceQuota(this.connection, namespace);
209    assertEquals(null, resQuota);
210  }
211
212  @Test
213  public void testUserQuotaUtil(TestInfo testInfo) throws Exception {
214    final TableName tableName = TableName.valueOf(testInfo.getTestMethod().get().getName());
215    final String namespace = "testNS";
216    final String user = "testUser";
217
218    Quotas quotaNamespace = Quotas.newBuilder()
219      .setThrottle(Throttle.newBuilder()
220        .setReqNum(ProtobufUtil.toTimedQuota(50000, TimeUnit.SECONDS, QuotaScope.MACHINE)).build())
221      .build();
222    Quotas quotaTable = Quotas.newBuilder().setThrottle(Throttle.newBuilder()
223      .setReqNum(ProtobufUtil.toTimedQuota(1000, TimeUnit.SECONDS, QuotaScope.MACHINE))
224      .setWriteNum(ProtobufUtil.toTimedQuota(600, TimeUnit.SECONDS, QuotaScope.MACHINE))
225      .setReadSize(ProtobufUtil.toTimedQuota(10000, TimeUnit.SECONDS, QuotaScope.MACHINE)).build())
226      .build();
227    Quotas quota = Quotas.newBuilder()
228      .setThrottle(Throttle.newBuilder()
229        .setReqSize(ProtobufUtil.toTimedQuota(8192, TimeUnit.SECONDS, QuotaScope.MACHINE))
230        .setWriteSize(ProtobufUtil.toTimedQuota(4096, TimeUnit.SECONDS, QuotaScope.MACHINE))
231        .setReadNum(ProtobufUtil.toTimedQuota(1000, TimeUnit.SECONDS, QuotaScope.MACHINE)).build())
232      .build();
233
234    // Add user global quota
235    QuotaUtil.addUserQuota(this.connection, user, quota);
236    Quotas resQuota = QuotaUtil.getUserQuota(this.connection, user);
237    assertEquals(quota, resQuota);
238
239    // Add user quota for table
240    QuotaUtil.addUserQuota(this.connection, user, tableName, quotaTable);
241    Quotas resQuotaTable = QuotaUtil.getUserQuota(this.connection, user, tableName);
242    assertEquals(quotaTable, resQuotaTable);
243
244    // Add user quota for namespace
245    QuotaUtil.addUserQuota(this.connection, user, namespace, quotaNamespace);
246    Quotas resQuotaNS = QuotaUtil.getUserQuota(this.connection, user, namespace);
247    assertEquals(quotaNamespace, resQuotaNS);
248
249    // Delete user global quota
250    QuotaUtil.deleteUserQuota(this.connection, user);
251    resQuota = QuotaUtil.getUserQuota(this.connection, user);
252    assertEquals(null, resQuota);
253
254    // Delete user quota for table
255    QuotaUtil.deleteUserQuota(this.connection, user, tableName);
256    resQuotaTable = QuotaUtil.getUserQuota(this.connection, user, tableName);
257    assertEquals(null, resQuotaTable);
258
259    // Delete user quota for namespace
260    QuotaUtil.deleteUserQuota(this.connection, user, namespace);
261    resQuotaNS = QuotaUtil.getUserQuota(this.connection, user, namespace);
262    assertEquals(null, resQuotaNS);
263  }
264
265  @Test
266  public void testSerDeViolationPolicies(TestInfo testInfo) throws Exception {
267    final TableName tn1 = getUniqueTableName(testInfo);
268    final SpaceQuotaSnapshot snapshot1 =
269      new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.DISABLE), 512L, 1024L);
270    final TableName tn2 = getUniqueTableName(testInfo);
271    final SpaceQuotaSnapshot snapshot2 =
272      new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 512L, 1024L);
273    final TableName tn3 = getUniqueTableName(testInfo);
274    final SpaceQuotaSnapshot snapshot3 =
275      new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES), 512L, 1024L);
276    List<Put> puts = new ArrayList<>();
277    puts.add(QuotaTableUtil.createPutForSpaceSnapshot(tn1, snapshot1));
278    puts.add(QuotaTableUtil.createPutForSpaceSnapshot(tn2, snapshot2));
279    puts.add(QuotaTableUtil.createPutForSpaceSnapshot(tn3, snapshot3));
280    final Map<TableName, SpaceQuotaSnapshot> expectedPolicies = new HashMap<>();
281    expectedPolicies.put(tn1, snapshot1);
282    expectedPolicies.put(tn2, snapshot2);
283    expectedPolicies.put(tn3, snapshot3);
284
285    final Map<TableName, SpaceQuotaSnapshot> actualPolicies = new HashMap<>();
286    try (Table quotaTable = connection.getTable(QuotaUtil.QUOTA_TABLE_NAME)) {
287      quotaTable.put(puts);
288      ResultScanner scanner = quotaTable.getScanner(QuotaTableUtil.makeQuotaSnapshotScan());
289      for (Result r : scanner) {
290        QuotaTableUtil.extractQuotaSnapshot(r, actualPolicies);
291      }
292      scanner.close();
293    }
294
295    assertEquals(expectedPolicies, actualPolicies);
296  }
297
298  @Test
299  public void testSerdeTableSnapshotSizes() throws Exception {
300    TableName tn1 = TableName.valueOf("tn1");
301    TableName tn2 = TableName.valueOf("tn2");
302    try (Table quotaTable = connection.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
303      for (int i = 0; i < 3; i++) {
304        Put p = QuotaTableUtil.createPutForSnapshotSize(tn1, "tn1snap" + i, 1024L * (1 + i));
305        quotaTable.put(p);
306      }
307      for (int i = 0; i < 3; i++) {
308        Put p = QuotaTableUtil.createPutForSnapshotSize(tn2, "tn2snap" + i, 2048L * (1 + i));
309        quotaTable.put(p);
310      }
311
312      verifyTableSnapshotSize(quotaTable, tn1, "tn1snap0", 1024L);
313      verifyTableSnapshotSize(quotaTable, tn1, "tn1snap1", 2048L);
314      verifyTableSnapshotSize(quotaTable, tn1, "tn1snap2", 3072L);
315
316      verifyTableSnapshotSize(quotaTable, tn2, "tn2snap0", 2048L);
317      verifyTableSnapshotSize(quotaTable, tn2, "tn2snap1", 4096L);
318      verifyTableSnapshotSize(quotaTable, tn2, "tn2snap2", 6144L);
319
320      cleanUpSnapshotSizes();
321    }
322  }
323
324  @Test
325  public void testReadNamespaceSnapshotSizes() throws Exception {
326    String ns1 = "ns1";
327    String ns2 = "ns2";
328    String defaultNs = NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR;
329    try (Table quotaTable = connection.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
330      quotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns1, 1024L));
331      quotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns2, 2048L));
332      quotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(defaultNs, 8192L));
333
334      assertEquals(1024L, QuotaTableUtil.getNamespaceSnapshotSize(connection, ns1));
335      assertEquals(2048L, QuotaTableUtil.getNamespaceSnapshotSize(connection, ns2));
336      assertEquals(8192L, QuotaTableUtil.getNamespaceSnapshotSize(connection, defaultNs));
337
338      cleanUpSnapshotSizes();
339    }
340  }
341
342  private TableName getUniqueTableName(TestInfo testInfo) {
343    return TableName.valueOf(testInfo.getTestMethod().get().getName() + "_" + tableNameCounter++);
344  }
345
346  private void verifyTableSnapshotSize(Table quotaTable, TableName tn, String snapshotName,
347    long expectedSize) throws IOException {
348    Result r = quotaTable.get(QuotaTableUtil.makeGetForSnapshotSize(tn, snapshotName));
349    CellScanner cs = r.cellScanner();
350    assertTrue(cs.advance());
351    Cell c = cs.current();
352    assertEquals(expectedSize,
353      QuotaProtos.SpaceQuotaSnapshot.parseFrom(
354        UnsafeByteOperations.unsafeWrap(c.getValueArray(), c.getValueOffset(), c.getValueLength()))
355        .getQuotaUsage());
356    assertFalse(cs.advance());
357  }
358
359  private void cleanUpSnapshotSizes() throws IOException {
360    try (Table t = connection.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
361      QuotaTableUtil.createDeletesForExistingTableSnapshotSizes(connection);
362      List<Delete> deletes =
363        QuotaTableUtil.createDeletesForExistingNamespaceSnapshotSizes(connection);
364      deletes.addAll(QuotaTableUtil.createDeletesForExistingTableSnapshotSizes(connection));
365      t.delete(deletes);
366    }
367  }
368}