Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.quotas;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023
024import java.io.IOException;
025import java.util.ArrayList;
026import java.util.HashMap;
027import java.util.List;
028import java.util.Map;
029import java.util.concurrent.TimeUnit;
030import org.apache.hadoop.hbase.Cell;
031import org.apache.hadoop.hbase.CellScanner;
032import org.apache.hadoop.hbase.HBaseClassTestRule;
033import org.apache.hadoop.hbase.HBaseTestingUtility;
034import org.apache.hadoop.hbase.HConstants;
035import org.apache.hadoop.hbase.NamespaceDescriptor;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.client.Connection;
038import org.apache.hadoop.hbase.client.ConnectionFactory;
039import org.apache.hadoop.hbase.client.Put;
040import org.apache.hadoop.hbase.client.Result;
041import org.apache.hadoop.hbase.client.ResultScanner;
042import org.apache.hadoop.hbase.client.Table;
043import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
044import org.apache.hadoop.hbase.testclassification.MasterTests;
045import org.apache.hadoop.hbase.testclassification.MediumTests;
046import org.junit.After;
047import org.junit.AfterClass;
048import org.junit.Before;
049import org.junit.BeforeClass;
050import org.junit.ClassRule;
051import org.junit.Rule;
052import org.junit.Test;
053import org.junit.experimental.categories.Category;
054import org.junit.rules.TestName;
055
056import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
057
058import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
059import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos;
060import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
061import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Throttle;
062
063/**
064 * Test the quota table helpers (e.g. CRUD operations)
065 */
066@Category({MasterTests.class, MediumTests.class})
067public class TestQuotaTableUtil {
068
069  @ClassRule
070  public static final HBaseClassTestRule CLASS_RULE =
071      HBaseClassTestRule.forClass(TestQuotaTableUtil.class);
072
073  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
074  private Connection connection;
075  private int tableNameCounter;
076
077  @Rule
078  public TestName testName = new TestName();
079
080  @Rule
081  public TestName name = new TestName();
082
083  @BeforeClass
084  public static void setUpBeforeClass() throws Exception {
085    TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
086    TEST_UTIL.getConfiguration().setInt(QuotaCache.REFRESH_CONF_KEY, 2000);
087    TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
088    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
089    TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
090    TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
091    TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
092    TEST_UTIL.startMiniCluster(1);
093    TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME);
094  }
095
096  @AfterClass
097  public static void tearDownAfterClass() throws Exception {
098    TEST_UTIL.shutdownMiniCluster();
099  }
100
101  @Before
102  public void before() throws IOException {
103    this.connection = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
104    this.tableNameCounter = 0;
105  }
106
107  @After
108  public void after() throws IOException {
109    this.connection.close();
110  }
111
112  @Test
113  public void testTableQuotaUtil() throws Exception {
114    final TableName tableName = TableName.valueOf(name.getMethodName());
115
116    Quotas quota = Quotas.newBuilder()
117              .setThrottle(Throttle.newBuilder()
118                .setReqNum(ProtobufUtil.toTimedQuota(1000, TimeUnit.SECONDS, QuotaScope.MACHINE))
119                .setWriteNum(ProtobufUtil.toTimedQuota(600, TimeUnit.SECONDS, QuotaScope.MACHINE))
120                .setReadSize(ProtobufUtil.toTimedQuota(8192, TimeUnit.SECONDS, QuotaScope.MACHINE))
121                .build())
122              .build();
123
124    // Add user quota and verify it
125    QuotaUtil.addTableQuota(this.connection, tableName, quota);
126    Quotas resQuota = QuotaUtil.getTableQuota(this.connection, tableName);
127    assertEquals(quota, resQuota);
128
129    // Remove user quota and verify it
130    QuotaUtil.deleteTableQuota(this.connection, tableName);
131    resQuota = QuotaUtil.getTableQuota(this.connection, tableName);
132    assertEquals(null, resQuota);
133  }
134
135  @Test
136  public void testNamespaceQuotaUtil() throws Exception {
137    final String namespace = "testNamespaceQuotaUtilNS";
138
139    Quotas quota = Quotas.newBuilder()
140              .setThrottle(Throttle.newBuilder()
141                .setReqNum(ProtobufUtil.toTimedQuota(1000, TimeUnit.SECONDS, QuotaScope.MACHINE))
142                .setWriteNum(ProtobufUtil.toTimedQuota(600, TimeUnit.SECONDS, QuotaScope.MACHINE))
143                .setReadSize(ProtobufUtil.toTimedQuota(8192, TimeUnit.SECONDS, QuotaScope.MACHINE))
144                .build())
145              .build();
146
147    // Add user quota and verify it
148    QuotaUtil.addNamespaceQuota(this.connection, namespace, quota);
149    Quotas resQuota = QuotaUtil.getNamespaceQuota(this.connection, namespace);
150    assertEquals(quota, resQuota);
151
152    // Remove user quota and verify it
153    QuotaUtil.deleteNamespaceQuota(this.connection, namespace);
154    resQuota = QuotaUtil.getNamespaceQuota(this.connection, namespace);
155    assertEquals(null, resQuota);
156  }
157
158  @Test
159  public void testUserQuotaUtil() throws Exception {
160    final TableName tableName = TableName.valueOf(name.getMethodName());
161    final String namespace = "testNS";
162    final String user = "testUser";
163
164    Quotas quotaNamespace = Quotas.newBuilder()
165              .setThrottle(Throttle.newBuilder()
166                .setReqNum(ProtobufUtil.toTimedQuota(50000, TimeUnit.SECONDS, QuotaScope.MACHINE))
167                .build())
168              .build();
169    Quotas quotaTable = Quotas.newBuilder()
170              .setThrottle(Throttle.newBuilder()
171                .setReqNum(ProtobufUtil.toTimedQuota(1000, TimeUnit.SECONDS, QuotaScope.MACHINE))
172                .setWriteNum(ProtobufUtil.toTimedQuota(600, TimeUnit.SECONDS, QuotaScope.MACHINE))
173                .setReadSize(ProtobufUtil.toTimedQuota(10000, TimeUnit.SECONDS, QuotaScope.MACHINE))
174                .build())
175              .build();
176    Quotas quota = Quotas.newBuilder()
177              .setThrottle(Throttle.newBuilder()
178                .setReqSize(ProtobufUtil.toTimedQuota(8192, TimeUnit.SECONDS, QuotaScope.MACHINE))
179                .setWriteSize(ProtobufUtil.toTimedQuota(4096, TimeUnit.SECONDS, QuotaScope.MACHINE))
180                .setReadNum(ProtobufUtil.toTimedQuota(1000, TimeUnit.SECONDS, QuotaScope.MACHINE))
181                .build())
182              .build();
183
184    // Add user global quota
185    QuotaUtil.addUserQuota(this.connection, user, quota);
186    Quotas resQuota = QuotaUtil.getUserQuota(this.connection, user);
187    assertEquals(quota, resQuota);
188
189    // Add user quota for table
190    QuotaUtil.addUserQuota(this.connection, user, tableName, quotaTable);
191    Quotas resQuotaTable = QuotaUtil.getUserQuota(this.connection, user, tableName);
192    assertEquals(quotaTable, resQuotaTable);
193
194    // Add user quota for namespace
195    QuotaUtil.addUserQuota(this.connection, user, namespace, quotaNamespace);
196    Quotas resQuotaNS = QuotaUtil.getUserQuota(this.connection, user, namespace);
197    assertEquals(quotaNamespace, resQuotaNS);
198
199    // Delete user global quota
200    QuotaUtil.deleteUserQuota(this.connection, user);
201    resQuota = QuotaUtil.getUserQuota(this.connection, user);
202    assertEquals(null, resQuota);
203
204    // Delete user quota for table
205    QuotaUtil.deleteUserQuota(this.connection, user, tableName);
206    resQuotaTable = QuotaUtil.getUserQuota(this.connection, user, tableName);
207    assertEquals(null, resQuotaTable);
208
209    // Delete user quota for namespace
210    QuotaUtil.deleteUserQuota(this.connection, user, namespace);
211    resQuotaNS = QuotaUtil.getUserQuota(this.connection, user, namespace);
212    assertEquals(null, resQuotaNS);
213  }
214
215  @Test
216  public void testSerDeViolationPolicies() throws Exception {
217    final TableName tn1 = getUniqueTableName();
218    final SpaceQuotaSnapshot snapshot1 = new SpaceQuotaSnapshot(
219        new SpaceQuotaStatus(SpaceViolationPolicy.DISABLE), 512L, 1024L);
220    final TableName tn2 = getUniqueTableName();
221    final SpaceQuotaSnapshot snapshot2 = new SpaceQuotaSnapshot(
222        new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 512L, 1024L);
223    final TableName tn3 = getUniqueTableName();
224    final SpaceQuotaSnapshot snapshot3 = new SpaceQuotaSnapshot(
225        new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES), 512L, 1024L);
226    List<Put> puts = new ArrayList<>();
227    puts.add(QuotaTableUtil.createPutForSpaceSnapshot(tn1, snapshot1));
228    puts.add(QuotaTableUtil.createPutForSpaceSnapshot(tn2, snapshot2));
229    puts.add(QuotaTableUtil.createPutForSpaceSnapshot(tn3, snapshot3));
230    final Map<TableName,SpaceQuotaSnapshot> expectedPolicies = new HashMap<>();
231    expectedPolicies.put(tn1, snapshot1);
232    expectedPolicies.put(tn2, snapshot2);
233    expectedPolicies.put(tn3, snapshot3);
234
235    final Map<TableName,SpaceQuotaSnapshot> actualPolicies = new HashMap<>();
236    try (Table quotaTable = connection.getTable(QuotaUtil.QUOTA_TABLE_NAME)) {
237      quotaTable.put(puts);
238      ResultScanner scanner = quotaTable.getScanner(QuotaTableUtil.makeQuotaSnapshotScan());
239      for (Result r : scanner) {
240        QuotaTableUtil.extractQuotaSnapshot(r, actualPolicies);
241      }
242      scanner.close();
243    }
244
245    assertEquals(expectedPolicies, actualPolicies);
246  }
247
248  @Test
249  public void testSerdeTableSnapshotSizes() throws Exception {
250    TableName tn1 = TableName.valueOf("tn1");
251    TableName tn2 = TableName.valueOf("tn2");
252    try (Table quotaTable = connection.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
253      for (int i = 0; i < 3; i++) {
254        Put p = QuotaTableUtil.createPutForSnapshotSize(tn1, "tn1snap" + i, 1024L * (1+i));
255        quotaTable.put(p);
256      }
257      for (int i = 0; i < 3; i++) {
258        Put p = QuotaTableUtil.createPutForSnapshotSize(tn2, "tn2snap" + i, 2048L * (1+i));
259        quotaTable.put(p);
260      }
261
262      verifyTableSnapshotSize(quotaTable, tn1, "tn1snap0", 1024L);
263      verifyTableSnapshotSize(quotaTable, tn1, "tn1snap1", 2048L);
264      verifyTableSnapshotSize(quotaTable, tn1, "tn1snap2", 3072L);
265
266      verifyTableSnapshotSize(quotaTable, tn2, "tn2snap0", 2048L);
267      verifyTableSnapshotSize(quotaTable, tn2, "tn2snap1", 4096L);
268      verifyTableSnapshotSize(quotaTable, tn2, "tn2snap2", 6144L);
269    }
270  }
271
272  @Test
273  public void testReadNamespaceSnapshotSizes() throws Exception {
274    String ns1 = "ns1";
275    String ns2 = "ns2";
276    String defaultNs = NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR;
277    try (Table quotaTable = connection.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
278      quotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns1, 1024L));
279      quotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns2, 2048L));
280      quotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(defaultNs, 8192L));
281
282      assertEquals(1024L, QuotaTableUtil.getNamespaceSnapshotSize(connection, ns1));
283      assertEquals(2048L, QuotaTableUtil.getNamespaceSnapshotSize(connection, ns2));
284      assertEquals(8192L, QuotaTableUtil.getNamespaceSnapshotSize(connection, defaultNs));
285    }
286  }
287
288  private TableName getUniqueTableName() {
289    return TableName.valueOf(testName.getMethodName() + "_" + tableNameCounter++);
290  }
291
292  private void verifyTableSnapshotSize(
293      Table quotaTable, TableName tn, String snapshotName, long expectedSize) throws IOException {
294    Result r = quotaTable.get(QuotaTableUtil.makeGetForSnapshotSize(tn, snapshotName));
295    CellScanner cs = r.cellScanner();
296    assertTrue(cs.advance());
297    Cell c = cs.current();
298    assertEquals(expectedSize, QuotaProtos.SpaceQuotaSnapshot.parseFrom(
299        UnsafeByteOperations.unsafeWrap(
300            c.getValueArray(), c.getValueOffset(), c.getValueLength())).getQuotaUsage());
301    assertFalse(cs.advance());
302  }
303}