Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.quotas;
019
020import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.doGets;
021import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.doPuts;
022import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.triggerNamespaceCacheRefresh;
023import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.triggerTableCacheRefresh;
024import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.triggerUserCacheRefresh;
025import static org.junit.jupiter.api.Assertions.assertEquals;
026import static org.junit.jupiter.api.Assertions.assertTrue;
027
028import java.util.concurrent.TimeUnit;
029import org.apache.hadoop.hbase.HBaseTestingUtil;
030import org.apache.hadoop.hbase.NamespaceDescriptor;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.client.Admin;
033import org.apache.hadoop.hbase.client.Table;
034import org.apache.hadoop.hbase.security.User;
035import org.apache.hadoop.hbase.testclassification.LargeTests;
036import org.apache.hadoop.hbase.testclassification.RegionServerTests;
037import org.apache.hadoop.hbase.util.Bytes;
038import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
039import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
040import org.junit.jupiter.api.AfterAll;
041import org.junit.jupiter.api.AfterEach;
042import org.junit.jupiter.api.BeforeAll;
043import org.junit.jupiter.api.Tag;
044import org.junit.jupiter.api.Test;
045
046@Tag(RegionServerTests.TAG)
047@Tag(LargeTests.TAG)
048public class TestClusterScopeQuotaThrottle {
049
050  private final static int REFRESH_TIME = 30 * 60000;
051  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
052
053  private final static TableName[] TABLE_NAMES =
054    new TableName[] { TableName.valueOf("TestQuotaAdmin0"), TableName.valueOf("TestQuotaAdmin1"),
055      TableName.valueOf("TestQuotaAdmin2") };
056  private final static byte[] FAMILY = Bytes.toBytes("cf");
057  private final static byte[] QUALIFIER = Bytes.toBytes("q");
058  private final static byte[][] SPLITS = new byte[][] { Bytes.toBytes("1") };
059  private static Table[] tables;
060
061  private final static String NAMESPACE = "TestNs";
062  private final static TableName TABLE_NAME = TableName.valueOf(NAMESPACE, "TestTable");
063  private static Table table;
064
065  @BeforeAll
066  public static void setUpBeforeClass() throws Exception {
067    TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
068    TEST_UTIL.getConfiguration().setInt(QuotaCache.REFRESH_CONF_KEY, REFRESH_TIME);
069    TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
070    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
071    TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
072    TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
073    TEST_UTIL.startMiniCluster(2);
074    TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME);
075
076    tables = new Table[TABLE_NAMES.length];
077    for (int i = 0; i < TABLE_NAMES.length; ++i) {
078      TEST_UTIL.createTable(TABLE_NAMES[i], FAMILY);
079      TEST_UTIL.waitTableAvailable(TABLE_NAMES[i]);
080      tables[i] = TEST_UTIL.getConnection().getTableBuilder(TABLE_NAMES[i], null)
081        .setOperationTimeout(10000).build();
082    }
083    TEST_UTIL.getAdmin().createNamespace(NamespaceDescriptor.create(NAMESPACE).build());
084    TEST_UTIL.createTable(TABLE_NAME, FAMILY, SPLITS);
085    TEST_UTIL.waitTableAvailable(TABLE_NAME);
086    table = TEST_UTIL.getConnection().getTableBuilder(TABLE_NAME, null).setOperationTimeout(10000)
087      .build();
088  }
089
090  @AfterAll
091  public static void tearDownAfterClass() throws Exception {
092    EnvironmentEdgeManager.reset();
093    for (int i = 0; i < tables.length; ++i) {
094      if (tables[i] != null) {
095        tables[i].close();
096        TEST_UTIL.deleteTable(TABLE_NAMES[i]);
097      }
098    }
099    TEST_UTIL.deleteTable(TABLE_NAME);
100    TEST_UTIL.getAdmin().deleteNamespace(NAMESPACE);
101    TEST_UTIL.shutdownMiniCluster();
102  }
103
104  @AfterEach
105  public void tearDown() throws Exception {
106    ThrottleQuotaTestUtil.clearQuotaCache(TEST_UTIL);
107  }
108
109  @Test
110  public void testNamespaceClusterScopeQuota() throws Exception {
111    final Admin admin = TEST_UTIL.getAdmin();
112    final String NAMESPACE = "default";
113
114    // Add 10req/min limit for write request in cluster scope
115    admin.setQuota(QuotaSettingsFactory.throttleNamespace(NAMESPACE, ThrottleType.WRITE_NUMBER, 10,
116      TimeUnit.MINUTES, QuotaScope.CLUSTER));
117    // Add 6req/min limit for read request in machine scope
118    admin.setQuota(QuotaSettingsFactory.throttleNamespace(NAMESPACE, ThrottleType.READ_NUMBER, 6,
119      TimeUnit.MINUTES, QuotaScope.MACHINE));
120    triggerNamespaceCacheRefresh(TEST_UTIL, false, TABLE_NAMES[0]);
121    // should execute at max 5 write requests and at max 3 read requests
122    assertEquals(5, doPuts(10, FAMILY, QUALIFIER, tables[0]));
123    assertEquals(6, doGets(10, tables[0]));
124    // Remove all the limits
125    admin.setQuota(QuotaSettingsFactory.unthrottleNamespace(NAMESPACE));
126    triggerNamespaceCacheRefresh(TEST_UTIL, true, TABLE_NAMES[0]);
127  }
128
129  @Test
130  public void testTableClusterScopeQuota() throws Exception {
131    final Admin admin = TEST_UTIL.getAdmin();
132    admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAME, ThrottleType.READ_NUMBER, 20,
133      TimeUnit.HOURS, QuotaScope.CLUSTER));
134    triggerTableCacheRefresh(TEST_UTIL, false, TABLE_NAME);
135    for (RegionServerThread rst : TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
136      for (TableName tableName : rst.getRegionServer().getOnlineTables()) {
137        if (tableName.getNameAsString().equals(TABLE_NAME.getNameAsString())) {
138          int rsRegionNum = rst.getRegionServer().getRegions(tableName).size();
139          if (rsRegionNum == 0) {
140            // If rs has 0 region, the machine limiter is 0 (20 * 0 / 2)
141            break;
142          } else if (rsRegionNum == 1) {
143            // If rs has 1 region, the machine limiter is 10 (20 * 1 / 2)
144            // Read rows from 1 region, so can read 10 first time and 0 second time
145            long count = doGets(20, table);
146            assertTrue(count == 0 || count == 10);
147          } else if (rsRegionNum == 2) {
148            // If rs has 2 regions, the machine limiter is 20 (20 * 2 / 2)
149            assertEquals(20, doGets(20, table));
150          }
151          break;
152        }
153      }
154    }
155    admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAME));
156    triggerTableCacheRefresh(TEST_UTIL, true, TABLE_NAME);
157  }
158
159  @Test
160  public void testUserClusterScopeQuota() throws Exception {
161    final Admin admin = TEST_UTIL.getAdmin();
162    final String userName = User.getCurrent().getShortName();
163
164    // Add 6req/min limit for read request in cluster scope
165    admin.setQuota(QuotaSettingsFactory.throttleUser(userName, ThrottleType.READ_NUMBER, 6,
166      TimeUnit.MINUTES, QuotaScope.CLUSTER));
167    // Add 6req/min limit for write request in machine scope
168    admin.setQuota(
169      QuotaSettingsFactory.throttleUser(userName, ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
170    triggerUserCacheRefresh(TEST_UTIL, false, TABLE_NAMES);
171    // should execute at max 6 read requests and at max 3 write write requests
172    assertEquals(6, doPuts(10, FAMILY, QUALIFIER, tables[0]));
173    assertEquals(3, doGets(10, tables[0]));
174    // Remove all the limits
175    admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
176    triggerUserCacheRefresh(TEST_UTIL, true, TABLE_NAMES);
177  }
178
179  @Test // Spews the log w/ triggering of scheduler? HBASE-24035
180  public void testUserNamespaceClusterScopeQuota() throws Exception {
181    final Admin admin = TEST_UTIL.getAdmin();
182    final String userName = User.getCurrent().getShortName();
183    final String namespace = TABLE_NAMES[0].getNamespaceAsString();
184
185    // Add 10req/min limit for read request in cluster scope
186    admin.setQuota(QuotaSettingsFactory.throttleUser(userName, namespace, ThrottleType.READ_NUMBER,
187      10, TimeUnit.MINUTES, QuotaScope.CLUSTER));
188    // Add 6req/min limit for write request in machine scope
189    admin.setQuota(QuotaSettingsFactory.throttleUser(userName, namespace, ThrottleType.WRITE_NUMBER,
190      6, TimeUnit.MINUTES));
191    triggerUserCacheRefresh(TEST_UTIL, false, TABLE_NAMES[0]);
192    // should execute at max 5 read requests and at max 6 write requests
193    assertEquals(5, doGets(10, tables[0]));
194    assertEquals(6, doPuts(10, FAMILY, QUALIFIER, tables[0]));
195
196    // Remove all the limits
197    admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, namespace));
198    triggerUserCacheRefresh(TEST_UTIL, true, TABLE_NAMES[0]);
199  }
200
201  @Test
202  public void testUserTableClusterScopeQuota() throws Exception {
203    final Admin admin = TEST_UTIL.getAdmin();
204    final String userName = User.getCurrent().getShortName();
205    admin.setQuota(QuotaSettingsFactory.throttleUser(userName, TABLE_NAME, ThrottleType.READ_NUMBER,
206      20, TimeUnit.HOURS, QuotaScope.CLUSTER));
207    triggerUserCacheRefresh(TEST_UTIL, false, TABLE_NAME);
208    for (RegionServerThread rst : TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
209      for (TableName tableName : rst.getRegionServer().getOnlineTables()) {
210        if (tableName.getNameAsString().equals(TABLE_NAME.getNameAsString())) {
211          int rsRegionNum = rst.getRegionServer().getRegions(tableName).size();
212          if (rsRegionNum == 0) {
213            // If rs has 0 region, the machine limiter is 0 (20 * 0 / 2)
214            break;
215          } else if (rsRegionNum == 1) {
216            // If rs has 1 region, the machine limiter is 10 (20 * 1 / 2)
217            // Read rows from 1 region, so can read 10 first time and 0 second time
218            long count = doGets(20, table);
219            assertTrue(count == 0 || count == 10);
220          } else if (rsRegionNum == 2) {
221            // If rs has 2 regions, the machine limiter is 20 (20 * 2 / 2)
222            assertEquals(20, doGets(20, table));
223          }
224          break;
225        }
226      }
227    }
228    admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
229    triggerUserCacheRefresh(TEST_UTIL, true, TABLE_NAME);
230  }
231}