Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.quotas;
019
020import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.doGets;
021import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.doPuts;
022import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.triggerNamespaceCacheRefresh;
023import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.triggerTableCacheRefresh;
024import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.triggerUserCacheRefresh;
025import static org.junit.Assert.assertEquals;
026import static org.junit.Assert.assertTrue;
027
028import java.util.concurrent.TimeUnit;
029
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.HBaseTestingUtility;
032import org.apache.hadoop.hbase.HConstants;
033import org.apache.hadoop.hbase.NamespaceDescriptor;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.client.Admin;
036import org.apache.hadoop.hbase.client.Table;
037import org.apache.hadoop.hbase.security.User;
038import org.apache.hadoop.hbase.testclassification.LargeTests;
039import org.apache.hadoop.hbase.testclassification.RegionServerTests;
040import org.apache.hadoop.hbase.util.Bytes;
041import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
042import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
043import org.junit.After;
044import org.junit.AfterClass;
045import org.junit.BeforeClass;
046import org.junit.ClassRule;
047import org.junit.Test;
048import org.junit.experimental.categories.Category;
049
050@Category({ RegionServerTests.class, LargeTests.class })
051public class TestClusterScopeQuotaThrottle {
052
053  @ClassRule
054  public static final HBaseClassTestRule CLASS_RULE =
055      HBaseClassTestRule.forClass(TestClusterScopeQuotaThrottle.class);
056
057  private final static int REFRESH_TIME = 30 * 60000;
058  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
059
060  private final static TableName[] TABLE_NAMES =
061      new TableName[] { TableName.valueOf("TestQuotaAdmin0"), TableName.valueOf("TestQuotaAdmin1"),
062          TableName.valueOf("TestQuotaAdmin2") };
063  private final static byte[] FAMILY = Bytes.toBytes("cf");
064  private final static byte[] QUALIFIER = Bytes.toBytes("q");
065  private final static byte[][] SPLITS = new byte[][] { Bytes.toBytes("1") };
066  private static Table[] tables;
067
068  private final static String NAMESPACE = "TestNs";
069  private final static TableName TABLE_NAME = TableName.valueOf(NAMESPACE, "TestTable");
070  private static Table table;
071
072  @BeforeClass
073  public static void setUpBeforeClass() throws Exception {
074    TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
075    TEST_UTIL.getConfiguration().setInt(QuotaCache.REFRESH_CONF_KEY, REFRESH_TIME);
076    TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
077    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
078    TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
079    TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
080    TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
081    TEST_UTIL.startMiniCluster(2);
082    TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME);
083    QuotaCache.TEST_FORCE_REFRESH = true;
084
085    tables = new Table[TABLE_NAMES.length];
086    for (int i = 0; i < TABLE_NAMES.length; ++i) {
087      tables[i] = TEST_UTIL.createTable(TABLE_NAMES[i], FAMILY);
088      TEST_UTIL.waitTableAvailable(TABLE_NAMES[i]);
089    }
090    TEST_UTIL.getAdmin().createNamespace(NamespaceDescriptor.create(NAMESPACE).build());
091    table = TEST_UTIL.createTable(TABLE_NAME, FAMILY, SPLITS);
092    TEST_UTIL.waitTableAvailable(TABLE_NAME);
093  }
094
095  @AfterClass
096  public static void tearDownAfterClass() throws Exception {
097    EnvironmentEdgeManager.reset();
098    for (int i = 0; i < tables.length; ++i) {
099      if (tables[i] != null) {
100        tables[i].close();
101        TEST_UTIL.deleteTable(TABLE_NAMES[i]);
102      }
103    }
104    TEST_UTIL.deleteTable(TABLE_NAME);
105    TEST_UTIL.getAdmin().deleteNamespace(NAMESPACE);
106    TEST_UTIL.shutdownMiniCluster();
107  }
108
109  @After
110  public void tearDown() throws Exception {
111    ThrottleQuotaTestUtil.clearQuotaCache(TEST_UTIL);
112  }
113
114  @Test
115  public void testNamespaceClusterScopeQuota() throws Exception {
116    final Admin admin = TEST_UTIL.getAdmin();
117    final String NAMESPACE = "default";
118
119    // Add 10req/min limit for write request in cluster scope
120    admin.setQuota(QuotaSettingsFactory.throttleNamespace(NAMESPACE, ThrottleType.WRITE_NUMBER, 10,
121      TimeUnit.MINUTES, QuotaScope.CLUSTER));
122    // Add 6req/min limit for read request in machine scope
123    admin.setQuota(QuotaSettingsFactory.throttleNamespace(NAMESPACE, ThrottleType.READ_NUMBER, 6,
124      TimeUnit.MINUTES, QuotaScope.MACHINE));
125    triggerNamespaceCacheRefresh(TEST_UTIL, false, TABLE_NAMES[0]);
126    // should execute at max 5 write requests and at max 3 read requests
127    assertEquals(5, doPuts(10, FAMILY, QUALIFIER, tables[0]));
128    assertEquals(6, doGets(10, tables[0]));
129    // Remove all the limits
130    admin.setQuota(QuotaSettingsFactory.unthrottleNamespace(NAMESPACE));
131    triggerNamespaceCacheRefresh(TEST_UTIL, true, TABLE_NAMES[0]);
132  }
133
134  @Test
135  public void testTableClusterScopeQuota() throws Exception {
136    final Admin admin = TEST_UTIL.getAdmin();
137    admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAME, ThrottleType.READ_NUMBER, 20,
138      TimeUnit.HOURS, QuotaScope.CLUSTER));
139    triggerTableCacheRefresh(TEST_UTIL, false, TABLE_NAME);
140    for (RegionServerThread rst : TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
141      for (TableName tableName : rst.getRegionServer().getOnlineTables()) {
142        if (tableName.getNameAsString().equals(TABLE_NAME.getNameAsString())) {
143          int rsRegionNum = rst.getRegionServer().getRegions(tableName).size();
144          if (rsRegionNum == 0) {
145            // If rs has 0 region, the machine limiter is 0 (20 * 0 / 2)
146            break;
147          } else if (rsRegionNum == 1) {
148            // If rs has 1 region, the machine limiter is 10 (20 * 1 / 2)
149            // Read rows from 1 region, so can read 10 first time and 0 second time
150            long count = doGets(20, table);
151            assertTrue(count == 0 || count == 10);
152          } else if (rsRegionNum == 2) {
153            // If rs has 2 regions, the machine limiter is 20 (20 * 2 / 2)
154            assertEquals(20, doGets(20, table));
155          }
156          break;
157        }
158      }
159    }
160    admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAME));
161    triggerTableCacheRefresh(TEST_UTIL, true, TABLE_NAME);
162  }
163
164  @Test
165  public void testUserClusterScopeQuota() throws Exception {
166    final Admin admin = TEST_UTIL.getAdmin();
167    final String userName = User.getCurrent().getShortName();
168
169    // Add 6req/min limit for read request in cluster scope
170    admin.setQuota(QuotaSettingsFactory.throttleUser(userName, ThrottleType.READ_NUMBER, 6,
171      TimeUnit.MINUTES, QuotaScope.CLUSTER));
172    // Add 6req/min limit for write request in machine scope
173    admin.setQuota(
174      QuotaSettingsFactory.throttleUser(userName, ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
175    triggerUserCacheRefresh(TEST_UTIL, false, TABLE_NAMES);
176    // should execute at max 6 read requests and at max 3 write write requests
177    assertEquals(6, doPuts(10, FAMILY, QUALIFIER, tables[0]));
178    assertEquals(3, doGets(10, tables[0]));
179    // Remove all the limits
180    admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
181    triggerUserCacheRefresh(TEST_UTIL, true, TABLE_NAMES);
182  }
183
184  @org.junit.Ignore @Test // Spews the log w/ triggering of scheduler? HBASE-24035
185  public void testUserNamespaceClusterScopeQuota() throws Exception {
186    final Admin admin = TEST_UTIL.getAdmin();
187    final String userName = User.getCurrent().getShortName();
188    final String namespace = TABLE_NAMES[0].getNamespaceAsString();
189
190    // Add 10req/min limit for read request in cluster scope
191    admin.setQuota(QuotaSettingsFactory.throttleUser(userName, namespace, ThrottleType.READ_NUMBER,
192      10, TimeUnit.MINUTES, QuotaScope.CLUSTER));
193    // Add 6req/min limit for write request in machine scope
194    admin.setQuota(QuotaSettingsFactory.throttleUser(userName, namespace, ThrottleType.WRITE_NUMBER,
195      6, TimeUnit.MINUTES));
196    triggerUserCacheRefresh(TEST_UTIL, false, TABLE_NAMES[0]);
197    // should execute at max 5 read requests and at max 6 write requests
198    assertEquals(5, doGets(10, tables[0]));
199    assertEquals(6, doPuts(10, FAMILY, QUALIFIER, tables[0]));
200
201    // Remove all the limits
202    admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, namespace));
203    triggerUserCacheRefresh(TEST_UTIL, true, TABLE_NAMES[0]);
204  }
205
206  @Test
207  public void testUserTableClusterScopeQuota() throws Exception {
208    final Admin admin = TEST_UTIL.getAdmin();
209    final String userName = User.getCurrent().getShortName();
210    admin.setQuota(QuotaSettingsFactory.throttleUser(userName, TABLE_NAME, ThrottleType.READ_NUMBER,
211      20, TimeUnit.HOURS, QuotaScope.CLUSTER));
212    triggerUserCacheRefresh(TEST_UTIL, false, TABLE_NAME);
213    for (RegionServerThread rst : TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
214      for (TableName tableName : rst.getRegionServer().getOnlineTables()) {
215        if (tableName.getNameAsString().equals(TABLE_NAME.getNameAsString())) {
216          int rsRegionNum = rst.getRegionServer().getRegions(tableName).size();
217          if (rsRegionNum == 0) {
218            // If rs has 0 region, the machine limiter is 0 (20 * 0 / 2)
219            break;
220          } else if (rsRegionNum == 1) {
221            // If rs has 1 region, the machine limiter is 10 (20 * 1 / 2)
222            // Read rows from 1 region, so can read 10 first time and 0 second time
223            long count = doGets(20, table);
224            assertTrue(count == 0 || count == 10);
225          } else if (rsRegionNum == 2) {
226            // If rs has 2 regions, the machine limiter is 20 (20 * 2 / 2)
227            assertEquals(20, doGets(20, table));
228          }
229          break;
230        }
231      }
232    }
233    admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
234    triggerUserCacheRefresh(TEST_UTIL, true, TABLE_NAME);
235  }
236}