Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.quotas;
019
020import static org.junit.Assert.assertEquals;
021
022import java.util.Objects;
023import java.util.concurrent.TimeUnit;
024import org.apache.hadoop.hbase.HBaseClassTestRule;
025import org.apache.hadoop.hbase.HBaseTestingUtility;
026import org.apache.hadoop.hbase.HConstants;
027import org.apache.hadoop.hbase.TableName;
028import org.apache.hadoop.hbase.client.Admin;
029import org.apache.hadoop.hbase.client.Get;
030import org.apache.hadoop.hbase.client.Put;
031import org.apache.hadoop.hbase.client.Table;
032import org.apache.hadoop.hbase.security.User;
033import org.apache.hadoop.hbase.testclassification.MediumTests;
034import org.apache.hadoop.hbase.testclassification.RegionServerTests;
035import org.apache.hadoop.hbase.util.Bytes;
036import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
037import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
038import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
039import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
040import org.junit.After;
041import org.junit.AfterClass;
042import org.junit.BeforeClass;
043import org.junit.ClassRule;
044import org.junit.Ignore;
045import org.junit.Test;
046import org.junit.experimental.categories.Category;
047import org.slf4j.Logger;
048import org.slf4j.LoggerFactory;
049
050@Ignore // Disabled because flakey. Fails ~30% on a resource constrained GCE though not on Apache.
051@Category({RegionServerTests.class, MediumTests.class})
052public class TestQuotaThrottle {
053
054  @ClassRule
055  public static final HBaseClassTestRule CLASS_RULE =
056      HBaseClassTestRule.forClass(TestQuotaThrottle.class);
057
058  private final static Logger LOG = LoggerFactory.getLogger(TestQuotaThrottle.class);
059
060  private final static int REFRESH_TIME = 30 * 60000;
061
062  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
063  private final static byte[] FAMILY = Bytes.toBytes("cf");
064  private final static byte[] QUALIFIER = Bytes.toBytes("q");
065
066  private final static TableName[] TABLE_NAMES = new TableName[] {
067    TableName.valueOf("TestQuotaAdmin0"),
068    TableName.valueOf("TestQuotaAdmin1"),
069    TableName.valueOf("TestQuotaAdmin2")
070  };
071
072  private static ManualEnvironmentEdge envEdge;
073  private static Table[] tables;
074
075  @BeforeClass
076  public static void setUpBeforeClass() throws Exception {
077    TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
078    TEST_UTIL.getConfiguration().setInt(QuotaCache.REFRESH_CONF_KEY, REFRESH_TIME);
079    TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
080    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
081    TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
082    TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
083    TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
084    TEST_UTIL.startMiniCluster(1);
085    TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME);
086    QuotaCache.TEST_FORCE_REFRESH = true;
087
088    tables = new Table[TABLE_NAMES.length];
089    for (int i = 0; i < TABLE_NAMES.length; ++i) {
090      tables[i] = TEST_UTIL.createTable(TABLE_NAMES[i], FAMILY);
091    }
092
093    envEdge = new ManualEnvironmentEdge();
094    envEdge.setValue(EnvironmentEdgeManager.currentTime());
095    EnvironmentEdgeManagerTestHelper.injectEdge(envEdge);
096  }
097
098  @AfterClass
099  public static void tearDownAfterClass() throws Exception {
100    EnvironmentEdgeManager.reset();
101    for (int i = 0; i < tables.length; ++i) {
102      if (tables[i] != null) {
103        tables[i].close();
104        TEST_UTIL.deleteTable(TABLE_NAMES[i]);
105      }
106    }
107
108    TEST_UTIL.shutdownMiniCluster();
109  }
110
111  @After
112  public void tearDown() throws Exception {
113    for (RegionServerThread rst: TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
114      RegionServerRpcQuotaManager quotaManager = rst.getRegionServer().getRegionServerRpcQuotaManager();
115      QuotaCache quotaCache = quotaManager.getQuotaCache();
116      quotaCache.getNamespaceQuotaCache().clear();
117      quotaCache.getTableQuotaCache().clear();
118      quotaCache.getUserQuotaCache().clear();
119    }
120  }
121
122  @Test
123  public void testUserGlobalThrottle() throws Exception {
124    final Admin admin = TEST_UTIL.getAdmin();
125    final String userName = User.getCurrent().getShortName();
126
127    // Add 6req/min limit
128    admin.setQuota(QuotaSettingsFactory
129      .throttleUser(userName, ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES));
130    triggerUserCacheRefresh(false, TABLE_NAMES);
131
132    // should execute at max 6 requests
133    assertEquals(6, doPuts(100, tables));
134
135    // wait a minute and you should get other 6 requests executed
136    waitMinuteQuota();
137    assertEquals(6, doPuts(100, tables));
138
139    // Remove all the limits
140    admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
141    triggerUserCacheRefresh(true, TABLE_NAMES);
142    assertEquals(60, doPuts(60, tables));
143    assertEquals(60, doGets(60, tables));
144  }
145
146  @Test
147  public void testUserGlobalReadAndWriteThrottle() throws Exception {
148    final Admin admin = TEST_UTIL.getAdmin();
149    final String userName = User.getCurrent().getShortName();
150
151    // Add 6req/min limit for read request
152    admin.setQuota(QuotaSettingsFactory
153      .throttleUser(userName, ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
154    triggerUserCacheRefresh(false, TABLE_NAMES);
155
156    // not limit for write request and should execute at max 6 read requests
157    assertEquals(60, doPuts(60, tables));
158    assertEquals(6, doGets(100, tables));
159
160    waitMinuteQuota();
161
162    // Add 6req/min limit for write request
163    admin.setQuota(QuotaSettingsFactory
164      .throttleUser(userName, ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
165    triggerUserCacheRefresh(false, TABLE_NAMES);
166
167    // should execute at max 6 read requests and at max 6 write write requests
168    assertEquals(6, doGets(100, tables));
169    assertEquals(6, doPuts(60, tables));
170
171    // Remove all the limits
172    admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
173    triggerUserCacheRefresh(true, TABLE_NAMES);
174    assertEquals(60, doPuts(60, tables));
175    assertEquals(60, doGets(60, tables));
176  }
177
178  @Test
179  public void testUserTableThrottle() throws Exception {
180    final Admin admin = TEST_UTIL.getAdmin();
181    final String userName = User.getCurrent().getShortName();
182
183    // Add 6req/min limit
184    admin.setQuota(QuotaSettingsFactory
185      .throttleUser(userName, TABLE_NAMES[0], ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES));
186    triggerUserCacheRefresh(false, TABLE_NAMES[0]);
187
188    // should execute at max 6 requests on tables[0] and have no limit on tables[1]
189    assertEquals(6, doPuts(100, tables[0]));
190    assertEquals(30, doPuts(30, tables[1]));
191
192    // wait a minute and you should get other 6 requests executed
193    waitMinuteQuota();
194    assertEquals(6, doPuts(100, tables[0]));
195
196    // Remove all the limits
197    admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, TABLE_NAMES[0]));
198    triggerUserCacheRefresh(true, TABLE_NAMES);
199    assertEquals(60, doPuts(60, tables));
200    assertEquals(60, doGets(60, tables));
201  }
202
203  @Test
204  public void testUserTableReadAndWriteThrottle() throws Exception {
205    final Admin admin = TEST_UTIL.getAdmin();
206    final String userName = User.getCurrent().getShortName();
207
208    // Add 6req/min limit for write request on tables[0]
209    admin.setQuota(QuotaSettingsFactory
210      .throttleUser(userName, TABLE_NAMES[0], ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
211    triggerUserCacheRefresh(false, TABLE_NAMES[0]);
212
213    // should execute at max 6 write requests and have no limit for read request
214    assertEquals(6, doPuts(100, tables[0]));
215    assertEquals(60, doGets(60, tables[0]));
216
217    // no limit on tables[1]
218    assertEquals(60, doPuts(60, tables[1]));
219    assertEquals(60, doGets(60, tables[1]));
220
221    // wait a minute and you should get other 6  write requests executed
222    waitMinuteQuota();
223
224    // Add 6req/min limit for read request on tables[0]
225    admin.setQuota(QuotaSettingsFactory
226      .throttleUser(userName, TABLE_NAMES[0], ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
227    triggerUserCacheRefresh(false, TABLE_NAMES[0]);
228
229    // should execute at max 6 read requests and at max 6 write requests
230    assertEquals(6, doPuts(100, tables[0]));
231    assertEquals(6, doGets(60, tables[0]));
232
233    // no limit on tables[1]
234    assertEquals(30, doPuts(30, tables[1]));
235    assertEquals(30, doGets(30, tables[1]));
236
237    // Remove all the limits
238    admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, TABLE_NAMES[0]));
239    triggerUserCacheRefresh(true, TABLE_NAMES);
240    assertEquals(60, doPuts(60, tables));
241    assertEquals(60, doGets(60, tables));
242  }
243
244  @Test
245  public void testUserNamespaceThrottle() throws Exception {
246    final Admin admin = TEST_UTIL.getAdmin();
247    final String userName = User.getCurrent().getShortName();
248    final String NAMESPACE = "default";
249
250    // Add 6req/min limit
251    admin.setQuota(QuotaSettingsFactory
252      .throttleUser(userName, NAMESPACE, ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES));
253    triggerUserCacheRefresh(false, TABLE_NAMES[0]);
254
255    // should execute at max 6 requests on tables[0] and have no limit on tables[1]
256    assertEquals(6, doPuts(100, tables[0]));
257
258    // wait a minute and you should get other 6 requests executed
259    waitMinuteQuota();
260    assertEquals(6, doPuts(100, tables[1]));
261
262    // Remove all the limits
263    admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, NAMESPACE));
264    triggerUserCacheRefresh(true, TABLE_NAMES);
265    assertEquals(60, doPuts(60, tables));
266    assertEquals(60, doGets(60, tables));
267  }
268
269  @Test
270  public void testUserNamespaceReadAndWriteThrottle() throws Exception {
271    final Admin admin = TEST_UTIL.getAdmin();
272    final String userName = User.getCurrent().getShortName();
273    final String NAMESPACE = "default";
274
275    // Add 6req/min limit for read request
276    admin.setQuota(QuotaSettingsFactory
277      .throttleUser(userName, NAMESPACE, ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
278    triggerUserCacheRefresh(false, TABLE_NAMES[0]);
279
280    // should execute at max 6 read requests and have no limit for write request
281    assertEquals(6, doGets(60, tables[0]));
282    assertEquals(60, doPuts(60, tables[0]));
283
284    waitMinuteQuota();
285
286    // Add 6req/min limit for write request, too
287    admin.setQuota(QuotaSettingsFactory
288      .throttleUser(userName, NAMESPACE, ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
289    triggerUserCacheRefresh(false, TABLE_NAMES[0]);
290
291    // should execute at max 6 read requests and at max 6 write requests
292    assertEquals(6, doGets(60, tables[0]));
293    assertEquals(6, doPuts(60, tables[0]));
294
295    // Remove all the limits
296    admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, NAMESPACE));
297    triggerUserCacheRefresh(true, TABLE_NAMES);
298    assertEquals(60, doPuts(60, tables));
299    assertEquals(60, doGets(60, tables));
300  }
301
302  @Test
303  public void testTableGlobalThrottle() throws Exception {
304    final Admin admin = TEST_UTIL.getAdmin();
305
306    // Add 6req/min limit
307    admin.setQuota(QuotaSettingsFactory
308      .throttleTable(TABLE_NAMES[0], ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES));
309    triggerTableCacheRefresh(false, TABLE_NAMES[0]);
310
311    // should execute at max 6 requests
312    assertEquals(6, doPuts(100, tables[0]));
313    // should have no limits
314    assertEquals(30, doPuts(30, tables[1]));
315
316    // wait a minute and you should get other 6 requests executed
317    waitMinuteQuota();
318    assertEquals(6, doPuts(100, tables[0]));
319
320    // Remove all the limits
321    admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0]));
322    triggerTableCacheRefresh(true, TABLE_NAMES[0]);
323    assertEquals(80, doGets(80, tables[0], tables[1]));
324  }
325
326  @Test
327  public void testTableGlobalReadAndWriteThrottle() throws Exception {
328    final Admin admin = TEST_UTIL.getAdmin();
329
330    // Add 6req/min limit for read request
331    admin.setQuota(QuotaSettingsFactory
332      .throttleTable(TABLE_NAMES[0], ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
333    triggerTableCacheRefresh(false, TABLE_NAMES[0]);
334
335    // should execute at max 6 read requests and have no limit for write request
336    assertEquals(6, doGets(100, tables[0]));
337    assertEquals(100, doPuts(100, tables[0]));
338    // should have no limits on tables[1]
339    assertEquals(30, doPuts(30, tables[1]));
340    assertEquals(30, doGets(30, tables[1]));
341
342    // wait a minute and you should get other 6 requests executed
343    waitMinuteQuota();
344
345    // Add 6req/min limit for write request, too
346    admin.setQuota(QuotaSettingsFactory
347      .throttleTable(TABLE_NAMES[0], ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
348    triggerTableCacheRefresh(false, TABLE_NAMES[0]);
349
350    // should execute at max 6 read requests and at max 6 write requests
351    assertEquals(6, doGets(100, tables[0]));
352    assertEquals(6, doPuts(100, tables[0]));
353    // should have no limits on tables[1]
354    assertEquals(30, doPuts(30, tables[1]));
355    assertEquals(30, doGets(30, tables[1]));
356
357    // Remove all the limits
358    admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0]));
359    triggerTableCacheRefresh(true, TABLE_NAMES[0]);
360    assertEquals(80, doGets(80, tables[0], tables[1]));
361  }
362
363  @Test
364  public void testNamespaceGlobalThrottle() throws Exception {
365    final Admin admin = TEST_UTIL.getAdmin();
366    final String NAMESPACE = "default";
367
368    // Add 6req/min limit
369    admin.setQuota(QuotaSettingsFactory
370      .throttleNamespace(NAMESPACE, ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES));
371    triggerNamespaceCacheRefresh(false, TABLE_NAMES[0]);
372
373    // should execute at max 6 requests
374    assertEquals(6, doPuts(100, tables[0]));
375
376    // wait a minute and you should get other 6 requests executed
377    waitMinuteQuota();
378    assertEquals(6, doPuts(100, tables[1]));
379
380    admin.setQuota(QuotaSettingsFactory.unthrottleNamespace(NAMESPACE));
381    triggerNamespaceCacheRefresh(true, TABLE_NAMES[0]);
382    assertEquals(40, doPuts(40, tables[0]));
383  }
384
385  @Test
386  public void testNamespaceGlobalReadAndWriteThrottle() throws Exception {
387    final Admin admin = TEST_UTIL.getAdmin();
388    final String NAMESPACE = "default";
389
390    // Add 6req/min limit for write request
391    admin.setQuota(QuotaSettingsFactory
392      .throttleNamespace(NAMESPACE, ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
393    triggerNamespaceCacheRefresh(false, TABLE_NAMES[0]);
394
395    // should execute at max 6 write requests and no limit for read request
396    assertEquals(6, doPuts(100, tables[0]));
397    assertEquals(100, doGets(100, tables[0]));
398
399    // wait a minute and you should get other 6 requests executed
400    waitMinuteQuota();
401
402    // Add 6req/min limit for read request, too
403    admin.setQuota(QuotaSettingsFactory
404      .throttleNamespace(NAMESPACE, ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
405    triggerNamespaceCacheRefresh(false, TABLE_NAMES[0]);
406
407    // should execute at max 6 write requests and at max 6 read requests
408    assertEquals(6, doPuts(100, tables[0]));
409    assertEquals(6, doGets(100, tables[0]));
410
411    admin.setQuota(QuotaSettingsFactory.unthrottleNamespace(NAMESPACE));
412    triggerNamespaceCacheRefresh(true, TABLE_NAMES[0]);
413    assertEquals(40, doPuts(40, tables[0]));
414  }
415
416  @Test
417  public void testUserAndTableThrottle() throws Exception {
418    final Admin admin = TEST_UTIL.getAdmin();
419    final String userName = User.getCurrent().getShortName();
420
421    // Add 6req/min limit for the user on tables[0]
422    admin.setQuota(QuotaSettingsFactory
423      .throttleUser(userName, TABLE_NAMES[0], ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES));
424    triggerUserCacheRefresh(false, TABLE_NAMES[0]);
425    // Add 12req/min limit for the user
426    admin.setQuota(QuotaSettingsFactory
427      .throttleUser(userName, ThrottleType.REQUEST_NUMBER, 12, TimeUnit.MINUTES));
428    triggerUserCacheRefresh(false, TABLE_NAMES[1], TABLE_NAMES[2]);
429    // Add 8req/min limit for the tables[1]
430    admin.setQuota(QuotaSettingsFactory
431      .throttleTable(TABLE_NAMES[1], ThrottleType.REQUEST_NUMBER, 8, TimeUnit.MINUTES));
432    triggerTableCacheRefresh(false, TABLE_NAMES[1]);
433    // Add a lower table level throttle on tables[0]
434    admin.setQuota(QuotaSettingsFactory
435      .throttleTable(TABLE_NAMES[0], ThrottleType.REQUEST_NUMBER, 3, TimeUnit.MINUTES));
436    triggerTableCacheRefresh(false, TABLE_NAMES[0]);
437
438    // should execute at max 12 requests
439    assertEquals(12, doGets(100, tables[2]));
440
441    // should execute at max 8 requests
442    waitMinuteQuota();
443    assertEquals(8, doGets(100, tables[1]));
444
445    // should execute at max 3 requests
446    waitMinuteQuota();
447    assertEquals(3, doPuts(100, tables[0]));
448
449    // Remove all the throttling rules
450    admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, TABLE_NAMES[0]));
451    admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
452    triggerUserCacheRefresh(true, TABLE_NAMES[0], TABLE_NAMES[1]);
453
454    admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[1]));
455    triggerTableCacheRefresh(true, TABLE_NAMES[1]);
456    waitMinuteQuota();
457    assertEquals(40, doGets(40, tables[1]));
458
459    admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0]));
460    triggerTableCacheRefresh(true, TABLE_NAMES[0]);
461    waitMinuteQuota();
462    assertEquals(40, doGets(40, tables[0]));
463  }
464
465  @Test
466  public void testUserGlobalBypassThrottle() throws Exception {
467    final Admin admin = TEST_UTIL.getAdmin();
468    final String userName = User.getCurrent().getShortName();
469    final String NAMESPACE = "default";
470
471    // Add 6req/min limit for tables[0]
472    admin.setQuota(QuotaSettingsFactory
473      .throttleTable(TABLE_NAMES[0], ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES));
474    triggerTableCacheRefresh(false, TABLE_NAMES[0]);
475    // Add 13req/min limit for the user
476    admin.setQuota(QuotaSettingsFactory
477      .throttleNamespace(NAMESPACE, ThrottleType.REQUEST_NUMBER, 13, TimeUnit.MINUTES));
478    triggerNamespaceCacheRefresh(false, TABLE_NAMES[1]);
479
480    // should execute at max 6 requests on table[0] and (13 - 6) on table[1]
481    assertEquals(6, doPuts(100, tables[0]));
482    assertEquals(7, doGets(100, tables[1]));
483    waitMinuteQuota();
484
485    // Set the global bypass for the user
486    admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, true));
487    admin.setQuota(QuotaSettingsFactory
488      .throttleUser(userName, TABLE_NAMES[2], ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES));
489    triggerUserCacheRefresh(false, TABLE_NAMES[2]);
490    assertEquals(30, doGets(30, tables[0]));
491    assertEquals(30, doGets(30, tables[1]));
492    waitMinuteQuota();
493
494    // Remove the global bypass
495    // should execute at max 6 requests on table[0] and (13 - 6) on table[1]
496    admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, false));
497    admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, TABLE_NAMES[2]));
498    triggerUserCacheRefresh(true, TABLE_NAMES[2]);
499    assertEquals(6, doPuts(100, tables[0]));
500    assertEquals(7, doGets(100, tables[1]));
501
502    // unset throttle
503    admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0]));
504    admin.setQuota(QuotaSettingsFactory.unthrottleNamespace(NAMESPACE));
505    waitMinuteQuota();
506    triggerTableCacheRefresh(true, TABLE_NAMES[0]);
507    triggerNamespaceCacheRefresh(true, TABLE_NAMES[1]);
508    assertEquals(30, doGets(30, tables[0]));
509    assertEquals(30, doGets(30, tables[1]));
510  }
511
512  @Test
513  public void testTableExistsGetThrottle() throws Exception {
514    final Admin admin = TEST_UTIL.getAdmin();
515
516    // Add throttle quota
517    admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAMES[0], ThrottleType.REQUEST_NUMBER,
518      100, TimeUnit.MINUTES));
519    triggerTableCacheRefresh(false, TABLE_NAMES[0]);
520
521    Table table = TEST_UTIL.getConnection().getTable(TABLE_NAMES[0]);
522    // An exists call when having throttle quota
523    table.exists(new Get(Bytes.toBytes("abc")));
524
525    admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0]));
526    triggerTableCacheRefresh(true, TABLE_NAMES[0]);
527  }
528
529  private int doPuts(int maxOps, final Table... tables) throws Exception {
530    int count = 0;
531    try {
532      while (count < maxOps) {
533        Put put = new Put(Bytes.toBytes("row-" + count));
534        put.addColumn(FAMILY, QUALIFIER, Bytes.toBytes("data-" + count));
535        for (final Table table: tables) {
536          table.put(put);
537        }
538        count += tables.length;
539      }
540    } catch (RpcThrottlingException e) {
541      LOG.error("put failed after nRetries=" + count, e);
542    }
543    return count;
544  }
545
546  private long doGets(int maxOps, final Table... tables) throws Exception {
547    int count = 0;
548    try {
549      while (count < maxOps) {
550        Get get = new Get(Bytes.toBytes("row-" + count));
551        for (final Table table: tables) {
552          table.get(get);
553        }
554        count += tables.length;
555      }
556    } catch (RpcThrottlingException e) {
557      LOG.error("get failed after nRetries=" + count, e);
558    }
559    return count;
560  }
561
562  private void triggerUserCacheRefresh(boolean bypass, TableName... tables) throws Exception {
563    triggerCacheRefresh(bypass, true, false, false, tables);
564  }
565
566  private void triggerTableCacheRefresh(boolean bypass, TableName... tables) throws Exception {
567    triggerCacheRefresh(bypass, false, true, false, tables);
568  }
569
570  private void triggerNamespaceCacheRefresh(boolean bypass, TableName... tables) throws Exception {
571    triggerCacheRefresh(bypass, false, false, true, tables);
572  }
573
574  private void triggerCacheRefresh(boolean bypass, boolean userLimiter, boolean tableLimiter,
575      boolean nsLimiter, final TableName... tables) throws Exception {
576    envEdge.incValue(2 * REFRESH_TIME);
577    for (RegionServerThread rst: TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
578      RegionServerRpcQuotaManager quotaManager = rst.getRegionServer().getRegionServerRpcQuotaManager();
579      QuotaCache quotaCache = quotaManager.getQuotaCache();
580
581      quotaCache.triggerCacheRefresh();
582      // sleep for cache update
583      Thread.sleep(250);
584
585      for (TableName table: tables) {
586        quotaCache.getTableLimiter(table);
587      }
588
589      boolean isUpdated = false;
590      while (!isUpdated) {
591        quotaCache.triggerCacheRefresh();
592        isUpdated = true;
593        for (TableName table: tables) {
594          boolean isBypass = true;
595          if (userLimiter) {
596            isBypass &= quotaCache.getUserLimiter(User.getCurrent().getUGI(), table).isBypass();
597          }
598          if (tableLimiter) {
599            isBypass &= quotaCache.getTableLimiter(table).isBypass();
600          }
601          if (nsLimiter) {
602            isBypass &= quotaCache.getNamespaceLimiter(table.getNamespaceAsString()).isBypass();
603          }
604          if (isBypass != bypass) {
605            envEdge.incValue(100);
606            isUpdated = false;
607            break;
608          }
609        }
610      }
611
612      LOG.debug("QuotaCache");
613      LOG.debug(Objects.toString(quotaCache.getNamespaceQuotaCache()));
614      LOG.debug(Objects.toString(quotaCache.getTableQuotaCache()));
615      LOG.debug(Objects.toString(quotaCache.getUserQuotaCache()));
616    }
617  }
618
619  private void waitMinuteQuota() {
620    envEdge.incValue(70000);
621  }
622}