001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.quotas;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertTrue;
022
023import java.io.IOException;
024import java.util.Set;
025import java.util.concurrent.TimeUnit;
026import java.util.concurrent.atomic.AtomicLong;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.HBaseTestingUtil;
029import org.apache.hadoop.hbase.NamespaceDescriptor;
030import org.apache.hadoop.hbase.TableName;
031import org.apache.hadoop.hbase.client.Admin;
032import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
033import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
034import org.apache.hadoop.hbase.client.Connection;
035import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
036import org.apache.hadoop.hbase.master.HMaster;
037import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
038import org.apache.hadoop.hbase.testclassification.MediumTests;
039import org.apache.hadoop.hbase.util.Bytes;
040import org.junit.jupiter.api.AfterAll;
041import org.junit.jupiter.api.BeforeAll;
042import org.junit.jupiter.api.BeforeEach;
043import org.junit.jupiter.api.Tag;
044import org.junit.jupiter.api.Test;
045import org.junit.jupiter.api.TestInfo;
046
047/**
048 * Test class for {@link MasterQuotasObserver}.
049 */
050@Tag(MediumTests.TAG)
051public class TestMasterQuotasObserver {
052
053  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
054  private static SpaceQuotaHelperForTests helper;
055
056  @BeforeAll
057  public static void setUp() throws Exception {
058    Configuration conf = TEST_UTIL.getConfiguration();
059    conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
060    TEST_UTIL.startMiniCluster(1);
061  }
062
063  @AfterAll
064  public static void tearDown() throws Exception {
065    TEST_UTIL.shutdownMiniCluster();
066  }
067
068  @BeforeEach
069  public void removeAllQuotas(TestInfo testInfo) throws Exception {
070    if (helper == null) {
071      helper = new SpaceQuotaHelperForTests(TEST_UTIL,
072        () -> testInfo.getTestMethod().get().getName(), new AtomicLong());
073    }
074    final Connection conn = TEST_UTIL.getConnection();
075    // Wait for the quota table to be created
076    if (!conn.getAdmin().tableExists(QuotaUtil.QUOTA_TABLE_NAME)) {
077      helper.waitForQuotaTable(conn);
078    } else {
079      // Or, clean up any quotas from previous test runs.
080      helper.removeAllQuotas(conn);
081      assertEquals(0, helper.listNumDefinedQuotas(conn));
082    }
083  }
084
085  @Test
086  public void testTableSpaceQuotaRemoved(TestInfo testInfo) throws Exception {
087    final Connection conn = TEST_UTIL.getConnection();
088    final Admin admin = conn.getAdmin();
089    final TableName tn = TableName.valueOf(testInfo.getTestMethod().get().getName());
090    // Drop the table if it somehow exists
091    if (admin.tableExists(tn)) {
092      dropTable(admin, tn);
093    }
094    createTable(admin, tn);
095    assertEquals(0, getNumSpaceQuotas());
096
097    // Set space quota
098    QuotaSettings settings =
099      QuotaSettingsFactory.limitTableSpace(tn, 1024L, SpaceViolationPolicy.NO_INSERTS);
100    admin.setQuota(settings);
101    assertEquals(1, getNumSpaceQuotas());
102
103    // Drop the table and observe the Space quota being automatically deleted as well
104    dropTable(admin, tn);
105    assertEquals(0, getNumSpaceQuotas());
106  }
107
108  @Test
109  public void testTableRPCQuotaRemoved(TestInfo testInfo) throws Exception {
110    final Connection conn = TEST_UTIL.getConnection();
111    final Admin admin = conn.getAdmin();
112    final TableName tn = TableName.valueOf(testInfo.getTestMethod().get().getName());
113    // Drop the table if it somehow exists
114    if (admin.tableExists(tn)) {
115      dropTable(admin, tn);
116    }
117
118    createTable(admin, tn);
119    assertEquals(0, getThrottleQuotas());
120
121    // Set RPC quota
122    QuotaSettings settings =
123      QuotaSettingsFactory.throttleTable(tn, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS);
124    admin.setQuota(settings);
125
126    assertEquals(1, getThrottleQuotas());
127
128    // Delete the table and observe the RPC quota being automatically deleted as well
129    dropTable(admin, tn);
130    assertEquals(0, getThrottleQuotas());
131  }
132
133  @Test
134  public void testTableSpaceAndRPCQuotaRemoved(TestInfo testInfo) throws Exception {
135    final Connection conn = TEST_UTIL.getConnection();
136    final Admin admin = conn.getAdmin();
137    final TableName tn = TableName.valueOf(testInfo.getTestMethod().get().getName());
138    // Drop the table if it somehow exists
139    if (admin.tableExists(tn)) {
140      dropTable(admin, tn);
141    }
142    createTable(admin, tn);
143    assertEquals(0, getNumSpaceQuotas());
144    assertEquals(0, getThrottleQuotas());
145    // Set Both quotas
146    QuotaSettings settings =
147      QuotaSettingsFactory.limitTableSpace(tn, 1024L, SpaceViolationPolicy.NO_INSERTS);
148    admin.setQuota(settings);
149    settings =
150      QuotaSettingsFactory.throttleTable(tn, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS);
151    admin.setQuota(settings);
152
153    assertEquals(1, getNumSpaceQuotas());
154    assertEquals(1, getThrottleQuotas());
155
156    // Remove Space quota
157    settings = QuotaSettingsFactory.removeTableSpaceLimit(tn);
158    admin.setQuota(settings);
159    assertEquals(0, getNumSpaceQuotas());
160    assertEquals(1, getThrottleQuotas());
161
162    // Set back the space quota
163    settings = QuotaSettingsFactory.limitTableSpace(tn, 1024L, SpaceViolationPolicy.NO_INSERTS);
164    admin.setQuota(settings);
165    assertEquals(1, getNumSpaceQuotas());
166    assertEquals(1, getThrottleQuotas());
167
168    // Remove the throttle quota
169    settings = QuotaSettingsFactory.unthrottleTable(tn);
170    admin.setQuota(settings);
171    assertEquals(1, getNumSpaceQuotas());
172    assertEquals(0, getThrottleQuotas());
173
174    // Set back the throttle quota
175    settings =
176      QuotaSettingsFactory.throttleTable(tn, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS);
177    admin.setQuota(settings);
178    assertEquals(1, getNumSpaceQuotas());
179    assertEquals(1, getThrottleQuotas());
180
181    // Drop the table and check that both the quotas have been dropped as well
182    dropTable(admin, tn);
183
184    assertEquals(0, getNumSpaceQuotas());
185    assertEquals(0, getThrottleQuotas());
186  }
187
188  @Test
189  public void testNamespaceSpaceQuotaRemoved(TestInfo testInfo) throws Exception {
190    final Connection conn = TEST_UTIL.getConnection();
191    final Admin admin = conn.getAdmin();
192    final String ns = testInfo.getTestMethod().get().getName();
193    // Drop the ns if it somehow exists
194    if (namespaceExists(ns)) {
195      admin.deleteNamespace(ns);
196    }
197
198    // Create the ns
199    NamespaceDescriptor desc = NamespaceDescriptor.create(ns).build();
200    admin.createNamespace(desc);
201    assertEquals(0, getNumSpaceQuotas());
202
203    // Set a quota
204    QuotaSettings settings =
205      QuotaSettingsFactory.limitNamespaceSpace(ns, 1024L, SpaceViolationPolicy.NO_INSERTS);
206    admin.setQuota(settings);
207    assertEquals(1, getNumSpaceQuotas());
208
209    // Delete the namespace and observe the quota being automatically deleted as well
210    admin.deleteNamespace(ns);
211    assertEquals(0, getNumSpaceQuotas());
212  }
213
214  @Test
215  public void testNamespaceRPCQuotaRemoved(TestInfo testInfo) throws Exception {
216    final Connection conn = TEST_UTIL.getConnection();
217    final Admin admin = conn.getAdmin();
218    final String ns = testInfo.getTestMethod().get().getName();
219    // Drop the ns if it somehow exists
220    if (namespaceExists(ns)) {
221      admin.deleteNamespace(ns);
222    }
223
224    // Create the ns
225    NamespaceDescriptor desc = NamespaceDescriptor.create(ns).build();
226    admin.createNamespace(desc);
227    assertEquals(0, getThrottleQuotas());
228
229    // Set a quota
230    QuotaSettings settings =
231      QuotaSettingsFactory.throttleNamespace(ns, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS);
232    admin.setQuota(settings);
233    assertEquals(1, getThrottleQuotas());
234
235    // Delete the namespace and observe the quota being automatically deleted as well
236    admin.deleteNamespace(ns);
237    assertEquals(0, getThrottleQuotas());
238  }
239
240  @Test
241  public void testNamespaceSpaceAndRPCQuotaRemoved(TestInfo testInfo) throws Exception {
242    final Connection conn = TEST_UTIL.getConnection();
243    final Admin admin = conn.getAdmin();
244    final String ns = testInfo.getTestMethod().get().getName();
245    // Drop the ns if it somehow exists
246    if (namespaceExists(ns)) {
247      admin.deleteNamespace(ns);
248    }
249
250    // Create the ns
251    NamespaceDescriptor desc = NamespaceDescriptor.create(ns).build();
252    admin.createNamespace(desc);
253
254    assertEquals(0, getNumSpaceQuotas());
255    assertEquals(0, getThrottleQuotas());
256
257    // Set Both quotas
258    QuotaSettings settings =
259      QuotaSettingsFactory.limitNamespaceSpace(ns, 1024L, SpaceViolationPolicy.NO_INSERTS);
260    admin.setQuota(settings);
261
262    settings =
263      QuotaSettingsFactory.throttleNamespace(ns, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS);
264    admin.setQuota(settings);
265
266    assertEquals(1, getNumSpaceQuotas());
267    assertEquals(1, getThrottleQuotas());
268
269    // Remove Space quota
270    settings = QuotaSettingsFactory.removeNamespaceSpaceLimit(ns);
271    admin.setQuota(settings);
272    assertEquals(0, getNumSpaceQuotas());
273    assertEquals(1, getThrottleQuotas());
274
275    // Set back the space quota
276    settings = QuotaSettingsFactory.limitNamespaceSpace(ns, 1024L, SpaceViolationPolicy.NO_INSERTS);
277    admin.setQuota(settings);
278    assertEquals(1, getNumSpaceQuotas());
279    assertEquals(1, getThrottleQuotas());
280
281    // Remove the throttle quota
282    settings = QuotaSettingsFactory.unthrottleNamespace(ns);
283    admin.setQuota(settings);
284    assertEquals(1, getNumSpaceQuotas());
285    assertEquals(0, getThrottleQuotas());
286
287    // Set back the throttle quota
288    settings =
289      QuotaSettingsFactory.throttleNamespace(ns, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS);
290    admin.setQuota(settings);
291    assertEquals(1, getNumSpaceQuotas());
292    assertEquals(1, getThrottleQuotas());
293
294    // Delete the namespace and check that both the quotas have been dropped as well
295    admin.deleteNamespace(ns);
296
297    assertEquals(0, getNumSpaceQuotas());
298    assertEquals(0, getThrottleQuotas());
299  }
300
301  @Test
302  public void testObserverAddedByDefault() throws Exception {
303    final HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
304    final MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
305    Set<String> coprocessorNames = cpHost.getCoprocessors();
306    assertTrue(coprocessorNames.contains(MasterQuotasObserver.class.getSimpleName()),
307      "Did not find MasterQuotasObserver in list of CPs: " + coprocessorNames);
308  }
309
310  public boolean namespaceExists(String ns) throws IOException {
311    NamespaceDescriptor[] descs = TEST_UTIL.getAdmin().listNamespaceDescriptors();
312    for (NamespaceDescriptor desc : descs) {
313      if (ns.equals(desc.getName())) {
314        return true;
315      }
316    }
317    return false;
318  }
319
320  public int getNumSpaceQuotas() throws Exception {
321    try (QuotaRetriever scanner = new QuotaRetriever(TEST_UTIL.getConnection())) {
322      int numSpaceQuotas = 0;
323      for (QuotaSettings quotaSettings : scanner) {
324        if (quotaSettings.getQuotaType() == QuotaType.SPACE) {
325          numSpaceQuotas++;
326        }
327      }
328      return numSpaceQuotas;
329    }
330  }
331
332  public int getThrottleQuotas() throws Exception {
333    try (QuotaRetriever scanner = new QuotaRetriever(TEST_UTIL.getConnection())) {
334      int throttleQuotas = 0;
335      for (QuotaSettings quotaSettings : scanner) {
336        if (quotaSettings.getQuotaType() == QuotaType.THROTTLE) {
337          throttleQuotas++;
338        }
339      }
340      return throttleQuotas;
341    }
342  }
343
344  private void createTable(Admin admin, TableName tn) throws Exception {
345    // Create a table
346    TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(tn);
347    ColumnFamilyDescriptor columnFamilyDescriptor =
348      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("F1")).build();
349    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
350    admin.createTable(tableDescriptorBuilder.build());
351  }
352
353  private void dropTable(Admin admin, TableName tn) throws Exception {
354    admin.disableTable(tn);
355    admin.deleteTable(tn);
356  }
357}