001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.quotas;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.util.Set;
025import java.util.concurrent.TimeUnit;
026import java.util.concurrent.atomic.AtomicLong;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.HBaseTestingUtility;
030import org.apache.hadoop.hbase.HColumnDescriptor;
031import org.apache.hadoop.hbase.HTableDescriptor;
032import org.apache.hadoop.hbase.NamespaceDescriptor;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.client.Admin;
035import org.apache.hadoop.hbase.client.Connection;
036import org.apache.hadoop.hbase.master.HMaster;
037import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
038import org.apache.hadoop.hbase.testclassification.MediumTests;
039import org.junit.AfterClass;
040import org.junit.Before;
041import org.junit.BeforeClass;
042import org.junit.ClassRule;
043import org.junit.Rule;
044import org.junit.Test;
045import org.junit.experimental.categories.Category;
046import org.junit.rules.TestName;
047
048/**
049 * Test class for {@link MasterQuotasObserver}.
050 */
051@Category(MediumTests.class)
052public class TestMasterQuotasObserver {
053
054  @ClassRule
055  public static final HBaseClassTestRule CLASS_RULE =
056    HBaseClassTestRule.forClass(TestMasterQuotasObserver.class);
057
058  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
059  private static SpaceQuotaHelperForTests helper;
060
061  @Rule
062  public TestName testName = new TestName();
063
064  @BeforeClass
065  public static void setUp() throws Exception {
066    Configuration conf = TEST_UTIL.getConfiguration();
067    conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
068    TEST_UTIL.startMiniCluster(1);
069  }
070
071  @AfterClass
072  public static void tearDown() throws Exception {
073    TEST_UTIL.shutdownMiniCluster();
074  }
075
076  @Before
077  public void removeAllQuotas() throws Exception {
078    if (helper == null) {
079      helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, new AtomicLong());
080    }
081    final Connection conn = TEST_UTIL.getConnection();
082    // Wait for the quota table to be created
083    if (!conn.getAdmin().tableExists(QuotaUtil.QUOTA_TABLE_NAME)) {
084      helper.waitForQuotaTable(conn);
085    } else {
086      // Or, clean up any quotas from previous test runs.
087      helper.removeAllQuotas(conn);
088      assertEquals(0, helper.listNumDefinedQuotas(conn));
089    }
090  }
091
092  @Test
093  public void testTableSpaceQuotaRemoved() throws Exception {
094    final Connection conn = TEST_UTIL.getConnection();
095    final Admin admin = conn.getAdmin();
096    final TableName tn = TableName.valueOf(testName.getMethodName());
097    // Drop the table if it somehow exists
098    if (admin.tableExists(tn)) {
099      dropTable(admin, tn);
100    }
101    createTable(admin, tn);
102    assertEquals(0, getNumSpaceQuotas());
103
104    // Set space quota
105    QuotaSettings settings =
106      QuotaSettingsFactory.limitTableSpace(tn, 1024L, SpaceViolationPolicy.NO_INSERTS);
107    admin.setQuota(settings);
108    assertEquals(1, getNumSpaceQuotas());
109
110    // Drop the table and observe the Space quota being automatically deleted as well
111    dropTable(admin, tn);
112    assertEquals(0, getNumSpaceQuotas());
113  }
114
115  @Test
116  public void testTableRPCQuotaRemoved() throws Exception {
117    final Connection conn = TEST_UTIL.getConnection();
118    final Admin admin = conn.getAdmin();
119    final TableName tn = TableName.valueOf(testName.getMethodName());
120    // Drop the table if it somehow exists
121    if (admin.tableExists(tn)) {
122      dropTable(admin, tn);
123    }
124
125    createTable(admin, tn);
126    assertEquals(0, getThrottleQuotas());
127
128    // Set RPC quota
129    QuotaSettings settings =
130      QuotaSettingsFactory.throttleTable(tn, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS);
131    admin.setQuota(settings);
132
133    assertEquals(1, getThrottleQuotas());
134
135    // Delete the table and observe the RPC quota being automatically deleted as well
136    dropTable(admin, tn);
137    assertEquals(0, getThrottleQuotas());
138  }
139
140  @Test
141  public void testTableSpaceAndRPCQuotaRemoved() throws Exception {
142    final Connection conn = TEST_UTIL.getConnection();
143    final Admin admin = conn.getAdmin();
144    final TableName tn = TableName.valueOf(testName.getMethodName());
145    // Drop the table if it somehow exists
146    if (admin.tableExists(tn)) {
147      dropTable(admin, tn);
148    }
149    createTable(admin, tn);
150    assertEquals(0, getNumSpaceQuotas());
151    assertEquals(0, getThrottleQuotas());
152    // Set Both quotas
153    QuotaSettings settings =
154      QuotaSettingsFactory.limitTableSpace(tn, 1024L, SpaceViolationPolicy.NO_INSERTS);
155    admin.setQuota(settings);
156    settings =
157      QuotaSettingsFactory.throttleTable(tn, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS);
158    admin.setQuota(settings);
159
160    assertEquals(1, getNumSpaceQuotas());
161    assertEquals(1, getThrottleQuotas());
162
163    // Remove Space quota
164    settings = QuotaSettingsFactory.removeTableSpaceLimit(tn);
165    admin.setQuota(settings);
166    assertEquals(0, getNumSpaceQuotas());
167    assertEquals(1, getThrottleQuotas());
168
169    // Set back the space quota
170    settings = QuotaSettingsFactory.limitTableSpace(tn, 1024L, SpaceViolationPolicy.NO_INSERTS);
171    admin.setQuota(settings);
172    assertEquals(1, getNumSpaceQuotas());
173    assertEquals(1, getThrottleQuotas());
174
175    // Remove the throttle quota
176    settings = QuotaSettingsFactory.unthrottleTable(tn);
177    admin.setQuota(settings);
178    assertEquals(1, getNumSpaceQuotas());
179    assertEquals(0, getThrottleQuotas());
180
181    // Set back the throttle quota
182    settings =
183      QuotaSettingsFactory.throttleTable(tn, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS);
184    admin.setQuota(settings);
185    assertEquals(1, getNumSpaceQuotas());
186    assertEquals(1, getThrottleQuotas());
187
188    // Drop the table and check that both the quotas have been dropped as well
189    dropTable(admin, tn);
190
191    assertEquals(0, getNumSpaceQuotas());
192    assertEquals(0, getThrottleQuotas());
193  }
194
195  @Test
196  public void testNamespaceSpaceQuotaRemoved() throws Exception {
197    final Connection conn = TEST_UTIL.getConnection();
198    final Admin admin = conn.getAdmin();
199    final String ns = testName.getMethodName();
200    // Drop the ns if it somehow exists
201    if (namespaceExists(ns)) {
202      admin.deleteNamespace(ns);
203    }
204
205    // Create the ns
206    NamespaceDescriptor desc = NamespaceDescriptor.create(ns).build();
207    admin.createNamespace(desc);
208    assertEquals(0, getNumSpaceQuotas());
209
210    // Set a quota
211    QuotaSettings settings =
212      QuotaSettingsFactory.limitNamespaceSpace(ns, 1024L, SpaceViolationPolicy.NO_INSERTS);
213    admin.setQuota(settings);
214    assertEquals(1, getNumSpaceQuotas());
215
216    // Delete the namespace and observe the quota being automatically deleted as well
217    admin.deleteNamespace(ns);
218    assertEquals(0, getNumSpaceQuotas());
219  }
220
221  @Test
222  public void testNamespaceRPCQuotaRemoved() throws Exception {
223    final Connection conn = TEST_UTIL.getConnection();
224    final Admin admin = conn.getAdmin();
225    final String ns = testName.getMethodName();
226    // Drop the ns if it somehow exists
227    if (namespaceExists(ns)) {
228      admin.deleteNamespace(ns);
229    }
230
231    // Create the ns
232    NamespaceDescriptor desc = NamespaceDescriptor.create(ns).build();
233    admin.createNamespace(desc);
234    assertEquals(0, getThrottleQuotas());
235
236    // Set a quota
237    QuotaSettings settings =
238      QuotaSettingsFactory.throttleNamespace(ns, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS);
239    admin.setQuota(settings);
240    assertEquals(1, getThrottleQuotas());
241
242    // Delete the namespace and observe the quota being automatically deleted as well
243    admin.deleteNamespace(ns);
244    assertEquals(0, getThrottleQuotas());
245  }
246
247  @Test
248  public void testNamespaceSpaceAndRPCQuotaRemoved() throws Exception {
249    final Connection conn = TEST_UTIL.getConnection();
250    final Admin admin = conn.getAdmin();
251    final String ns = testName.getMethodName();
252    // Drop the ns if it somehow exists
253    if (namespaceExists(ns)) {
254      admin.deleteNamespace(ns);
255    }
256
257    // Create the ns
258    NamespaceDescriptor desc = NamespaceDescriptor.create(ns).build();
259    admin.createNamespace(desc);
260
261    assertEquals(0, getNumSpaceQuotas());
262    assertEquals(0, getThrottleQuotas());
263
264    // Set Both quotas
265    QuotaSettings settings =
266      QuotaSettingsFactory.limitNamespaceSpace(ns, 1024L, SpaceViolationPolicy.NO_INSERTS);
267    admin.setQuota(settings);
268
269    settings =
270      QuotaSettingsFactory.throttleNamespace(ns, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS);
271    admin.setQuota(settings);
272
273    assertEquals(1, getNumSpaceQuotas());
274    assertEquals(1, getThrottleQuotas());
275
276    // Remove Space quota
277    settings = QuotaSettingsFactory.removeNamespaceSpaceLimit(ns);
278    admin.setQuota(settings);
279    assertEquals(0, getNumSpaceQuotas());
280    assertEquals(1, getThrottleQuotas());
281
282    // Set back the space quota
283    settings = QuotaSettingsFactory.limitNamespaceSpace(ns, 1024L, SpaceViolationPolicy.NO_INSERTS);
284    admin.setQuota(settings);
285    assertEquals(1, getNumSpaceQuotas());
286    assertEquals(1, getThrottleQuotas());
287
288    // Remove the throttle quota
289    settings = QuotaSettingsFactory.unthrottleNamespace(ns);
290    admin.setQuota(settings);
291    assertEquals(1, getNumSpaceQuotas());
292    assertEquals(0, getThrottleQuotas());
293
294    // Set back the throttle quota
295    settings =
296      QuotaSettingsFactory.throttleNamespace(ns, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS);
297    admin.setQuota(settings);
298    assertEquals(1, getNumSpaceQuotas());
299    assertEquals(1, getThrottleQuotas());
300
301    // Delete the namespace and check that both the quotas have been dropped as well
302    admin.deleteNamespace(ns);
303
304    assertEquals(0, getNumSpaceQuotas());
305    assertEquals(0, getThrottleQuotas());
306  }
307
308  @Test
309  public void testObserverAddedByDefault() throws Exception {
310    final HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
311    final MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
312    Set<String> coprocessorNames = cpHost.getCoprocessors();
313    assertTrue("Did not find MasterQuotasObserver in list of CPs: " + coprocessorNames,
314      coprocessorNames.contains(MasterQuotasObserver.class.getSimpleName()));
315  }
316
317  public boolean namespaceExists(String ns) throws IOException {
318    NamespaceDescriptor[] descs = TEST_UTIL.getAdmin().listNamespaceDescriptors();
319    for (NamespaceDescriptor desc : descs) {
320      if (ns.equals(desc.getName())) {
321        return true;
322      }
323    }
324    return false;
325  }
326
327  public int getNumSpaceQuotas() throws Exception {
328    QuotaRetriever scanner = QuotaRetriever.open(TEST_UTIL.getConfiguration());
329    int numSpaceQuotas = 0;
330    for (QuotaSettings quotaSettings : scanner) {
331      if (quotaSettings.getQuotaType() == QuotaType.SPACE) {
332        numSpaceQuotas++;
333      }
334    }
335    return numSpaceQuotas;
336  }
337
338  public int getThrottleQuotas() throws Exception {
339    QuotaRetriever scanner = QuotaRetriever.open(TEST_UTIL.getConfiguration());
340    int throttleQuotas = 0;
341    for (QuotaSettings quotaSettings : scanner) {
342      if (quotaSettings.getQuotaType() == QuotaType.THROTTLE) {
343        throttleQuotas++;
344      }
345    }
346    return throttleQuotas;
347  }
348
349  private void createTable(Admin admin, TableName tn) throws Exception {
350    // Create a table
351    HTableDescriptor tableDesc = new HTableDescriptor(tn);
352    tableDesc.addFamily(new HColumnDescriptor("F1"));
353    admin.createTable(tableDesc);
354  }
355
356  private void dropTable(Admin admin, TableName tn) throws Exception {
357    admin.disableTable(tn);
358    admin.deleteTable(tn);
359  }
360}