Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.quotas;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNull;
022import static org.junit.Assert.fail;
023
024import java.io.IOException;
025import java.security.PrivilegedExceptionAction;
026import java.util.Map;
027import java.util.concurrent.Callable;
028import java.util.concurrent.atomic.AtomicLong;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.hbase.DoNotRetryIOException;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseTestingUtility;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.Waiter;
035import org.apache.hadoop.hbase.Waiter.Predicate;
036import org.apache.hadoop.hbase.client.Admin;
037import org.apache.hadoop.hbase.client.Connection;
038import org.apache.hadoop.hbase.client.ConnectionFactory;
039import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
040import org.apache.hadoop.hbase.regionserver.HRegionServer;
041import org.apache.hadoop.hbase.security.access.AccessControlClient;
042import org.apache.hadoop.hbase.security.access.AccessController;
043import org.apache.hadoop.hbase.security.access.Permission.Action;
044import org.apache.hadoop.hbase.testclassification.MediumTests;
045import org.apache.hadoop.hbase.util.Bytes;
046import org.apache.hadoop.security.UserGroupInformation;
047import org.junit.AfterClass;
048import org.junit.Before;
049import org.junit.BeforeClass;
050import org.junit.ClassRule;
051import org.junit.Rule;
052import org.junit.Test;
053import org.junit.experimental.categories.Category;
054import org.junit.rules.TestName;
055import org.slf4j.Logger;
056import org.slf4j.LoggerFactory;
057
058/**
059 * Test class to verify that the HBase superuser can override quotas.
060 */
061@Category(MediumTests.class)
062public class TestSuperUserQuotaPermissions {
063
064  @ClassRule
065  public static final HBaseClassTestRule CLASS_RULE =
066    HBaseClassTestRule.forClass(TestSuperUserQuotaPermissions.class);
067
068  private static final Logger LOG = LoggerFactory.getLogger(TestSuperUserQuotaPermissions.class);
069  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
070  // Default to the user running the tests
071  private static final String SUPERUSER_NAME = System.getProperty("user.name");
072  private static final UserGroupInformation SUPERUSER_UGI =
073    UserGroupInformation.createUserForTesting(SUPERUSER_NAME, new String[0]);
074  private static final String REGULARUSER_NAME = "quota_regularuser";
075  private static final UserGroupInformation REGULARUSER_UGI =
076    UserGroupInformation.createUserForTesting(REGULARUSER_NAME, new String[0]);
077  private static final AtomicLong COUNTER = new AtomicLong(0);
078
079  @Rule
080  public TestName testName = new TestName();
081  private SpaceQuotaHelperForTests helper;
082
083  @BeforeClass
084  public static void setupMiniCluster() throws Exception {
085    Configuration conf = TEST_UTIL.getConfiguration();
086    // Increase the frequency of some of the chores for responsiveness of the test
087    SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
088
089    conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, AccessController.class.getName());
090    conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, AccessController.class.getName());
091    conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, AccessController.class.getName());
092    conf.setBoolean("hbase.security.exec.permission.checks", true);
093    conf.setBoolean("hbase.security.authorization", true);
094    conf.set("hbase.superuser", SUPERUSER_NAME);
095
096    TEST_UTIL.startMiniCluster(1);
097  }
098
099  @AfterClass
100  public static void tearDown() throws Exception {
101    TEST_UTIL.shutdownMiniCluster();
102  }
103
104  @Before
105  public void removeAllQuotas() throws Exception {
106    final Connection conn = TEST_UTIL.getConnection();
107    if (helper == null) {
108      helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, COUNTER);
109    }
110    // Wait for the quota table to be created
111    if (!conn.getAdmin().tableExists(QuotaUtil.QUOTA_TABLE_NAME)) {
112      helper.waitForQuotaTable(conn);
113    } else {
114      // Or, clean up any quotas from previous test runs.
115      helper.removeAllQuotas(conn);
116      assertEquals(0, helper.listNumDefinedQuotas(conn));
117    }
118  }
119
120  @Test
121  public void testSuperUserCanStillCompact() throws Exception {
122    // Create a table and write enough data to push it into quota violation
123    final TableName tn = doAsSuperUser(new Callable<TableName>() {
124      @Override
125      public TableName call() throws Exception {
126        try (Connection conn = getConnection()) {
127          Admin admin = conn.getAdmin();
128          final TableName tn = helper.createTableWithRegions(admin, 5);
129          // Grant the normal user permissions
130          try {
131            AccessControlClient.grant(conn, tn, REGULARUSER_NAME, null, null, Action.READ,
132              Action.WRITE);
133          } catch (Throwable t) {
134            if (t instanceof Exception) {
135              throw (Exception) t;
136            }
137            throw new Exception(t);
138          }
139          return tn;
140        }
141      }
142    });
143
144    // Write a bunch of data as our end-user
145    doAsRegularUser(new Callable<Void>() {
146      @Override
147      public Void call() throws Exception {
148        try (Connection conn = getConnection()) {
149          // Write way more data so that we have HFiles > numRegions after flushes
150          // helper.writeData flushes itself, so no need to flush explicitly
151          helper.writeData(tn, 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE);
152          helper.writeData(tn, 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE);
153          return null;
154        }
155      }
156    });
157
158    final long sizeLimit = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
159    QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(tn, sizeLimit,
160      SpaceViolationPolicy.NO_WRITES_COMPACTIONS);
161
162    try (Connection conn = getConnection()) {
163      conn.getAdmin().setQuota(settings);
164    }
165
166    waitForTableToEnterQuotaViolation(tn);
167
168    // Should throw an exception, unprivileged users cannot compact due to the quota
169    try {
170      doAsRegularUser(new Callable<Void>() {
171        @Override
172        public Void call() throws Exception {
173          try (Connection conn = getConnection()) {
174            conn.getAdmin().majorCompact(tn);
175            return null;
176          }
177        }
178      });
179      fail("Expected an exception trying to compact a table with a quota violation");
180    } catch (DoNotRetryIOException e) {
181      // Expected Exception.
182      LOG.debug("message", e);
183    }
184
185    try {
186      // Should not throw an exception (superuser can do anything)
187      doAsSuperUser(new Callable<Void>() {
188        @Override
189        public Void call() throws Exception {
190          try (Connection conn = getConnection()) {
191            conn.getAdmin().majorCompact(tn);
192            return null;
193          }
194        }
195      });
196    } catch (Exception e) {
197      // Unexpected Exception.
198      LOG.debug("message", e);
199      fail("Did not expect an exception to be thrown while a super user tries "
200        + "to compact a table with a quota violation");
201    }
202    int numberOfRegions = TEST_UTIL.getAdmin().getRegions(tn).size();
203    waitForHFilesCountLessorEqual(tn, Bytes.toBytes("f1"), numberOfRegions);
204  }
205
206  @Test
207  public void testSuperuserCanRemoveQuota() throws Exception {
208    // Create a table and write enough data to push it into quota violation
209    final TableName tn = doAsSuperUser(new Callable<TableName>() {
210      @Override
211      public TableName call() throws Exception {
212        try (Connection conn = getConnection()) {
213          final Admin admin = conn.getAdmin();
214          final TableName tn = helper.createTableWithRegions(admin, 5);
215          final long sizeLimit = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
216          QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(tn, sizeLimit,
217            SpaceViolationPolicy.NO_WRITES_COMPACTIONS);
218          admin.setQuota(settings);
219          // Grant the normal user permission to create a table and set a quota
220          try {
221            AccessControlClient.grant(conn, tn, REGULARUSER_NAME, null, null, Action.READ,
222              Action.WRITE);
223          } catch (Throwable t) {
224            if (t instanceof Exception) {
225              throw (Exception) t;
226            }
227            throw new Exception(t);
228          }
229          return tn;
230        }
231      }
232    });
233
234    // Write a bunch of data as our end-user
235    doAsRegularUser(new Callable<Void>() {
236      @Override
237      public Void call() throws Exception {
238        try (Connection conn = getConnection()) {
239          helper.writeData(tn, 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE);
240          return null;
241        }
242      }
243    });
244
245    // Wait for the table to hit quota violation
246    waitForTableToEnterQuotaViolation(tn);
247
248    // Try to be "bad" and remove the quota as the end user (we want to write more data!)
249    doAsRegularUser(new Callable<Void>() {
250      @Override
251      public Void call() throws Exception {
252        try (Connection conn = getConnection()) {
253          final Admin admin = conn.getAdmin();
254          QuotaSettings qs = QuotaSettingsFactory.removeTableSpaceLimit(tn);
255          try {
256            admin.setQuota(qs);
257            fail("Expected that an unprivileged user should not be allowed to remove a quota");
258          } catch (Exception e) {
259            // pass
260          }
261          return null;
262        }
263      }
264    });
265
266    // Verify that the superuser can remove the quota
267    doAsSuperUser(new Callable<Void>() {
268      @Override
269      public Void call() throws Exception {
270        try (Connection conn = getConnection()) {
271          final Admin admin = conn.getAdmin();
272          QuotaSettings qs = QuotaSettingsFactory.removeTableSpaceLimit(tn);
273          admin.setQuota(qs);
274          assertNull(helper.getTableSpaceQuota(conn, tn));
275          return null;
276        }
277      }
278    });
279  }
280
281  private Connection getConnection() throws IOException {
282    return ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
283  }
284
285  private <T> T doAsSuperUser(Callable<T> task) throws Exception {
286    return doAsUser(SUPERUSER_UGI, task);
287  }
288
289  private <T> T doAsRegularUser(Callable<T> task) throws Exception {
290    return doAsUser(REGULARUSER_UGI, task);
291  }
292
293  private <T> T doAsUser(UserGroupInformation ugi, Callable<T> task) throws Exception {
294    return ugi.doAs(new PrivilegedExceptionAction<T>() {
295      @Override
296      public T run() throws Exception {
297        return task.call();
298      }
299    });
300  }
301
302  private void waitForTableToEnterQuotaViolation(TableName tn) throws Exception {
303    // Verify that the RegionServer has the quota in violation
304    final HRegionServer rs = TEST_UTIL.getHBaseCluster().getRegionServer(0);
305    Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, 1000, new Predicate<Exception>() {
306      @Override
307      public boolean evaluate() throws Exception {
308        Map<TableName, SpaceQuotaSnapshot> snapshots =
309          rs.getRegionServerSpaceQuotaManager().copyQuotaSnapshots();
310        SpaceQuotaSnapshot snapshot = snapshots.get(tn);
311        if (snapshot == null) {
312          LOG.info("Found no snapshot for " + tn);
313          return false;
314        }
315        LOG.info("Found snapshot " + snapshot);
316        return snapshot.getQuotaStatus().isInViolation();
317      }
318    });
319  }
320
321  private void waitForHFilesCountLessorEqual(TableName tn, byte[] cf, int count) throws Exception {
322    Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, 1000, new Predicate<Exception>() {
323      @Override
324      public boolean evaluate() throws Exception {
325        return TEST_UTIL.getNumHFiles(tn, cf) <= count;
326      }
327    });
328  }
329}