View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
3    * agreements. See the NOTICE file distributed with this work for additional information regarding
4    * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
5    * "License"); you may not use this file except in compliance with the License. You may obtain a
6    * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
7    * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
8    * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
9    * for the specific language governing permissions and limitations under the License.
10   */
11  
12  package org.apache.hadoop.hbase.quotas;
13  
14  import java.io.ByteArrayInputStream;
15  import java.io.ByteArrayOutputStream;
16  import java.io.IOException;
17  import java.util.List;
18  import java.util.Map;
19  import java.util.regex.Pattern;
20  
21  import org.apache.commons.logging.Log;
22  import org.apache.commons.logging.LogFactory;
23  import org.apache.hadoop.hbase.NamespaceDescriptor;
24  import org.apache.hadoop.hbase.TableName;
25  import org.apache.hadoop.hbase.classification.InterfaceAudience;
26  import org.apache.hadoop.hbase.classification.InterfaceStability;
27  import org.apache.hadoop.hbase.client.Connection;
28  import org.apache.hadoop.hbase.client.Get;
29  import org.apache.hadoop.hbase.client.Result;
30  import org.apache.hadoop.hbase.client.Scan;
31  import org.apache.hadoop.hbase.client.Table;
32  import org.apache.hadoop.hbase.filter.CompareFilter;
33  import org.apache.hadoop.hbase.filter.Filter;
34  import org.apache.hadoop.hbase.filter.FilterList;
35  import org.apache.hadoop.hbase.filter.QualifierFilter;
36  import org.apache.hadoop.hbase.filter.RegexStringComparator;
37  import org.apache.hadoop.hbase.filter.RowFilter;
38  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
39  import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Quotas;
40  import org.apache.hadoop.hbase.util.Bytes;
41  import org.apache.hadoop.hbase.util.Strings;
42  
43  /**
44   * Helper class to interact with the quota table.
45   * 
46   * <pre>
47   *     ROW-KEY      FAM/QUAL        DATA
48   *   n.&lt;namespace&gt; q:s         &lt;global-quotas&gt;
49   *   t.&lt;table&gt;     q:s         &lt;global-quotas&gt;
50   *   u.&lt;user&gt;      q:s         &lt;global-quotas&gt;
51   *   u.&lt;user&gt;      q:s.&lt;table&gt; &lt;table-quotas&gt;
52   *   u.&lt;user&gt;      q:s.&lt;ns&gt;:   &lt;namespace-quotas&gt;
53   * </pre>
54   */
55  @InterfaceAudience.Private
56  @InterfaceStability.Evolving
57  public class QuotaTableUtil {
58    private static final Log LOG = LogFactory.getLog(QuotaTableUtil.class);
59  
60    /** System table for quotas */
61    public static final TableName QUOTA_TABLE_NAME = TableName.valueOf(
62      NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, "quota");
63  
64    protected static final byte[] QUOTA_FAMILY_INFO = Bytes.toBytes("q");
65    protected static final byte[] QUOTA_FAMILY_USAGE = Bytes.toBytes("u");
66    protected static final byte[] QUOTA_QUALIFIER_SETTINGS = Bytes.toBytes("s");
67    protected static final byte[] QUOTA_QUALIFIER_SETTINGS_PREFIX = Bytes.toBytes("s.");
68    protected static final byte[] QUOTA_USER_ROW_KEY_PREFIX = Bytes.toBytes("u.");
69    protected static final byte[] QUOTA_TABLE_ROW_KEY_PREFIX = Bytes.toBytes("t.");
70    protected static final byte[] QUOTA_NAMESPACE_ROW_KEY_PREFIX = Bytes.toBytes("n.");
71  
72    /*
73     * ========================================================================= Quota "settings"
74     * helpers
75     */
76    public static Quotas getTableQuota(final Connection connection, final TableName table)
77        throws IOException {
78      return getQuotas(connection, getTableRowKey(table));
79    }
80  
81    public static Quotas getNamespaceQuota(final Connection connection, final String namespace)
82        throws IOException {
83      return getQuotas(connection, getNamespaceRowKey(namespace));
84    }
85  
86    public static Quotas getUserQuota(final Connection connection, final String user)
87        throws IOException {
88      return getQuotas(connection, getUserRowKey(user));
89    }
90  
91    public static Quotas getUserQuota(final Connection connection, final String user,
92        final TableName table) throws IOException {
93      return getQuotas(connection, getUserRowKey(user), getSettingsQualifierForUserTable(table));
94    }
95  
96    public static Quotas getUserQuota(final Connection connection, final String user,
97        final String namespace) throws IOException {
98      return getQuotas(connection, getUserRowKey(user),
99        getSettingsQualifierForUserNamespace(namespace));
100   }
101 
102   private static Quotas getQuotas(final Connection connection, final byte[] rowKey)
103       throws IOException {
104     return getQuotas(connection, rowKey, QUOTA_QUALIFIER_SETTINGS);
105   }
106 
107   private static Quotas getQuotas(final Connection connection, final byte[] rowKey,
108       final byte[] qualifier) throws IOException {
109     Get get = new Get(rowKey);
110     get.addColumn(QUOTA_FAMILY_INFO, qualifier);
111     Result result = doGet(connection, get);
112     if (result.isEmpty()) {
113       return null;
114     }
115     return quotasFromData(result.getValue(QUOTA_FAMILY_INFO, qualifier));
116   }
117 
118   public static Get makeGetForTableQuotas(final TableName table) {
119     Get get = new Get(getTableRowKey(table));
120     get.addFamily(QUOTA_FAMILY_INFO);
121     return get;
122   }
123 
124   public static Get makeGetForNamespaceQuotas(final String namespace) {
125     Get get = new Get(getNamespaceRowKey(namespace));
126     get.addFamily(QUOTA_FAMILY_INFO);
127     return get;
128   }
129 
130   public static Get makeGetForUserQuotas(final String user, final Iterable<TableName> tables,
131       final Iterable<String> namespaces) {
132     Get get = new Get(getUserRowKey(user));
133     get.addColumn(QUOTA_FAMILY_INFO, QUOTA_QUALIFIER_SETTINGS);
134     for (final TableName table : tables) {
135       get.addColumn(QUOTA_FAMILY_INFO, getSettingsQualifierForUserTable(table));
136     }
137     for (final String ns : namespaces) {
138       get.addColumn(QUOTA_FAMILY_INFO, getSettingsQualifierForUserNamespace(ns));
139     }
140     return get;
141   }
142 
143   public static Scan makeScan(final QuotaFilter filter) {
144     Scan scan = new Scan();
145     scan.addFamily(QUOTA_FAMILY_INFO);
146     if (filter != null && !filter.isNull()) {
147       scan.setFilter(makeFilter(filter));
148     }
149     return scan;
150   }
151 
152   /**
153    * converts quotafilter to serializeable filterlists.
154    */
155   public static Filter makeFilter(final QuotaFilter filter) {
156     FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
157     if (!Strings.isEmpty(filter.getUserFilter())) {
158       FilterList userFilters = new FilterList(FilterList.Operator.MUST_PASS_ONE);
159       boolean hasFilter = false;
160 
161       if (!Strings.isEmpty(filter.getNamespaceFilter())) {
162         FilterList nsFilters = new FilterList(FilterList.Operator.MUST_PASS_ALL);
163         nsFilters.addFilter(new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(
164             getUserRowKeyRegex(filter.getUserFilter()), 0)));
165         nsFilters.addFilter(new QualifierFilter(CompareFilter.CompareOp.EQUAL,
166             new RegexStringComparator(getSettingsQualifierRegexForUserNamespace(filter
167                 .getNamespaceFilter()), 0)));
168         userFilters.addFilter(nsFilters);
169         hasFilter = true;
170       }
171       if (!Strings.isEmpty(filter.getTableFilter())) {
172         FilterList tableFilters = new FilterList(FilterList.Operator.MUST_PASS_ALL);
173         tableFilters.addFilter(new RowFilter(CompareFilter.CompareOp.EQUAL,
174             new RegexStringComparator(getUserRowKeyRegex(filter.getUserFilter()), 0)));
175         tableFilters.addFilter(new QualifierFilter(CompareFilter.CompareOp.EQUAL,
176             new RegexStringComparator(
177                 getSettingsQualifierRegexForUserTable(filter.getTableFilter()), 0)));
178         userFilters.addFilter(tableFilters);
179         hasFilter = true;
180       }
181       if (!hasFilter) {
182         userFilters.addFilter(new RowFilter(CompareFilter.CompareOp.EQUAL,
183             new RegexStringComparator(getUserRowKeyRegex(filter.getUserFilter()), 0)));
184       }
185 
186       filterList.addFilter(userFilters);
187     } else if (!Strings.isEmpty(filter.getTableFilter())) {
188       filterList.addFilter(new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(
189           getTableRowKeyRegex(filter.getTableFilter()), 0)));
190     } else if (!Strings.isEmpty(filter.getNamespaceFilter())) {
191       filterList.addFilter(new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(
192           getNamespaceRowKeyRegex(filter.getNamespaceFilter()), 0)));
193     }
194     return filterList;
195   }
196 
197   public static interface UserQuotasVisitor {
198     void visitUserQuotas(final String userName, final Quotas quotas) throws IOException;
199 
200     void visitUserQuotas(final String userName, final TableName table, final Quotas quotas)
201         throws IOException;
202 
203     void visitUserQuotas(final String userName, final String namespace, final Quotas quotas)
204         throws IOException;
205   }
206 
207   public static interface TableQuotasVisitor {
208     void visitTableQuotas(final TableName tableName, final Quotas quotas) throws IOException;
209   }
210 
211   public static interface NamespaceQuotasVisitor {
212     void visitNamespaceQuotas(final String namespace, final Quotas quotas) throws IOException;
213   }
214 
215   public static interface QuotasVisitor extends UserQuotasVisitor, TableQuotasVisitor,
216       NamespaceQuotasVisitor {
217   }
218 
219   public static void parseResult(final Result result, final QuotasVisitor visitor)
220       throws IOException {
221     byte[] row = result.getRow();
222     if (isNamespaceRowKey(row)) {
223       parseNamespaceResult(result, visitor);
224     } else if (isTableRowKey(row)) {
225       parseTableResult(result, visitor);
226     } else if (isUserRowKey(row)) {
227       parseUserResult(result, visitor);
228     } else {
229       LOG.warn("unexpected row-key: " + Bytes.toString(row));
230     }
231   }
232 
233   public static void
234       parseNamespaceResult(final Result result, final NamespaceQuotasVisitor visitor)
235           throws IOException {
236     String namespace = getNamespaceFromRowKey(result.getRow());
237     parseNamespaceResult(namespace, result, visitor);
238   }
239 
240   protected static void parseNamespaceResult(final String namespace, final Result result,
241       final NamespaceQuotasVisitor visitor) throws IOException {
242     byte[] data = result.getValue(QUOTA_FAMILY_INFO, QUOTA_QUALIFIER_SETTINGS);
243     if (data != null) {
244       Quotas quotas = quotasFromData(data);
245       visitor.visitNamespaceQuotas(namespace, quotas);
246     }
247   }
248 
249   public static void parseTableResult(final Result result, final TableQuotasVisitor visitor)
250       throws IOException {
251     TableName table = getTableFromRowKey(result.getRow());
252     parseTableResult(table, result, visitor);
253   }
254 
255   protected static void parseTableResult(final TableName table, final Result result,
256       final TableQuotasVisitor visitor) throws IOException {
257     byte[] data = result.getValue(QUOTA_FAMILY_INFO, QUOTA_QUALIFIER_SETTINGS);
258     if (data != null) {
259       Quotas quotas = quotasFromData(data);
260       visitor.visitTableQuotas(table, quotas);
261     }
262   }
263 
264   public static void parseUserResult(final Result result, final UserQuotasVisitor visitor)
265       throws IOException {
266     String userName = getUserFromRowKey(result.getRow());
267     parseUserResult(userName, result, visitor);
268   }
269 
270   protected static void parseUserResult(final String userName, final Result result,
271       final UserQuotasVisitor visitor) throws IOException {
272     Map<byte[], byte[]> familyMap = result.getFamilyMap(QUOTA_FAMILY_INFO);
273     if (familyMap == null || familyMap.isEmpty()) return;
274 
275     for (Map.Entry<byte[], byte[]> entry : familyMap.entrySet()) {
276       Quotas quotas = quotasFromData(entry.getValue());
277       if (Bytes.startsWith(entry.getKey(), QUOTA_QUALIFIER_SETTINGS_PREFIX)) {
278         String name = Bytes.toString(entry.getKey(), QUOTA_QUALIFIER_SETTINGS_PREFIX.length);
279         if (name.charAt(name.length() - 1) == TableName.NAMESPACE_DELIM) {
280           String namespace = name.substring(0, name.length() - 1);
281           visitor.visitUserQuotas(userName, namespace, quotas);
282         } else {
283           TableName table = TableName.valueOf(name);
284           visitor.visitUserQuotas(userName, table, quotas);
285         }
286       } else if (Bytes.equals(entry.getKey(), QUOTA_QUALIFIER_SETTINGS)) {
287         visitor.visitUserQuotas(userName, quotas);
288       }
289     }
290   }
291 
292   /*
293    * ========================================================================= Quotas protobuf
294    * helpers
295    */
296   protected static Quotas quotasFromData(final byte[] data) throws IOException {
297     int magicLen = ProtobufUtil.lengthOfPBMagic();
298     if (!ProtobufUtil.isPBMagicPrefix(data, 0, magicLen)) {
299       throw new IOException("Missing pb magic prefix");
300     }
301     return Quotas.parseFrom(new ByteArrayInputStream(data, magicLen, data.length - magicLen));
302   }
303 
304   protected static byte[] quotasToData(final Quotas data) throws IOException {
305     ByteArrayOutputStream stream = new ByteArrayOutputStream();
306     stream.write(ProtobufUtil.PB_MAGIC);
307     data.writeTo(stream);
308     return stream.toByteArray();
309   }
310 
311   public static boolean isEmptyQuota(final Quotas quotas) {
312     boolean hasSettings = false;
313     hasSettings |= quotas.hasThrottle();
314     hasSettings |= quotas.hasBypassGlobals();
315     return !hasSettings;
316   }
317 
318   /*
319    * ========================================================================= HTable helpers
320    */
321   protected static Result doGet(final Connection connection, final Get get) throws IOException {
322     try (Table table = connection.getTable(QUOTA_TABLE_NAME)) {
323       return table.get(get);
324     }
325   }
326 
327   protected static Result[] doGet(final Connection connection, final List<Get> gets)
328       throws IOException {
329     try (Table table = connection.getTable(QUOTA_TABLE_NAME)) {
330       return table.get(gets);
331     }
332   }
333 
334   /*
335    * ========================================================================= Quota table row key
336    * helpers
337    */
338   protected static byte[] getUserRowKey(final String user) {
339     return Bytes.add(QUOTA_USER_ROW_KEY_PREFIX, Bytes.toBytes(user));
340   }
341 
342   protected static byte[] getTableRowKey(final TableName table) {
343     return Bytes.add(QUOTA_TABLE_ROW_KEY_PREFIX, table.getName());
344   }
345 
346   protected static byte[] getNamespaceRowKey(final String namespace) {
347     return Bytes.add(QUOTA_NAMESPACE_ROW_KEY_PREFIX, Bytes.toBytes(namespace));
348   }
349 
350   protected static byte[] getSettingsQualifierForUserTable(final TableName tableName) {
351     return Bytes.add(QUOTA_QUALIFIER_SETTINGS_PREFIX, tableName.getName());
352   }
353 
354   protected static byte[] getSettingsQualifierForUserNamespace(final String namespace) {
355     return Bytes.add(QUOTA_QUALIFIER_SETTINGS_PREFIX,
356       Bytes.toBytes(namespace + TableName.NAMESPACE_DELIM));
357   }
358 
359   protected static String getUserRowKeyRegex(final String user) {
360     return getRowKeyRegEx(QUOTA_USER_ROW_KEY_PREFIX, user);
361   }
362 
363   protected static String getTableRowKeyRegex(final String table) {
364     return getRowKeyRegEx(QUOTA_TABLE_ROW_KEY_PREFIX, table);
365   }
366 
367   protected static String getNamespaceRowKeyRegex(final String namespace) {
368     return getRowKeyRegEx(QUOTA_NAMESPACE_ROW_KEY_PREFIX, namespace);
369   }
370 
371   private static String getRowKeyRegEx(final byte[] prefix, final String regex) {
372     return '^' + Pattern.quote(Bytes.toString(prefix)) + regex + '$';
373   }
374 
375   protected static String getSettingsQualifierRegexForUserTable(final String table) {
376     return '^' + Pattern.quote(Bytes.toString(QUOTA_QUALIFIER_SETTINGS_PREFIX)) + table + "(?<!"
377         + Pattern.quote(Character.toString(TableName.NAMESPACE_DELIM)) + ")$";
378   }
379 
380   protected static String getSettingsQualifierRegexForUserNamespace(final String namespace) {
381     return '^' + Pattern.quote(Bytes.toString(QUOTA_QUALIFIER_SETTINGS_PREFIX)) + namespace
382         + Pattern.quote(Character.toString(TableName.NAMESPACE_DELIM)) + '$';
383   }
384 
385   protected static boolean isNamespaceRowKey(final byte[] key) {
386     return Bytes.startsWith(key, QUOTA_NAMESPACE_ROW_KEY_PREFIX);
387   }
388 
389   protected static String getNamespaceFromRowKey(final byte[] key) {
390     return Bytes.toString(key, QUOTA_NAMESPACE_ROW_KEY_PREFIX.length);
391   }
392 
393   protected static boolean isTableRowKey(final byte[] key) {
394     return Bytes.startsWith(key, QUOTA_TABLE_ROW_KEY_PREFIX);
395   }
396 
397   protected static TableName getTableFromRowKey(final byte[] key) {
398     return TableName.valueOf(Bytes.toString(key, QUOTA_TABLE_ROW_KEY_PREFIX.length));
399   }
400 
401   protected static boolean isUserRowKey(final byte[] key) {
402     return Bytes.startsWith(key, QUOTA_USER_ROW_KEY_PREFIX);
403   }
404 
405   protected static String getUserFromRowKey(final byte[] key) {
406     return Bytes.toString(key, QUOTA_USER_ROW_KEY_PREFIX.length);
407   }
408 }