Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.thrift2;
019
020import static java.nio.ByteBuffer.wrap;
021import static org.junit.Assert.assertArrayEquals;
022import static org.junit.Assert.assertEquals;
023import static org.junit.Assert.assertNull;
024import static org.junit.Assert.fail;
025
026import java.io.IOException;
027import java.nio.ByteBuffer;
028import java.security.PrivilegedExceptionAction;
029import java.util.ArrayList;
030import java.util.Arrays;
031import java.util.Collections;
032import java.util.Comparator;
033import java.util.List;
034import org.apache.hadoop.conf.Configuration;
035import org.apache.hadoop.hbase.HBaseClassTestRule;
036import org.apache.hadoop.hbase.HBaseTestingUtil;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.client.Admin;
039import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
040import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
041import org.apache.hadoop.hbase.client.Connection;
042import org.apache.hadoop.hbase.client.ConnectionFactory;
043import org.apache.hadoop.hbase.client.TableDescriptor;
044import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
045import org.apache.hadoop.hbase.security.User;
046import org.apache.hadoop.hbase.security.UserProvider;
047import org.apache.hadoop.hbase.security.visibility.ScanLabelGenerator;
048import org.apache.hadoop.hbase.security.visibility.SimpleScanLabelGenerator;
049import org.apache.hadoop.hbase.security.visibility.VisibilityClient;
050import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
051import org.apache.hadoop.hbase.security.visibility.VisibilityTestUtil;
052import org.apache.hadoop.hbase.security.visibility.VisibilityUtils;
053import org.apache.hadoop.hbase.testclassification.ClientTests;
054import org.apache.hadoop.hbase.testclassification.MediumTests;
055import org.apache.hadoop.hbase.thrift2.generated.TAppend;
056import org.apache.hadoop.hbase.thrift2.generated.TAuthorization;
057import org.apache.hadoop.hbase.thrift2.generated.TCellVisibility;
058import org.apache.hadoop.hbase.thrift2.generated.TColumn;
059import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement;
060import org.apache.hadoop.hbase.thrift2.generated.TColumnValue;
061import org.apache.hadoop.hbase.thrift2.generated.TGet;
062import org.apache.hadoop.hbase.thrift2.generated.TIllegalArgument;
063import org.apache.hadoop.hbase.thrift2.generated.TIncrement;
064import org.apache.hadoop.hbase.thrift2.generated.TPut;
065import org.apache.hadoop.hbase.thrift2.generated.TResult;
066import org.apache.hadoop.hbase.thrift2.generated.TScan;
067import org.apache.hadoop.hbase.util.Bytes;
068import org.junit.AfterClass;
069import org.junit.Assert;
070import org.junit.Before;
071import org.junit.BeforeClass;
072import org.junit.ClassRule;
073import org.junit.Test;
074import org.junit.experimental.categories.Category;
075import org.slf4j.Logger;
076import org.slf4j.LoggerFactory;
077
078import org.apache.hadoop.hbase.shaded.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
079
080@Category({ ClientTests.class, MediumTests.class })
081public class TestThriftHBaseServiceHandlerWithLabels {
082
083  @ClassRule
084  public static final HBaseClassTestRule CLASS_RULE =
085    HBaseClassTestRule.forClass(TestThriftHBaseServiceHandlerWithLabels.class);
086
087  private static final Logger LOG =
088    LoggerFactory.getLogger(TestThriftHBaseServiceHandlerWithLabels.class);
089  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
090
091  // Static names for tables, columns, rows, and values
092  private static byte[] tableAname = Bytes.toBytes("tableA");
093  private static byte[] familyAname = Bytes.toBytes("familyA");
094  private static byte[] familyBname = Bytes.toBytes("familyB");
095  private static byte[] qualifierAname = Bytes.toBytes("qualifierA");
096  private static byte[] qualifierBname = Bytes.toBytes("qualifierB");
097  private static byte[] valueAname = Bytes.toBytes("valueA");
098  private static byte[] valueBname = Bytes.toBytes("valueB");
099  private static ColumnFamilyDescriptor[] families = new ColumnFamilyDescriptor[] {
100    ColumnFamilyDescriptorBuilder.newBuilder(familyAname).setMaxVersions(3).build(),
101    ColumnFamilyDescriptorBuilder.newBuilder(familyBname).setMaxVersions(2).build() };
102
103  private final static String TOPSECRET = "topsecret";
104  private final static String PUBLIC = "public";
105  private final static String PRIVATE = "private";
106  private final static String CONFIDENTIAL = "confidential";
107  private final static String SECRET = "secret";
108  private static User SUPERUSER;
109
110  private static Configuration conf;
111
112  public void assertTColumnValuesEqual(List<TColumnValue> columnValuesA,
113    List<TColumnValue> columnValuesB) {
114    assertEquals(columnValuesA.size(), columnValuesB.size());
115    Comparator<TColumnValue> comparator = new Comparator<TColumnValue>() {
116      @Override
117      public int compare(TColumnValue o1, TColumnValue o2) {
118        return Bytes.compareTo(Bytes.add(o1.getFamily(), o1.getQualifier()),
119          Bytes.add(o2.getFamily(), o2.getQualifier()));
120      }
121    };
122    Collections.sort(columnValuesA, comparator);
123    Collections.sort(columnValuesB, comparator);
124
125    for (int i = 0; i < columnValuesA.size(); i++) {
126      TColumnValue a = columnValuesA.get(i);
127      TColumnValue b = columnValuesB.get(i);
128      assertArrayEquals(a.getFamily(), b.getFamily());
129      assertArrayEquals(a.getQualifier(), b.getQualifier());
130      assertArrayEquals(a.getValue(), b.getValue());
131    }
132  }
133
134  @BeforeClass
135  public static void beforeClass() throws Exception {
136    SUPERUSER = User.createUserForTesting(conf, "admin", new String[] { "supergroup" });
137    conf = UTIL.getConfiguration();
138    conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS, SimpleScanLabelGenerator.class,
139      ScanLabelGenerator.class);
140    conf.set("hbase.superuser", SUPERUSER.getShortName());
141    VisibilityTestUtil.enableVisiblityLabels(conf);
142    UTIL.startMiniCluster(1);
143    // Wait for the labels table to become available
144    UTIL.waitTableEnabled(VisibilityConstants.LABELS_TABLE_NAME.getName(), 50000);
145    createLabels();
146    Admin admin = UTIL.getAdmin();
147    TableDescriptor tableDescriptor = TableDescriptorBuilder
148      .newBuilder(TableName.valueOf(tableAname)).setColumnFamilies(Arrays.asList(families)).build();
149    admin.createTable(tableDescriptor);
150    admin.close();
151    setAuths();
152  }
153
154  private static void createLabels() throws IOException, InterruptedException {
155    PrivilegedExceptionAction<VisibilityLabelsResponse> action =
156      new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
157        @Override
158        public VisibilityLabelsResponse run() throws Exception {
159          String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
160          try (Connection conn = ConnectionFactory.createConnection(conf)) {
161            VisibilityClient.addLabels(conn, labels);
162          } catch (Throwable t) {
163            throw new IOException(t);
164          }
165          return null;
166        }
167      };
168    SUPERUSER.runAs(action);
169  }
170
171  private static void setAuths() throws IOException {
172    String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
173    try {
174      VisibilityClient.setAuths(UTIL.getConnection(), labels, User.getCurrent().getShortName());
175    } catch (Throwable t) {
176      throw new IOException(t);
177    }
178  }
179
180  @AfterClass
181  public static void afterClass() throws Exception {
182    UTIL.shutdownMiniCluster();
183  }
184
185  @Before
186  public void setup() throws Exception {
187
188  }
189
190  private ThriftHBaseServiceHandler createHandler() throws IOException {
191    return new ThriftHBaseServiceHandler(conf, UserProvider.instantiate(conf));
192  }
193
194  @Test
195  public void testScanWithVisibilityLabels() throws Exception {
196    ThriftHBaseServiceHandler handler = createHandler();
197    ByteBuffer table = wrap(tableAname);
198
199    // insert data
200    TColumnValue columnValue =
201      new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
202    List<TColumnValue> columnValues = new ArrayList<>(1);
203    columnValues.add(columnValue);
204    for (int i = 0; i < 10; i++) {
205      TPut put = new TPut(wrap(Bytes.toBytes("testScan" + i)), columnValues);
206      if (i == 5) {
207        put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
208      } else {
209        put.setCellVisibility(new TCellVisibility()
210          .setExpression("(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
211      }
212      handler.put(table, put);
213    }
214
215    // create scan instance
216    TScan scan = new TScan();
217    List<TColumn> columns = new ArrayList<>(1);
218    TColumn column = new TColumn();
219    column.setFamily(familyAname);
220    column.setQualifier(qualifierAname);
221    columns.add(column);
222    scan.setColumns(columns);
223    scan.setStartRow(Bytes.toBytes("testScan"));
224    scan.setStopRow(Bytes.toBytes("testScan\uffff"));
225
226    TAuthorization tauth = new TAuthorization();
227    List<String> labels = new ArrayList<>(2);
228    labels.add(SECRET);
229    labels.add(PRIVATE);
230    tauth.setLabels(labels);
231    scan.setAuthorizations(tauth);
232    // get scanner and rows
233    int scanId = handler.openScanner(table, scan);
234    List<TResult> results = handler.getScannerRows(scanId, 10);
235    assertEquals(9, results.size());
236    Assert.assertFalse(Bytes.equals(results.get(5).getRow(), Bytes.toBytes("testScan" + 5)));
237    for (int i = 0; i < 9; i++) {
238      if (i < 5) {
239        assertArrayEquals(Bytes.toBytes("testScan" + i), results.get(i).getRow());
240      } else if (i == 5) {
241        continue;
242      } else {
243        assertArrayEquals(Bytes.toBytes("testScan" + (i + 1)), results.get(i).getRow());
244      }
245    }
246
247    // check that we are at the end of the scan
248    results = handler.getScannerRows(scanId, 9);
249    assertEquals(0, results.size());
250
251    // close scanner and check that it was indeed closed
252    handler.closeScanner(scanId);
253    try {
254      handler.getScannerRows(scanId, 9);
255      fail("Scanner id should be invalid");
256    } catch (TIllegalArgument e) {
257    }
258  }
259
260  @Test
261  public void testGetScannerResultsWithAuthorizations() throws Exception {
262    ThriftHBaseServiceHandler handler = createHandler();
263    ByteBuffer table = wrap(tableAname);
264
265    // insert data
266    TColumnValue columnValue =
267      new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
268    List<TColumnValue> columnValues = new ArrayList<>(1);
269    columnValues.add(columnValue);
270    for (int i = 0; i < 20; i++) {
271      TPut put =
272        new TPut(wrap(Bytes.toBytes("testGetScannerResults" + pad(i, (byte) 2))), columnValues);
273      if (i == 3) {
274        put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
275      } else {
276        put.setCellVisibility(new TCellVisibility()
277          .setExpression("(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
278      }
279      handler.put(table, put);
280    }
281
282    // create scan instance
283    TScan scan = new TScan();
284    List<TColumn> columns = new ArrayList<>(1);
285    TColumn column = new TColumn();
286    column.setFamily(familyAname);
287    column.setQualifier(qualifierAname);
288    columns.add(column);
289    scan.setColumns(columns);
290    scan.setStartRow(Bytes.toBytes("testGetScannerResults"));
291
292    // get 5 rows and check the returned results
293    scan.setStopRow(Bytes.toBytes("testGetScannerResults05"));
294    TAuthorization tauth = new TAuthorization();
295    List<String> labels = new ArrayList<>(2);
296    labels.add(SECRET);
297    labels.add(PRIVATE);
298    tauth.setLabels(labels);
299    scan.setAuthorizations(tauth);
300    List<TResult> results = handler.getScannerResults(table, scan, 5);
301    assertEquals(4, results.size());
302    for (int i = 0; i < 4; i++) {
303      if (i < 3) {
304        assertArrayEquals(Bytes.toBytes("testGetScannerResults" + pad(i, (byte) 2)),
305          results.get(i).getRow());
306      } else if (i == 3) {
307        continue;
308      } else {
309        assertArrayEquals(Bytes.toBytes("testGetScannerResults" + pad(i + 1, (byte) 2)),
310          results.get(i).getRow());
311      }
312    }
313  }
314
315  @Test
316  public void testGetsWithLabels() throws Exception {
317    ThriftHBaseServiceHandler handler = createHandler();
318    byte[] rowName = Bytes.toBytes("testPutGet");
319    ByteBuffer table = wrap(tableAname);
320
321    List<TColumnValue> columnValues = new ArrayList<>(2);
322    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
323    columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname)));
324    TPut put = new TPut(wrap(rowName), columnValues);
325
326    put.setColumnValues(columnValues);
327    put.setCellVisibility(new TCellVisibility()
328      .setExpression("(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
329    handler.put(table, put);
330    TGet get = new TGet(wrap(rowName));
331    TAuthorization tauth = new TAuthorization();
332    List<String> labels = new ArrayList<>(2);
333    labels.add(SECRET);
334    labels.add(PRIVATE);
335    tauth.setLabels(labels);
336    get.setAuthorizations(tauth);
337    TResult result = handler.get(table, get);
338    assertArrayEquals(rowName, result.getRow());
339    List<TColumnValue> returnedColumnValues = result.getColumnValues();
340    assertTColumnValuesEqual(columnValues, returnedColumnValues);
341  }
342
343  @Test
344  public void testIncrementWithTags() throws Exception {
345    ThriftHBaseServiceHandler handler = createHandler();
346    byte[] rowName = Bytes.toBytes("testIncrementWithTags");
347    ByteBuffer table = wrap(tableAname);
348
349    List<TColumnValue> columnValues = new ArrayList<>(1);
350    columnValues
351      .add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(Bytes.toBytes(1L))));
352    TPut put = new TPut(wrap(rowName), columnValues);
353    put.setColumnValues(columnValues);
354    put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
355    handler.put(table, put);
356
357    List<TColumnIncrement> incrementColumns = new ArrayList<>(1);
358    incrementColumns.add(new TColumnIncrement(wrap(familyAname), wrap(qualifierAname)));
359    TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
360    increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
361    handler.increment(table, increment);
362
363    TGet get = new TGet(wrap(rowName));
364    TAuthorization tauth = new TAuthorization();
365    List<String> labels = new ArrayList<>(1);
366    labels.add(SECRET);
367    tauth.setLabels(labels);
368    get.setAuthorizations(tauth);
369    TResult result = handler.get(table, get);
370
371    assertArrayEquals(rowName, result.getRow());
372    assertEquals(1, result.getColumnValuesSize());
373    TColumnValue columnValue = result.getColumnValues().get(0);
374    assertArrayEquals(Bytes.toBytes(2L), columnValue.getValue());
375  }
376
377  @Test
378  public void testIncrementWithTagsWithNotMatchLabels() throws Exception {
379    ThriftHBaseServiceHandler handler = createHandler();
380    byte[] rowName = Bytes.toBytes("testIncrementWithTagsWithNotMatchLabels");
381    ByteBuffer table = wrap(tableAname);
382
383    List<TColumnValue> columnValues = new ArrayList<>(1);
384    columnValues
385      .add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(Bytes.toBytes(1L))));
386    TPut put = new TPut(wrap(rowName), columnValues);
387    put.setColumnValues(columnValues);
388    put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
389    handler.put(table, put);
390
391    List<TColumnIncrement> incrementColumns = new ArrayList<>(1);
392    incrementColumns.add(new TColumnIncrement(wrap(familyAname), wrap(qualifierAname)));
393    TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
394    increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
395    handler.increment(table, increment);
396
397    TGet get = new TGet(wrap(rowName));
398    TAuthorization tauth = new TAuthorization();
399    List<String> labels = new ArrayList<>(1);
400    labels.add(PUBLIC);
401    tauth.setLabels(labels);
402    get.setAuthorizations(tauth);
403    TResult result = handler.get(table, get);
404    assertNull(result.getRow());
405  }
406
407  @Test
408  public void testAppend() throws Exception {
409    ThriftHBaseServiceHandler handler = createHandler();
410    byte[] rowName = Bytes.toBytes("testAppend");
411    ByteBuffer table = wrap(tableAname);
412    byte[] v1 = Bytes.toBytes(1L);
413    byte[] v2 = Bytes.toBytes(5L);
414    List<TColumnValue> columnValues = new ArrayList<>(1);
415    columnValues
416      .add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(Bytes.toBytes(1L))));
417    TPut put = new TPut(wrap(rowName), columnValues);
418    put.setColumnValues(columnValues);
419    put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
420    handler.put(table, put);
421
422    List<TColumnValue> appendColumns = new ArrayList<>(1);
423    appendColumns.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(v2)));
424    TAppend append = new TAppend(wrap(rowName), appendColumns);
425    append.setCellVisibility(new TCellVisibility().setExpression(SECRET));
426    handler.append(table, append);
427
428    TGet get = new TGet(wrap(rowName));
429    TAuthorization tauth = new TAuthorization();
430    List<String> labels = new ArrayList<>(1);
431    labels.add(SECRET);
432    tauth.setLabels(labels);
433    get.setAuthorizations(tauth);
434    TResult result = handler.get(table, get);
435
436    assertArrayEquals(rowName, result.getRow());
437    assertEquals(1, result.getColumnValuesSize());
438    TColumnValue columnValue = result.getColumnValues().get(0);
439    assertArrayEquals(Bytes.add(v1, v2), columnValue.getValue());
440  }
441
442  /**
443   * Padding numbers to make comparison of sort order easier in a for loop The number to pad.
444   * @param n   The number to pad.
445   * @param pad The length to pad up to.
446   * @return The padded number as a string.
447   */
448  private String pad(int n, byte pad) {
449    String res = Integer.toString(n);
450
451    while (res.length() < pad) {
452      res = "0" + res;
453    }
454
455    return res;
456  }
457}