001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.thrift2;
019
020import static java.nio.ByteBuffer.wrap;
021import static org.junit.Assert.assertArrayEquals;
022import static org.junit.Assert.assertEquals;
023import static org.junit.Assert.assertNull;
024import static org.junit.Assert.fail;
025
026import java.io.IOException;
027import java.nio.ByteBuffer;
028import java.security.PrivilegedExceptionAction;
029import java.util.ArrayList;
030import java.util.Arrays;
031import java.util.Collections;
032import java.util.Comparator;
033import java.util.List;
034import org.apache.hadoop.conf.Configuration;
035import org.apache.hadoop.hbase.HBaseClassTestRule;
036import org.apache.hadoop.hbase.HBaseTestingUtil;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.client.Admin;
039import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
040import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
041import org.apache.hadoop.hbase.client.Connection;
042import org.apache.hadoop.hbase.client.ConnectionFactory;
043import org.apache.hadoop.hbase.client.TableDescriptor;
044import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
045import org.apache.hadoop.hbase.security.User;
046import org.apache.hadoop.hbase.security.UserProvider;
047import org.apache.hadoop.hbase.security.visibility.ScanLabelGenerator;
048import org.apache.hadoop.hbase.security.visibility.SimpleScanLabelGenerator;
049import org.apache.hadoop.hbase.security.visibility.VisibilityClient;
050import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
051import org.apache.hadoop.hbase.security.visibility.VisibilityTestUtil;
052import org.apache.hadoop.hbase.security.visibility.VisibilityUtils;
053import org.apache.hadoop.hbase.testclassification.ClientTests;
054import org.apache.hadoop.hbase.testclassification.MediumTests;
055import org.apache.hadoop.hbase.thrift2.generated.TAppend;
056import org.apache.hadoop.hbase.thrift2.generated.TAuthorization;
057import org.apache.hadoop.hbase.thrift2.generated.TCellVisibility;
058import org.apache.hadoop.hbase.thrift2.generated.TColumn;
059import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement;
060import org.apache.hadoop.hbase.thrift2.generated.TColumnValue;
061import org.apache.hadoop.hbase.thrift2.generated.TGet;
062import org.apache.hadoop.hbase.thrift2.generated.TIllegalArgument;
063import org.apache.hadoop.hbase.thrift2.generated.TIncrement;
064import org.apache.hadoop.hbase.thrift2.generated.TPut;
065import org.apache.hadoop.hbase.thrift2.generated.TResult;
066import org.apache.hadoop.hbase.thrift2.generated.TScan;
067import org.apache.hadoop.hbase.util.Bytes;
068import org.junit.AfterClass;
069import org.junit.Assert;
070import org.junit.Before;
071import org.junit.BeforeClass;
072import org.junit.ClassRule;
073import org.junit.Test;
074import org.junit.experimental.categories.Category;
075import org.slf4j.Logger;
076import org.slf4j.LoggerFactory;
077
078import org.apache.hadoop.hbase.shaded.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
079
080@Category({ClientTests.class, MediumTests.class})
081public class TestThriftHBaseServiceHandlerWithLabels {
082
083  @ClassRule
084  public static final HBaseClassTestRule CLASS_RULE =
085      HBaseClassTestRule.forClass(TestThriftHBaseServiceHandlerWithLabels.class);
086
087  private static final Logger LOG = LoggerFactory
088    .getLogger(TestThriftHBaseServiceHandlerWithLabels.class);
089  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
090
091  // Static names for tables, columns, rows, and values
092  private static byte[] tableAname = Bytes.toBytes("tableA");
093  private static byte[] familyAname = Bytes.toBytes("familyA");
094  private static byte[] familyBname = Bytes.toBytes("familyB");
095  private static byte[] qualifierAname = Bytes.toBytes("qualifierA");
096  private static byte[] qualifierBname = Bytes.toBytes("qualifierB");
097  private static byte[] valueAname = Bytes.toBytes("valueA");
098  private static byte[] valueBname = Bytes.toBytes("valueB");
099  private static ColumnFamilyDescriptor[] families = new ColumnFamilyDescriptor[] {
100    ColumnFamilyDescriptorBuilder.newBuilder(familyAname).setMaxVersions(3).build(),
101    ColumnFamilyDescriptorBuilder.newBuilder(familyBname).setMaxVersions(2).build() };
102
103  private final static String TOPSECRET = "topsecret";
104  private final static String PUBLIC = "public";
105  private final static String PRIVATE = "private";
106  private final static String CONFIDENTIAL = "confidential";
107  private final static String SECRET = "secret";
108  private static User SUPERUSER;
109
110  private static Configuration conf;
111
112  public void assertTColumnValuesEqual(List<TColumnValue> columnValuesA,
113      List<TColumnValue> columnValuesB) {
114    assertEquals(columnValuesA.size(), columnValuesB.size());
115    Comparator<TColumnValue> comparator = new Comparator<TColumnValue>() {
116      @Override
117      public int compare(TColumnValue o1, TColumnValue o2) {
118        return Bytes.compareTo(Bytes.add(o1.getFamily(), o1.getQualifier()),
119            Bytes.add(o2.getFamily(), o2.getQualifier()));
120      }
121    };
122    Collections.sort(columnValuesA, comparator);
123    Collections.sort(columnValuesB, comparator);
124
125    for (int i = 0; i < columnValuesA.size(); i++) {
126      TColumnValue a = columnValuesA.get(i);
127      TColumnValue b = columnValuesB.get(i);
128      assertArrayEquals(a.getFamily(), b.getFamily());
129      assertArrayEquals(a.getQualifier(), b.getQualifier());
130      assertArrayEquals(a.getValue(), b.getValue());
131    }
132  }
133
134  @BeforeClass
135  public static void beforeClass() throws Exception {
136    SUPERUSER = User.createUserForTesting(conf, "admin",
137        new String[] { "supergroup" });
138    conf = UTIL.getConfiguration();
139    conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS,
140        SimpleScanLabelGenerator.class, ScanLabelGenerator.class);
141    conf.set("hbase.superuser", SUPERUSER.getShortName());
142    VisibilityTestUtil.enableVisiblityLabels(conf);
143    UTIL.startMiniCluster(1);
144    // Wait for the labels table to become available
145    UTIL.waitTableEnabled(VisibilityConstants.LABELS_TABLE_NAME.getName(), 50000);
146    createLabels();
147    Admin admin = UTIL.getAdmin();
148    TableDescriptor tableDescriptor = TableDescriptorBuilder
149      .newBuilder(TableName.valueOf(tableAname)).setColumnFamilies(Arrays.asList(families)).build();
150    admin.createTable(tableDescriptor);
151    admin.close();
152    setAuths();
153  }
154
155  private static void createLabels() throws IOException, InterruptedException {
156    PrivilegedExceptionAction<VisibilityLabelsResponse> action =
157        new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
158      @Override
159      public VisibilityLabelsResponse run() throws Exception {
160        String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
161        try (Connection conn = ConnectionFactory.createConnection(conf)) {
162          VisibilityClient.addLabels(conn, labels);
163        } catch (Throwable t) {
164          throw new IOException(t);
165        }
166        return null;
167      }
168    };
169    SUPERUSER.runAs(action);
170  }
171
172  private static void setAuths() throws IOException {
173    String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
174    try {
175      VisibilityClient.setAuths(UTIL.getConnection(), labels, User.getCurrent().getShortName());
176    } catch (Throwable t) {
177      throw new IOException(t);
178    }
179  }
180
181  @AfterClass
182  public static void afterClass() throws Exception {
183    UTIL.shutdownMiniCluster();
184  }
185
186  @Before
187  public void setup() throws Exception {
188
189  }
190
191  private ThriftHBaseServiceHandler createHandler() throws IOException {
192    return new ThriftHBaseServiceHandler(conf, UserProvider.instantiate(conf));
193  }
194
195  @Test
196  public void testScanWithVisibilityLabels() throws Exception {
197    ThriftHBaseServiceHandler handler = createHandler();
198    ByteBuffer table = wrap(tableAname);
199
200    // insert data
201    TColumnValue columnValue = new TColumnValue(wrap(familyAname),
202        wrap(qualifierAname), wrap(valueAname));
203    List<TColumnValue> columnValues = new ArrayList<>(1);
204    columnValues.add(columnValue);
205    for (int i = 0; i < 10; i++) {
206      TPut put = new TPut(wrap(Bytes.toBytes("testScan" + i)), columnValues);
207      if (i == 5) {
208        put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
209      } else {
210        put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET
211            + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
212      }
213      handler.put(table, put);
214    }
215
216    // create scan instance
217    TScan scan = new TScan();
218    List<TColumn> columns = new ArrayList<>(1);
219    TColumn column = new TColumn();
220    column.setFamily(familyAname);
221    column.setQualifier(qualifierAname);
222    columns.add(column);
223    scan.setColumns(columns);
224    scan.setStartRow(Bytes.toBytes("testScan"));
225    scan.setStopRow(Bytes.toBytes("testScan\uffff"));
226
227    TAuthorization tauth = new TAuthorization();
228    List<String> labels = new ArrayList<>(2);
229    labels.add(SECRET);
230    labels.add(PRIVATE);
231    tauth.setLabels(labels);
232    scan.setAuthorizations(tauth);
233    // get scanner and rows
234    int scanId = handler.openScanner(table, scan);
235    List<TResult> results = handler.getScannerRows(scanId, 10);
236    assertEquals(9, results.size());
237    Assert.assertFalse(Bytes.equals(results.get(5).getRow(), Bytes.toBytes("testScan" + 5)));
238    for (int i = 0; i < 9; i++) {
239      if (i < 5) {
240        assertArrayEquals(Bytes.toBytes("testScan" + i), results.get(i).getRow());
241      } else if (i == 5) {
242        continue;
243      } else {
244        assertArrayEquals(Bytes.toBytes("testScan" + (i + 1)), results.get(i)
245            .getRow());
246      }
247    }
248
249    // check that we are at the end of the scan
250    results = handler.getScannerRows(scanId, 9);
251    assertEquals(0, results.size());
252
253    // close scanner and check that it was indeed closed
254    handler.closeScanner(scanId);
255    try {
256      handler.getScannerRows(scanId, 9);
257      fail("Scanner id should be invalid");
258    } catch (TIllegalArgument e) {
259    }
260  }
261
262  @Test
263  public void testGetScannerResultsWithAuthorizations() throws Exception {
264    ThriftHBaseServiceHandler handler = createHandler();
265    ByteBuffer table = wrap(tableAname);
266
267    // insert data
268    TColumnValue columnValue = new TColumnValue(wrap(familyAname),
269        wrap(qualifierAname), wrap(valueAname));
270    List<TColumnValue> columnValues = new ArrayList<>(1);
271    columnValues.add(columnValue);
272    for (int i = 0; i < 20; i++) {
273      TPut put = new TPut(
274          wrap(Bytes.toBytes("testGetScannerResults" + pad(i, (byte) 2))), columnValues);
275      if (i == 3) {
276        put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
277      } else {
278        put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET
279            + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
280      }
281      handler.put(table, put);
282    }
283
284    // create scan instance
285    TScan scan = new TScan();
286    List<TColumn> columns = new ArrayList<>(1);
287    TColumn column = new TColumn();
288    column.setFamily(familyAname);
289    column.setQualifier(qualifierAname);
290    columns.add(column);
291    scan.setColumns(columns);
292    scan.setStartRow(Bytes.toBytes("testGetScannerResults"));
293
294    // get 5 rows and check the returned results
295    scan.setStopRow(Bytes.toBytes("testGetScannerResults05"));
296    TAuthorization tauth = new TAuthorization();
297    List<String> labels = new ArrayList<>(2);
298    labels.add(SECRET);
299    labels.add(PRIVATE);
300    tauth.setLabels(labels);
301    scan.setAuthorizations(tauth);
302    List<TResult> results = handler.getScannerResults(table, scan, 5);
303    assertEquals(4, results.size());
304    for (int i = 0; i < 4; i++) {
305      if (i < 3) {
306        assertArrayEquals(
307            Bytes.toBytes("testGetScannerResults" + pad(i, (byte) 2)), results.get(i).getRow());
308      } else if (i == 3) {
309        continue;
310      } else {
311        assertArrayEquals(
312            Bytes.toBytes("testGetScannerResults" + pad(i + 1, (byte) 2)), results.get(i).getRow());
313      }
314    }
315  }
316
317  @Test
318  public void testGetsWithLabels() throws Exception {
319    ThriftHBaseServiceHandler handler = createHandler();
320    byte[] rowName = Bytes.toBytes("testPutGet");
321    ByteBuffer table = wrap(tableAname);
322
323    List<TColumnValue> columnValues = new ArrayList<>(2);
324    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
325        wrap(valueAname)));
326    columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname),
327        wrap(valueBname)));
328    TPut put = new TPut(wrap(rowName), columnValues);
329
330    put.setColumnValues(columnValues);
331    put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET + "|"
332        + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
333    handler.put(table, put);
334    TGet get = new TGet(wrap(rowName));
335    TAuthorization tauth = new TAuthorization();
336    List<String> labels = new ArrayList<>(2);
337    labels.add(SECRET);
338    labels.add(PRIVATE);
339    tauth.setLabels(labels);
340    get.setAuthorizations(tauth);
341    TResult result = handler.get(table, get);
342    assertArrayEquals(rowName, result.getRow());
343    List<TColumnValue> returnedColumnValues = result.getColumnValues();
344    assertTColumnValuesEqual(columnValues, returnedColumnValues);
345  }
346
347  @Test
348  public void testIncrementWithTags() throws Exception {
349    ThriftHBaseServiceHandler handler = createHandler();
350    byte[] rowName = Bytes.toBytes("testIncrementWithTags");
351    ByteBuffer table = wrap(tableAname);
352
353    List<TColumnValue> columnValues = new ArrayList<>(1);
354    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
355        wrap(Bytes.toBytes(1L))));
356    TPut put = new TPut(wrap(rowName), columnValues);
357    put.setColumnValues(columnValues);
358    put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
359    handler.put(table, put);
360
361    List<TColumnIncrement> incrementColumns = new ArrayList<>(1);
362    incrementColumns.add(new TColumnIncrement(wrap(familyAname),
363        wrap(qualifierAname)));
364    TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
365    increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
366    handler.increment(table, increment);
367
368    TGet get = new TGet(wrap(rowName));
369    TAuthorization tauth = new TAuthorization();
370    List<String> labels = new ArrayList<>(1);
371    labels.add(SECRET);
372    tauth.setLabels(labels);
373    get.setAuthorizations(tauth);
374    TResult result = handler.get(table, get);
375
376    assertArrayEquals(rowName, result.getRow());
377    assertEquals(1, result.getColumnValuesSize());
378    TColumnValue columnValue = result.getColumnValues().get(0);
379    assertArrayEquals(Bytes.toBytes(2L), columnValue.getValue());
380  }
381
382  @Test
383  public void testIncrementWithTagsWithNotMatchLabels() throws Exception {
384    ThriftHBaseServiceHandler handler = createHandler();
385    byte[] rowName = Bytes.toBytes("testIncrementWithTagsWithNotMatchLabels");
386    ByteBuffer table = wrap(tableAname);
387
388    List<TColumnValue> columnValues = new ArrayList<>(1);
389    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
390        wrap(Bytes.toBytes(1L))));
391    TPut put = new TPut(wrap(rowName), columnValues);
392    put.setColumnValues(columnValues);
393    put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
394    handler.put(table, put);
395
396    List<TColumnIncrement> incrementColumns = new ArrayList<>(1);
397    incrementColumns.add(new TColumnIncrement(wrap(familyAname),
398        wrap(qualifierAname)));
399    TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
400    increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
401    handler.increment(table, increment);
402
403    TGet get = new TGet(wrap(rowName));
404    TAuthorization tauth = new TAuthorization();
405    List<String> labels = new ArrayList<>(1);
406    labels.add(PUBLIC);
407    tauth.setLabels(labels);
408    get.setAuthorizations(tauth);
409    TResult result = handler.get(table, get);
410    assertNull(result.getRow());
411  }
412
413  @Test
414  public void testAppend() throws Exception {
415    ThriftHBaseServiceHandler handler = createHandler();
416    byte[] rowName = Bytes.toBytes("testAppend");
417    ByteBuffer table = wrap(tableAname);
418    byte[] v1 = Bytes.toBytes(1L);
419    byte[] v2 = Bytes.toBytes(5L);
420    List<TColumnValue> columnValues = new ArrayList<>(1);
421    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
422        wrap(Bytes.toBytes(1L))));
423    TPut put = new TPut(wrap(rowName), columnValues);
424    put.setColumnValues(columnValues);
425    put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
426    handler.put(table, put);
427
428    List<TColumnValue> appendColumns = new ArrayList<>(1);
429    appendColumns.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
430        wrap(v2)));
431    TAppend append = new TAppend(wrap(rowName), appendColumns);
432    append.setCellVisibility(new TCellVisibility().setExpression(SECRET));
433    handler.append(table, append);
434
435    TGet get = new TGet(wrap(rowName));
436    TAuthorization tauth = new TAuthorization();
437    List<String> labels = new ArrayList<>(1);
438    labels.add(SECRET);
439    tauth.setLabels(labels);
440    get.setAuthorizations(tauth);
441    TResult result = handler.get(table, get);
442
443    assertArrayEquals(rowName, result.getRow());
444    assertEquals(1, result.getColumnValuesSize());
445    TColumnValue columnValue = result.getColumnValues().get(0);
446    assertArrayEquals(Bytes.add(v1, v2), columnValue.getValue());
447  }
448
449  /**
450   * Padding numbers to make comparison of sort order easier in a for loop
451   *
452   * @param n
453   *          The number to pad.
454   * @param pad
455   *          The length to pad up to.
456   * @return The padded number as a string.
457   */
458  private String pad(int n, byte pad) {
459    String res = Integer.toString(n);
460
461    while (res.length() < pad) {
462      res = "0" + res;
463    }
464
465    return res;
466  }
467}