Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.thrift2;
019
020import static java.nio.ByteBuffer.wrap;
021import static org.junit.jupiter.api.Assertions.assertArrayEquals;
022import static org.junit.jupiter.api.Assertions.assertEquals;
023import static org.junit.jupiter.api.Assertions.assertFalse;
024import static org.junit.jupiter.api.Assertions.assertNull;
025import static org.junit.jupiter.api.Assertions.fail;
026
027import java.io.IOException;
028import java.nio.ByteBuffer;
029import java.security.PrivilegedExceptionAction;
030import java.util.ArrayList;
031import java.util.Arrays;
032import java.util.Collections;
033import java.util.Comparator;
034import java.util.List;
035import org.apache.hadoop.conf.Configuration;
036import org.apache.hadoop.hbase.HBaseTestingUtil;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.client.Admin;
039import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
040import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
041import org.apache.hadoop.hbase.client.Connection;
042import org.apache.hadoop.hbase.client.ConnectionFactory;
043import org.apache.hadoop.hbase.client.TableDescriptor;
044import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
045import org.apache.hadoop.hbase.security.User;
046import org.apache.hadoop.hbase.security.UserProvider;
047import org.apache.hadoop.hbase.security.visibility.ScanLabelGenerator;
048import org.apache.hadoop.hbase.security.visibility.SimpleScanLabelGenerator;
049import org.apache.hadoop.hbase.security.visibility.VisibilityClient;
050import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
051import org.apache.hadoop.hbase.security.visibility.VisibilityTestUtil;
052import org.apache.hadoop.hbase.security.visibility.VisibilityUtils;
053import org.apache.hadoop.hbase.testclassification.ClientTests;
054import org.apache.hadoop.hbase.testclassification.MediumTests;
055import org.apache.hadoop.hbase.thrift2.generated.TAppend;
056import org.apache.hadoop.hbase.thrift2.generated.TAuthorization;
057import org.apache.hadoop.hbase.thrift2.generated.TCellVisibility;
058import org.apache.hadoop.hbase.thrift2.generated.TColumn;
059import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement;
060import org.apache.hadoop.hbase.thrift2.generated.TColumnValue;
061import org.apache.hadoop.hbase.thrift2.generated.TDelete;
062import org.apache.hadoop.hbase.thrift2.generated.TGet;
063import org.apache.hadoop.hbase.thrift2.generated.TIllegalArgument;
064import org.apache.hadoop.hbase.thrift2.generated.TIncrement;
065import org.apache.hadoop.hbase.thrift2.generated.TPut;
066import org.apache.hadoop.hbase.thrift2.generated.TResult;
067import org.apache.hadoop.hbase.thrift2.generated.TScan;
068import org.apache.hadoop.hbase.util.Bytes;
069import org.junit.jupiter.api.AfterAll;
070import org.junit.jupiter.api.BeforeAll;
071import org.junit.jupiter.api.BeforeEach;
072import org.junit.jupiter.api.Tag;
073import org.junit.jupiter.api.Test;
074
075import org.apache.hadoop.hbase.shaded.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
076
077@Tag(ClientTests.TAG)
078@Tag(MediumTests.TAG)
079public class TestThriftHBaseServiceHandlerWithLabels {
080
081  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
082
083  // Static names for tables, columns, rows, and values
084  private static byte[] tableAname = Bytes.toBytes("tableA");
085  private static byte[] familyAname = Bytes.toBytes("familyA");
086  private static byte[] familyBname = Bytes.toBytes("familyB");
087  private static byte[] qualifierAname = Bytes.toBytes("qualifierA");
088  private static byte[] qualifierBname = Bytes.toBytes("qualifierB");
089  private static byte[] valueAname = Bytes.toBytes("valueA");
090  private static byte[] valueBname = Bytes.toBytes("valueB");
091  private static ColumnFamilyDescriptor[] families = new ColumnFamilyDescriptor[] {
092    ColumnFamilyDescriptorBuilder.newBuilder(familyAname).setMaxVersions(3).build(),
093    ColumnFamilyDescriptorBuilder.newBuilder(familyBname).setMaxVersions(2).build() };
094
095  private final static String TOPSECRET = "topsecret";
096  private final static String PUBLIC = "public";
097  private final static String PRIVATE = "private";
098  private final static String CONFIDENTIAL = "confidential";
099  private final static String SECRET = "secret";
100  private static User SUPERUSER;
101
102  private static Configuration conf;
103
104  public void assertTColumnValuesEqual(List<TColumnValue> columnValuesA,
105    List<TColumnValue> columnValuesB) {
106    assertEquals(columnValuesA.size(), columnValuesB.size());
107    Comparator<TColumnValue> comparator = new Comparator<TColumnValue>() {
108      @Override
109      public int compare(TColumnValue o1, TColumnValue o2) {
110        return Bytes.compareTo(Bytes.add(o1.getFamily(), o1.getQualifier()),
111          Bytes.add(o2.getFamily(), o2.getQualifier()));
112      }
113    };
114    Collections.sort(columnValuesA, comparator);
115    Collections.sort(columnValuesB, comparator);
116
117    for (int i = 0; i < columnValuesA.size(); i++) {
118      TColumnValue a = columnValuesA.get(i);
119      TColumnValue b = columnValuesB.get(i);
120      assertArrayEquals(a.getFamily(), b.getFamily());
121      assertArrayEquals(a.getQualifier(), b.getQualifier());
122      assertArrayEquals(a.getValue(), b.getValue());
123    }
124  }
125
126  @BeforeAll
127  public static void beforeClass() throws Exception {
128    SUPERUSER = User.createUserForTesting(conf, "admin", new String[] { "supergroup" });
129    conf = UTIL.getConfiguration();
130    conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS, SimpleScanLabelGenerator.class,
131      ScanLabelGenerator.class);
132    conf.set("hbase.superuser", SUPERUSER.getShortName());
133    VisibilityTestUtil.enableVisiblityLabels(conf);
134    UTIL.startMiniCluster(1);
135    // Wait for the labels table to become available
136    UTIL.waitTableEnabled(VisibilityConstants.LABELS_TABLE_NAME.getName(), 50000);
137    createLabels();
138    Admin admin = UTIL.getAdmin();
139    TableDescriptor tableDescriptor = TableDescriptorBuilder
140      .newBuilder(TableName.valueOf(tableAname)).setColumnFamilies(Arrays.asList(families)).build();
141    admin.createTable(tableDescriptor);
142    admin.close();
143    setAuths();
144  }
145
146  private static void createLabels() throws IOException, InterruptedException {
147    PrivilegedExceptionAction<VisibilityLabelsResponse> action =
148      new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
149        @Override
150        public VisibilityLabelsResponse run() throws Exception {
151          String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
152          try (Connection conn = ConnectionFactory.createConnection(conf)) {
153            VisibilityClient.addLabels(conn, labels);
154          } catch (Throwable t) {
155            throw new IOException(t);
156          }
157          return null;
158        }
159      };
160    SUPERUSER.runAs(action);
161  }
162
163  private static void setAuths() throws IOException {
164    String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
165    try {
166      VisibilityClient.setAuths(UTIL.getConnection(), labels, User.getCurrent().getShortName());
167    } catch (Throwable t) {
168      throw new IOException(t);
169    }
170  }
171
172  @AfterAll
173  public static void afterClass() throws Exception {
174    UTIL.shutdownMiniCluster();
175  }
176
177  @BeforeEach
178  public void setup() throws Exception {
179
180  }
181
182  private ThriftHBaseServiceHandler createHandler() throws IOException {
183    return new ThriftHBaseServiceHandler(conf, UserProvider.instantiate(conf));
184  }
185
186  @Test
187  public void testScanWithVisibilityLabels() throws Exception {
188    ThriftHBaseServiceHandler handler = createHandler();
189    ByteBuffer table = wrap(tableAname);
190
191    // insert data
192    TColumnValue columnValue =
193      new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
194    List<TColumnValue> columnValues = new ArrayList<>(1);
195    columnValues.add(columnValue);
196    for (int i = 0; i < 10; i++) {
197      TPut put = new TPut(wrap(Bytes.toBytes("testScan" + i)), columnValues);
198      if (i == 5) {
199        put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
200      } else {
201        put.setCellVisibility(new TCellVisibility()
202          .setExpression("(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
203      }
204      handler.put(table, put);
205    }
206
207    // create scan instance
208    TScan scan = new TScan();
209    List<TColumn> columns = new ArrayList<>(1);
210    TColumn column = new TColumn();
211    column.setFamily(familyAname);
212    column.setQualifier(qualifierAname);
213    columns.add(column);
214    scan.setColumns(columns);
215    scan.setStartRow(Bytes.toBytes("testScan"));
216    scan.setStopRow(Bytes.toBytes("testScan\uffff"));
217
218    TAuthorization tauth = new TAuthorization();
219    List<String> labels = new ArrayList<>(2);
220    labels.add(SECRET);
221    labels.add(PRIVATE);
222    tauth.setLabels(labels);
223    scan.setAuthorizations(tauth);
224    // get scanner and rows
225    int scanId = handler.openScanner(table, scan);
226    List<TResult> results = handler.getScannerRows(scanId, 10);
227    assertEquals(9, results.size());
228    assertFalse(Bytes.equals(results.get(5).getRow(), Bytes.toBytes("testScan" + 5)));
229    for (int i = 0; i < 9; i++) {
230      if (i < 5) {
231        assertArrayEquals(Bytes.toBytes("testScan" + i), results.get(i).getRow());
232      } else if (i == 5) {
233        continue;
234      } else {
235        assertArrayEquals(Bytes.toBytes("testScan" + (i + 1)), results.get(i).getRow());
236      }
237    }
238
239    // check that we are at the end of the scan
240    results = handler.getScannerRows(scanId, 9);
241    assertEquals(0, results.size());
242
243    // close scanner and check that it was indeed closed
244    handler.closeScanner(scanId);
245    try {
246      handler.getScannerRows(scanId, 9);
247      fail("Scanner id should be invalid");
248    } catch (TIllegalArgument e) {
249    }
250  }
251
252  @Test
253  public void testGetScannerResultsWithAuthorizations() throws Exception {
254    ThriftHBaseServiceHandler handler = createHandler();
255    ByteBuffer table = wrap(tableAname);
256
257    // insert data
258    TColumnValue columnValue =
259      new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
260    List<TColumnValue> columnValues = new ArrayList<>(1);
261    columnValues.add(columnValue);
262    for (int i = 0; i < 20; i++) {
263      TPut put =
264        new TPut(wrap(Bytes.toBytes("testGetScannerResults" + pad(i, (byte) 2))), columnValues);
265      if (i == 3) {
266        put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
267      } else {
268        put.setCellVisibility(new TCellVisibility()
269          .setExpression("(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
270      }
271      handler.put(table, put);
272    }
273
274    // create scan instance
275    TScan scan = new TScan();
276    List<TColumn> columns = new ArrayList<>(1);
277    TColumn column = new TColumn();
278    column.setFamily(familyAname);
279    column.setQualifier(qualifierAname);
280    columns.add(column);
281    scan.setColumns(columns);
282    scan.setStartRow(Bytes.toBytes("testGetScannerResults"));
283
284    // get 5 rows and check the returned results
285    scan.setStopRow(Bytes.toBytes("testGetScannerResults05"));
286    TAuthorization tauth = new TAuthorization();
287    List<String> labels = new ArrayList<>(2);
288    labels.add(SECRET);
289    labels.add(PRIVATE);
290    tauth.setLabels(labels);
291    scan.setAuthorizations(tauth);
292    List<TResult> results = handler.getScannerResults(table, scan, 5);
293    assertEquals(4, results.size());
294    for (int i = 0; i < 4; i++) {
295      if (i < 3) {
296        assertArrayEquals(Bytes.toBytes("testGetScannerResults" + pad(i, (byte) 2)),
297          results.get(i).getRow());
298      } else if (i == 3) {
299        continue;
300      } else {
301        assertArrayEquals(Bytes.toBytes("testGetScannerResults" + pad(i + 1, (byte) 2)),
302          results.get(i).getRow());
303      }
304    }
305  }
306
307  @Test
308  public void testGetsWithLabels() throws Exception {
309    ThriftHBaseServiceHandler handler = createHandler();
310    byte[] rowName = Bytes.toBytes("testPutGet");
311    ByteBuffer table = wrap(tableAname);
312
313    List<TColumnValue> columnValues = new ArrayList<>(2);
314    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
315    columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname)));
316    TPut put = new TPut(wrap(rowName), columnValues);
317
318    put.setColumnValues(columnValues);
319    put.setCellVisibility(new TCellVisibility()
320      .setExpression("(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
321    handler.put(table, put);
322    TGet get = new TGet(wrap(rowName));
323    TAuthorization tauth = new TAuthorization();
324    List<String> labels = new ArrayList<>(2);
325    labels.add(SECRET);
326    labels.add(PRIVATE);
327    tauth.setLabels(labels);
328    get.setAuthorizations(tauth);
329    TResult result = handler.get(table, get);
330    assertArrayEquals(rowName, result.getRow());
331    List<TColumnValue> returnedColumnValues = result.getColumnValues();
332    assertTColumnValuesEqual(columnValues, returnedColumnValues);
333  }
334
335  @Test
336  public void testIncrementWithTags() throws Exception {
337    ThriftHBaseServiceHandler handler = createHandler();
338    byte[] rowName = Bytes.toBytes("testIncrementWithTags");
339    ByteBuffer table = wrap(tableAname);
340
341    List<TColumnValue> columnValues = new ArrayList<>(1);
342    columnValues
343      .add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(Bytes.toBytes(1L))));
344    TPut put = new TPut(wrap(rowName), columnValues);
345    put.setColumnValues(columnValues);
346    put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
347    handler.put(table, put);
348
349    List<TColumnIncrement> incrementColumns = new ArrayList<>(1);
350    incrementColumns.add(new TColumnIncrement(wrap(familyAname), wrap(qualifierAname)));
351    TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
352    increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
353    handler.increment(table, increment);
354
355    TGet get = new TGet(wrap(rowName));
356    TAuthorization tauth = new TAuthorization();
357    List<String> labels = new ArrayList<>(1);
358    labels.add(SECRET);
359    tauth.setLabels(labels);
360    get.setAuthorizations(tauth);
361    TResult result = handler.get(table, get);
362
363    assertArrayEquals(rowName, result.getRow());
364    assertEquals(1, result.getColumnValuesSize());
365    TColumnValue columnValue = result.getColumnValues().get(0);
366    assertArrayEquals(Bytes.toBytes(2L), columnValue.getValue());
367  }
368
369  @Test
370  public void testIncrementWithTagsWithNotMatchLabels() throws Exception {
371    ThriftHBaseServiceHandler handler = createHandler();
372    byte[] rowName = Bytes.toBytes("testIncrementWithTagsWithNotMatchLabels");
373    ByteBuffer table = wrap(tableAname);
374
375    List<TColumnValue> columnValues = new ArrayList<>(1);
376    columnValues
377      .add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(Bytes.toBytes(1L))));
378    TPut put = new TPut(wrap(rowName), columnValues);
379    put.setColumnValues(columnValues);
380    put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
381    handler.put(table, put);
382
383    List<TColumnIncrement> incrementColumns = new ArrayList<>(1);
384    incrementColumns.add(new TColumnIncrement(wrap(familyAname), wrap(qualifierAname)));
385    TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
386    increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
387    handler.increment(table, increment);
388
389    TGet get = new TGet(wrap(rowName));
390    TAuthorization tauth = new TAuthorization();
391    List<String> labels = new ArrayList<>(1);
392    labels.add(PUBLIC);
393    tauth.setLabels(labels);
394    get.setAuthorizations(tauth);
395    TResult result = handler.get(table, get);
396    assertNull(result.getRow());
397  }
398
399  @Test
400  public void testAppend() throws Exception {
401    ThriftHBaseServiceHandler handler = createHandler();
402    byte[] rowName = Bytes.toBytes("testAppend");
403    ByteBuffer table = wrap(tableAname);
404    byte[] v1 = Bytes.toBytes(1L);
405    byte[] v2 = Bytes.toBytes(5L);
406    List<TColumnValue> columnValues = new ArrayList<>(1);
407    columnValues
408      .add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(Bytes.toBytes(1L))));
409    TPut put = new TPut(wrap(rowName), columnValues);
410    put.setColumnValues(columnValues);
411    put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
412    handler.put(table, put);
413
414    List<TColumnValue> appendColumns = new ArrayList<>(1);
415    appendColumns.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(v2)));
416    TAppend append = new TAppend(wrap(rowName), appendColumns);
417    append.setCellVisibility(new TCellVisibility().setExpression(SECRET));
418    handler.append(table, append);
419
420    TGet get = new TGet(wrap(rowName));
421    TAuthorization tauth = new TAuthorization();
422    List<String> labels = new ArrayList<>(1);
423    labels.add(SECRET);
424    tauth.setLabels(labels);
425    get.setAuthorizations(tauth);
426    TResult result = handler.get(table, get);
427
428    assertArrayEquals(rowName, result.getRow());
429    assertEquals(1, result.getColumnValuesSize());
430    TColumnValue columnValue = result.getColumnValues().get(0);
431    assertArrayEquals(Bytes.add(v1, v2), columnValue.getValue());
432  }
433
434  @Test
435  public void testDeleteWithLabels() throws Exception {
436    ThriftHBaseServiceHandler handler = createHandler();
437    byte[] rowName = "testPutGetDeleteGet".getBytes();
438    ByteBuffer table = wrap(tableAname);
439
440    // common auths
441    TAuthorization tauth = new TAuthorization();
442    List<String> labels = new ArrayList<String>();
443    labels.add(SECRET);
444    labels.add(PRIVATE);
445    tauth.setLabels(labels);
446
447    // put
448    List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
449    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
450    columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname)));
451    TPut put = new TPut(wrap(rowName), columnValues);
452
453    put.setColumnValues(columnValues);
454    put.setCellVisibility(new TCellVisibility()
455      .setExpression("(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
456    handler.put(table, put);
457
458    // verify put
459    TGet get = new TGet(wrap(rowName));
460    get.setAuthorizations(tauth);
461    TResult result = handler.get(table, get);
462    assertArrayEquals(rowName, result.getRow());
463    List<TColumnValue> returnedColumnValues = result.getColumnValues();
464    assertTColumnValuesEqual(columnValues, returnedColumnValues);
465
466    // delete
467    TDelete delete = new TDelete(wrap(rowName));
468    delete.setCellVisibility(new TCellVisibility()
469      .setExpression("(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
470    handler.deleteSingle(table, delete);
471
472    // verify delete
473    TGet get2 = new TGet(wrap(rowName));
474    get2.setAuthorizations(tauth);
475    TResult result2 = handler.get(table, get2);
476    assertNull(result2.getRow());
477  }
478
479  @Test
480  public void testDeleteWithLabelsNegativeTest() throws Exception {
481    ThriftHBaseServiceHandler handler = createHandler();
482    byte[] rowName = "testPutGetTryDeleteGet".getBytes();
483    ByteBuffer table = wrap(tableAname);
484
485    // common auths
486    TAuthorization tauth = new TAuthorization();
487    List<String> labels = new ArrayList<String>();
488    labels.add(SECRET);
489    labels.add(PRIVATE);
490    tauth.setLabels(labels);
491
492    // put
493    List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
494    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
495    columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname)));
496    TPut put = new TPut(wrap(rowName), columnValues);
497
498    put.setColumnValues(columnValues);
499    put.setCellVisibility(new TCellVisibility()
500      .setExpression("(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
501    handler.put(table, put);
502
503    // verify put
504    TGet get = new TGet(wrap(rowName));
505    get.setAuthorizations(tauth);
506    TResult result = handler.get(table, get);
507    assertArrayEquals(rowName, result.getRow());
508    List<TColumnValue> returnedColumnValues = result.getColumnValues();
509    assertTColumnValuesEqual(columnValues, returnedColumnValues);
510
511    // _try_ delete with _no_ CellVisibility
512    TDelete delete = new TDelete(wrap(rowName));
513    handler.deleteSingle(table, delete);
514
515    // verify delete did in fact _not_ work
516    TGet get2 = new TGet(wrap(rowName));
517    get2.setAuthorizations(tauth);
518    TResult result2 = handler.get(table, get2);
519    assertArrayEquals(rowName, result2.getRow());
520  }
521
522  /**
523   * Padding numbers to make comparison of sort order easier in a for loop The number to pad.
524   * @param n   The number to pad.
525   * @param pad The length to pad up to.
526   * @return The padded number as a string.
527   */
528  private String pad(int n, byte pad) {
529    String res = Integer.toString(n);
530
531    while (res.length() < pad) {
532      res = "0" + res;
533    }
534
535    return res;
536  }
537}