Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.security.visibility;
019
020import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_FAMILY;
021import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME;
022import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABEL_QUALIFIER;
023import static org.junit.jupiter.api.Assertions.assertEquals;
024import static org.junit.jupiter.api.Assertions.assertFalse;
025import static org.junit.jupiter.api.Assertions.assertNotNull;
026import static org.junit.jupiter.api.Assertions.assertNull;
027import static org.junit.jupiter.api.Assertions.assertTrue;
028import static org.junit.jupiter.api.Assertions.fail;
029
030import java.io.IOException;
031import java.security.PrivilegedExceptionAction;
032import java.util.ArrayList;
033import java.util.Collection;
034import java.util.List;
035import org.apache.hadoop.conf.Configuration;
036import org.apache.hadoop.hbase.Cell;
037import org.apache.hadoop.hbase.CellScanner;
038import org.apache.hadoop.hbase.HBaseTestingUtil;
039import org.apache.hadoop.hbase.HConstants;
040import org.apache.hadoop.hbase.TableName;
041import org.apache.hadoop.hbase.TableNameTestExtension;
042import org.apache.hadoop.hbase.client.Admin;
043import org.apache.hadoop.hbase.client.Append;
044import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
045import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
046import org.apache.hadoop.hbase.client.Connection;
047import org.apache.hadoop.hbase.client.ConnectionFactory;
048import org.apache.hadoop.hbase.client.Get;
049import org.apache.hadoop.hbase.client.Increment;
050import org.apache.hadoop.hbase.client.Put;
051import org.apache.hadoop.hbase.client.Result;
052import org.apache.hadoop.hbase.client.ResultScanner;
053import org.apache.hadoop.hbase.client.RowMutations;
054import org.apache.hadoop.hbase.client.Scan;
055import org.apache.hadoop.hbase.client.Table;
056import org.apache.hadoop.hbase.client.TableDescriptor;
057import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
058import org.apache.hadoop.hbase.client.security.SecurityCapability;
059import org.apache.hadoop.hbase.regionserver.BloomType;
060import org.apache.hadoop.hbase.regionserver.HRegion;
061import org.apache.hadoop.hbase.regionserver.HRegionServer;
062import org.apache.hadoop.hbase.regionserver.HStore;
063import org.apache.hadoop.hbase.regionserver.HStoreFile;
064import org.apache.hadoop.hbase.security.User;
065import org.apache.hadoop.hbase.util.Bytes;
066import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
067import org.junit.jupiter.api.AfterAll;
068import org.junit.jupiter.api.AfterEach;
069import org.junit.jupiter.api.Test;
070import org.junit.jupiter.api.extension.RegisterExtension;
071
072import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
073
074import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.RegionActionResult;
075import org.apache.hadoop.hbase.shaded.protobuf.generated.VisibilityLabelsProtos.GetAuthsResponse;
076import org.apache.hadoop.hbase.shaded.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
077
078/**
079 * Base test class for visibility labels basic features
080 */
081public abstract class VisibilityLabelsTestBase {
082
083  public static final String TOPSECRET = "topsecret";
084  public static final String PUBLIC = "public";
085  public static final String PRIVATE = "private";
086  public static final String CONFIDENTIAL = "confidential";
087  public static final String SECRET = "secret";
088  public static final String COPYRIGHT = "\u00A9ABC";
089  public static final String ACCENT = "\u0941";
090  public static final String UNICODE_VIS_TAG =
091    COPYRIGHT + "\"" + ACCENT + "\\" + SECRET + "\"" + "\u0027&\\";
092  public static final String UC1 = "\u0027\"\u002b";
093  public static final String UC2 = "\u002d\u003f";
094  public static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
095  public static final byte[] row1 = Bytes.toBytes("row1");
096  public static final byte[] row2 = Bytes.toBytes("row2");
097  public static final byte[] row3 = Bytes.toBytes("row3");
098  public static final byte[] row4 = Bytes.toBytes("row4");
099  public final static byte[] fam = Bytes.toBytes("info");
100  public final static byte[] qual = Bytes.toBytes("qual");
101  public final static byte[] value = Bytes.toBytes("value");
102  public static Configuration conf;
103
104  private volatile boolean killedRS = false;
105  public static User SUPERUSER, USER1;
106
107  @RegisterExtension
108  protected final TableNameTestExtension name = new TableNameTestExtension();
109
110  @AfterAll
111  public static void tearDownAfterClass() throws Exception {
112    TEST_UTIL.shutdownMiniCluster();
113  }
114
115  @AfterEach
116  public void tearDown() throws Exception {
117    killedRS = false;
118  }
119
120  @Test
121  public void testSecurityCapabilities() throws Exception {
122    List<SecurityCapability> capabilities =
123      TEST_UTIL.getConnection().getAdmin().getSecurityCapabilities();
124    assertTrue(capabilities.contains(SecurityCapability.CELL_VISIBILITY),
125      "CELL_VISIBILITY capability is missing");
126  }
127
128  @Test
129  public void testSimpleVisibilityLabels() throws Exception {
130    TableName tableName = name.getTableName();
131    try (Table table = createTableAndWriteDataWithLabels(tableName, SECRET + "|" + CONFIDENTIAL,
132      PRIVATE + "|" + CONFIDENTIAL)) {
133      Scan s = new Scan();
134      s.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL, PRIVATE));
135      ResultScanner scanner = table.getScanner(s);
136      Result[] next = scanner.next(3);
137
138      assertTrue(next.length == 2);
139      CellScanner cellScanner = next[0].cellScanner();
140      cellScanner.advance();
141      Cell current = cellScanner.current();
142      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
143        row1, 0, row1.length));
144      cellScanner = next[1].cellScanner();
145      cellScanner.advance();
146      current = cellScanner.current();
147      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
148        row2, 0, row2.length));
149    }
150  }
151
152  @Test
153  public void testSimpleVisibilityLabelsWithUniCodeCharacters() throws Exception {
154    TableName tableName = name.getTableName();
155    try (Table table = createTableAndWriteDataWithLabels(tableName,
156      SECRET + "|" + CellVisibility.quote(COPYRIGHT), "(" + CellVisibility.quote(COPYRIGHT) + "&"
157        + CellVisibility.quote(ACCENT) + ")|" + CONFIDENTIAL,
158      CellVisibility.quote(UNICODE_VIS_TAG) + "&" + SECRET)) {
159      Scan s = new Scan();
160      s.setAuthorizations(
161        new Authorizations(SECRET, CONFIDENTIAL, PRIVATE, COPYRIGHT, ACCENT, UNICODE_VIS_TAG));
162      ResultScanner scanner = table.getScanner(s);
163      Result[] next = scanner.next(3);
164      assertTrue(next.length == 3);
165      CellScanner cellScanner = next[0].cellScanner();
166      cellScanner.advance();
167      Cell current = cellScanner.current();
168      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
169        row1, 0, row1.length));
170      cellScanner = next[1].cellScanner();
171      cellScanner.advance();
172      current = cellScanner.current();
173      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
174        row2, 0, row2.length));
175      cellScanner = next[2].cellScanner();
176      cellScanner.advance();
177      current = cellScanner.current();
178      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
179        row3, 0, row3.length));
180    }
181  }
182
183  @Test
184  public void testAuthorizationsWithSpecialUnicodeCharacters() throws Exception {
185    TableName tableName = name.getTableName();
186    try (Table table = createTableAndWriteDataWithLabels(tableName,
187      CellVisibility.quote(UC1) + "|" + CellVisibility.quote(UC2), CellVisibility.quote(UC1),
188      CellVisibility.quote(UNICODE_VIS_TAG))) {
189      Scan s = new Scan();
190      s.setAuthorizations(new Authorizations(UC1, UC2, ACCENT, UNICODE_VIS_TAG));
191      ResultScanner scanner = table.getScanner(s);
192      Result[] next = scanner.next(3);
193      assertTrue(next.length == 3);
194      CellScanner cellScanner = next[0].cellScanner();
195      cellScanner.advance();
196      Cell current = cellScanner.current();
197      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
198        row1, 0, row1.length));
199      cellScanner = next[1].cellScanner();
200      cellScanner.advance();
201      current = cellScanner.current();
202      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
203        row2, 0, row2.length));
204      cellScanner = next[2].cellScanner();
205      cellScanner.advance();
206      current = cellScanner.current();
207      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
208        row3, 0, row3.length));
209    }
210  }
211
212  @Test
213  public void testVisibilityLabelsWithComplexLabels() throws Exception {
214    TableName tableName = name.getTableName();
215    try (Table table = createTableAndWriteDataWithLabels(tableName,
216      "(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET,
217      "(" + PRIVATE + "&" + CONFIDENTIAL + "&" + SECRET + ")",
218      "(" + PRIVATE + "&" + CONFIDENTIAL + "&" + SECRET + ")",
219      "(" + PRIVATE + "&" + CONFIDENTIAL + "&" + SECRET + ")")) {
220      Scan s = new Scan();
221      s.setAuthorizations(new Authorizations(TOPSECRET, CONFIDENTIAL, PRIVATE, PUBLIC, SECRET));
222      ResultScanner scanner = table.getScanner(s);
223      Result[] next = scanner.next(4);
224      assertEquals(3, next.length);
225      CellScanner cellScanner = next[0].cellScanner();
226      cellScanner.advance();
227      Cell current = cellScanner.current();
228      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
229        row2, 0, row2.length));
230      cellScanner = next[1].cellScanner();
231      cellScanner.advance();
232      current = cellScanner.current();
233      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
234        row3, 0, row3.length));
235      cellScanner = next[2].cellScanner();
236      cellScanner.advance();
237      current = cellScanner.current();
238      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
239        row4, 0, row4.length));
240    }
241  }
242
243  @Test
244  public void testVisibilityLabelsThatDoesNotPassTheCriteria() throws Exception {
245    TableName tableName = name.getTableName();
246    try (Table table = createTableAndWriteDataWithLabels(tableName,
247      "(" + SECRET + "|" + CONFIDENTIAL + ")", PRIVATE)) {
248      Scan s = new Scan();
249      s.setAuthorizations(new Authorizations(PUBLIC));
250      ResultScanner scanner = table.getScanner(s);
251      Result[] next = scanner.next(3);
252      assertTrue(next.length == 0);
253    }
254  }
255
256  @Test
257  public void testVisibilityLabelsInScanThatDoesNotMatchAnyDefinedLabels() throws Exception {
258    TableName tableName = name.getTableName();
259    try (Table table = createTableAndWriteDataWithLabels(tableName,
260      "(" + SECRET + "|" + CONFIDENTIAL + ")", PRIVATE)) {
261      Scan s = new Scan();
262      s.setAuthorizations(new Authorizations("SAMPLE"));
263      ResultScanner scanner = table.getScanner(s);
264      Result[] next = scanner.next(3);
265      assertTrue(next.length == 0);
266    }
267  }
268
269  @Test
270  public void testVisibilityLabelsWithGet() throws Exception {
271    TableName tableName = name.getTableName();
272    try (Table table = createTableAndWriteDataWithLabels(tableName,
273      SECRET + "&" + CONFIDENTIAL + "&!" + PRIVATE, SECRET + "&" + CONFIDENTIAL + "&" + PRIVATE)) {
274      Get get = new Get(row1);
275      get.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL));
276      Result result = table.get(get);
277      assertTrue(!result.isEmpty());
278      Cell cell = result.getColumnLatestCell(fam, qual);
279      assertTrue(Bytes.equals(value, 0, value.length, cell.getValueArray(), cell.getValueOffset(),
280        cell.getValueLength()));
281    }
282  }
283
284  @Test
285  public void testVisibilityLabelsOnKillingOfRSContainingLabelsTable() throws Exception {
286    List<RegionServerThread> regionServerThreads =
287      TEST_UTIL.getHBaseCluster().getRegionServerThreads();
288    int liveRS = 0;
289    for (RegionServerThread rsThreads : regionServerThreads) {
290      if (!rsThreads.getRegionServer().isAborted()) {
291        liveRS++;
292      }
293    }
294    if (liveRS == 1) {
295      TEST_UTIL.getHBaseCluster().startRegionServer();
296    }
297    Thread t1 = new Thread() {
298      @Override
299      public void run() {
300        List<RegionServerThread> regionServerThreads =
301          TEST_UTIL.getHBaseCluster().getRegionServerThreads();
302        for (RegionServerThread rsThread : regionServerThreads) {
303          List<HRegion> onlineRegions = rsThread.getRegionServer().getRegions(LABELS_TABLE_NAME);
304          if (onlineRegions.size() > 0) {
305            rsThread.getRegionServer().abort("Aborting ");
306            killedRS = true;
307            break;
308          }
309        }
310      }
311
312    };
313    t1.start();
314    final TableName tableName = name.getTableName();
315    Thread t = new Thread() {
316      @Override
317      public void run() {
318        try {
319          while (!killedRS) {
320            Thread.sleep(1);
321          }
322          createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL + ")",
323            PRIVATE);
324        } catch (Exception e) {
325        }
326      }
327    };
328    t.start();
329    regionServerThreads = TEST_UTIL.getHBaseCluster().getRegionServerThreads();
330    while (!killedRS) {
331      Thread.sleep(10);
332    }
333    regionServerThreads = TEST_UTIL.getHBaseCluster().getRegionServerThreads();
334    for (RegionServerThread rsThread : regionServerThreads) {
335      while (true) {
336        if (!rsThread.getRegionServer().isAborted()) {
337          List<HRegion> onlineRegions = rsThread.getRegionServer().getRegions(LABELS_TABLE_NAME);
338          if (onlineRegions.size() > 0) {
339            break;
340          } else {
341            Thread.sleep(10);
342          }
343        } else {
344          break;
345        }
346      }
347    }
348    TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000);
349    t.join();
350    try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
351      Scan s = new Scan();
352      s.setAuthorizations(new Authorizations(SECRET));
353      ResultScanner scanner = table.getScanner(s);
354      Result[] next = scanner.next(3);
355      assertTrue(next.length == 1);
356    }
357  }
358
359  @Test
360  public void testVisibilityLabelsOnRSRestart() throws Exception {
361    final TableName tableName = name.getTableName();
362    List<RegionServerThread> regionServerThreads =
363      TEST_UTIL.getHBaseCluster().getRegionServerThreads();
364    for (RegionServerThread rsThread : regionServerThreads) {
365      rsThread.getRegionServer().abort("Aborting ");
366    }
367    // Start one new RS
368    RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer();
369    waitForLabelsRegionAvailability(rs.getRegionServer());
370    try (Table table = createTableAndWriteDataWithLabels(tableName,
371      "(" + SECRET + "|" + CONFIDENTIAL + ")", PRIVATE)) {
372      Scan s = new Scan();
373      s.setAuthorizations(new Authorizations(SECRET));
374      ResultScanner scanner = table.getScanner(s);
375      Result[] next = scanner.next(3);
376      assertTrue(next.length == 1);
377    }
378  }
379
380  protected void waitForLabelsRegionAvailability(HRegionServer regionServer) {
381    while (!regionServer.isOnline()) {
382      try {
383        Thread.sleep(10);
384      } catch (InterruptedException e) {
385      }
386    }
387    while (regionServer.getRegions(LABELS_TABLE_NAME).isEmpty()) {
388      try {
389        Thread.sleep(10);
390      } catch (InterruptedException e) {
391      }
392    }
393  }
394
395  @Test
396  public void testVisibilityLabelsInGetThatDoesNotMatchAnyDefinedLabels() throws Exception {
397    TableName tableName = name.getTableName();
398    try (Table table = createTableAndWriteDataWithLabels(tableName,
399      "(" + SECRET + "|" + CONFIDENTIAL + ")", PRIVATE)) {
400      Get get = new Get(row1);
401      get.setAuthorizations(new Authorizations("SAMPLE"));
402      Result result = table.get(get);
403      assertTrue(result.isEmpty());
404    }
405  }
406
407  @Test
408  public void testSetAndGetUserAuths() throws Throwable {
409    final String user = "user1";
410    PrivilegedExceptionAction<Void> action = new PrivilegedExceptionAction<Void>() {
411      @Override
412      public Void run() throws Exception {
413        String[] auths = { SECRET, CONFIDENTIAL };
414        try (Connection conn = ConnectionFactory.createConnection(conf)) {
415          VisibilityClient.setAuths(conn, auths, user);
416        } catch (Throwable e) {
417          throw new IOException(e);
418        }
419        return null;
420      }
421    };
422    SUPERUSER.runAs(action);
423    try (Table ht = TEST_UTIL.getConnection().getTable(LABELS_TABLE_NAME)) {
424      Scan scan = new Scan();
425      scan.setAuthorizations(new Authorizations(VisibilityUtils.SYSTEM_LABEL));
426      ResultScanner scanner = ht.getScanner(scan);
427      Result result = null;
428      List<Result> results = new ArrayList<>();
429      while ((result = scanner.next()) != null) {
430        results.add(result);
431      }
432      List<String> auths = extractAuths(user, results);
433      assertTrue(auths.contains(SECRET));
434      assertTrue(auths.contains(CONFIDENTIAL));
435      assertEquals(2, auths.size());
436    }
437
438    action = new PrivilegedExceptionAction<Void>() {
439      @Override
440      public Void run() throws Exception {
441        GetAuthsResponse authsResponse = null;
442        try (Connection conn = ConnectionFactory.createConnection(conf)) {
443          authsResponse = VisibilityClient.getAuths(conn, user);
444        } catch (Throwable e) {
445          throw new IOException(e);
446        }
447        List<String> authsList = new ArrayList<>(authsResponse.getAuthList().size());
448        for (ByteString authBS : authsResponse.getAuthList()) {
449          authsList.add(Bytes.toString(authBS.toByteArray()));
450        }
451        assertEquals(2, authsList.size());
452        assertTrue(authsList.contains(SECRET));
453        assertTrue(authsList.contains(CONFIDENTIAL));
454        return null;
455      }
456    };
457    SUPERUSER.runAs(action);
458
459    // Try doing setAuths once again and there should not be any duplicates
460    action = new PrivilegedExceptionAction<Void>() {
461      @Override
462      public Void run() throws Exception {
463        String[] auths1 = { SECRET, CONFIDENTIAL };
464        GetAuthsResponse authsResponse = null;
465        try (Connection conn = ConnectionFactory.createConnection(conf)) {
466          VisibilityClient.setAuths(conn, auths1, user);
467          try {
468            authsResponse = VisibilityClient.getAuths(conn, user);
469          } catch (Throwable e) {
470            throw new IOException(e);
471          }
472        } catch (Throwable e) {
473        }
474        List<String> authsList = new ArrayList<>(authsResponse.getAuthList().size());
475        for (ByteString authBS : authsResponse.getAuthList()) {
476          authsList.add(Bytes.toString(authBS.toByteArray()));
477        }
478        assertEquals(2, authsList.size());
479        assertTrue(authsList.contains(SECRET));
480        assertTrue(authsList.contains(CONFIDENTIAL));
481        return null;
482      }
483    };
484    SUPERUSER.runAs(action);
485  }
486
487  protected List<String> extractAuths(String user, List<Result> results) {
488    List<String> auths = new ArrayList<>();
489    for (Result result : results) {
490      Cell labelCell = result.getColumnLatestCell(LABELS_TABLE_FAMILY, LABEL_QUALIFIER);
491      Cell userAuthCell = result.getColumnLatestCell(LABELS_TABLE_FAMILY, Bytes.toBytes(user));
492      if (userAuthCell != null) {
493        auths.add(Bytes.toString(labelCell.getValueArray(), labelCell.getValueOffset(),
494          labelCell.getValueLength()));
495      }
496    }
497    return auths;
498  }
499
500  @Test
501  public void testClearUserAuths() throws Throwable {
502    PrivilegedExceptionAction<Void> action = new PrivilegedExceptionAction<Void>() {
503      @Override
504      public Void run() throws Exception {
505        String[] auths = { SECRET, CONFIDENTIAL, PRIVATE };
506        String user = "testUser";
507        try (Connection conn = ConnectionFactory.createConnection(conf)) {
508          VisibilityClient.setAuths(conn, auths, user);
509        } catch (Throwable e) {
510          throw new IOException(e);
511        }
512        // Removing the auths for SECRET and CONFIDENTIAL for the user.
513        // Passing a non existing auth also.
514        auths = new String[] { SECRET, PUBLIC, CONFIDENTIAL };
515        VisibilityLabelsResponse response = null;
516        try (Connection conn = ConnectionFactory.createConnection(conf)) {
517          response = VisibilityClient.clearAuths(conn, auths, user);
518        } catch (Throwable e) {
519          fail("Should not have failed");
520        }
521        List<RegionActionResult> resultList = response.getResultList();
522        assertEquals(3, resultList.size());
523        assertTrue(resultList.get(0).getException().getValue().isEmpty());
524        assertEquals("org.apache.hadoop.hbase.DoNotRetryIOException",
525          resultList.get(1).getException().getName());
526        assertTrue(Bytes.toString(resultList.get(1).getException().getValue().toByteArray())
527          .contains("org.apache.hadoop.hbase.security.visibility.InvalidLabelException: "
528            + "Label 'public' is not set for the user testUser"));
529        assertTrue(resultList.get(2).getException().getValue().isEmpty());
530        try (Connection connection = ConnectionFactory.createConnection(conf);
531          Table ht = connection.getTable(LABELS_TABLE_NAME)) {
532          ResultScanner scanner = ht.getScanner(new Scan());
533          Result result = null;
534          List<Result> results = new ArrayList<>();
535          while ((result = scanner.next()) != null) {
536            results.add(result);
537          }
538          List<String> curAuths = extractAuths(user, results);
539          assertTrue(curAuths.contains(PRIVATE));
540          assertEquals(1, curAuths.size());
541        }
542
543        GetAuthsResponse authsResponse = null;
544        try (Connection conn = ConnectionFactory.createConnection(conf)) {
545          authsResponse = VisibilityClient.getAuths(conn, user);
546        } catch (Throwable e) {
547          throw new IOException(e);
548        }
549        List<String> authsList = new ArrayList<>(authsResponse.getAuthList().size());
550        for (ByteString authBS : authsResponse.getAuthList()) {
551          authsList.add(Bytes.toString(authBS.toByteArray()));
552        }
553        assertEquals(1, authsList.size());
554        assertTrue(authsList.contains(PRIVATE));
555        return null;
556      }
557    };
558    SUPERUSER.runAs(action);
559  }
560
561  @Test
562  public void testLabelsWithCheckAndPut() throws Throwable {
563    TableName tableName = name.getTableName();
564    try (Table table = TEST_UTIL.createTable(tableName, fam)) {
565      byte[] row1 = Bytes.toBytes("row1");
566      Put put = new Put(row1);
567      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value);
568      put.setCellVisibility(new CellVisibility(SECRET + " & " + CONFIDENTIAL));
569      table.checkAndMutate(row1, fam).qualifier(qual).ifNotExists().thenPut(put);
570      byte[] row2 = Bytes.toBytes("row2");
571      put = new Put(row2);
572      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value);
573      put.setCellVisibility(new CellVisibility(SECRET));
574      table.checkAndMutate(row2, fam).qualifier(qual).ifNotExists().thenPut(put);
575
576      Scan scan = new Scan();
577      scan.setAuthorizations(new Authorizations(SECRET));
578      ResultScanner scanner = table.getScanner(scan);
579      Result result = scanner.next();
580      assertTrue(!result.isEmpty());
581      assertTrue(Bytes.equals(row2, result.getRow()));
582      result = scanner.next();
583      assertNull(result);
584    }
585  }
586
587  @Test
588  public void testLabelsWithIncrement() throws Throwable {
589    TableName tableName = name.getTableName();
590    try (Table table = TEST_UTIL.createTable(tableName, fam)) {
591      byte[] row1 = Bytes.toBytes("row1");
592      byte[] val = Bytes.toBytes(1L);
593      Put put = new Put(row1);
594      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, val);
595      put.setCellVisibility(new CellVisibility(SECRET + " & " + CONFIDENTIAL));
596      table.put(put);
597      Get get = new Get(row1);
598      get.setAuthorizations(new Authorizations(SECRET));
599      Result result = table.get(get);
600      assertTrue(result.isEmpty());
601      table.incrementColumnValue(row1, fam, qual, 2L);
602      result = table.get(get);
603      assertTrue(result.isEmpty());
604      Increment increment = new Increment(row1);
605      increment.addColumn(fam, qual, 2L);
606      increment.setCellVisibility(new CellVisibility(SECRET));
607      table.increment(increment);
608      result = table.get(get);
609      assertTrue(!result.isEmpty());
610    }
611  }
612
613  @Test
614  public void testLabelsWithAppend() throws Throwable {
615    TableName tableName = name.getTableName();
616    try (Table table = TEST_UTIL.createTable(tableName, fam)) {
617      byte[] row1 = Bytes.toBytes("row1");
618      byte[] val = Bytes.toBytes("a");
619      Put put = new Put(row1);
620      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, val);
621      put.setCellVisibility(new CellVisibility(SECRET + " & " + CONFIDENTIAL));
622      table.put(put);
623      Get get = new Get(row1);
624      get.setAuthorizations(new Authorizations(SECRET));
625      Result result = table.get(get);
626      assertTrue(result.isEmpty());
627      Append append = new Append(row1);
628      append.addColumn(fam, qual, Bytes.toBytes("b"));
629      table.append(append);
630      result = table.get(get);
631      assertTrue(result.isEmpty());
632      append = new Append(row1);
633      append.addColumn(fam, qual, Bytes.toBytes("c"));
634      append.setCellVisibility(new CellVisibility(SECRET));
635      table.append(append);
636      result = table.get(get);
637      assertTrue(!result.isEmpty());
638    }
639  }
640
641  @Test
642  public void testUserShouldNotDoDDLOpOnLabelsTable() throws Exception {
643    Admin admin = TEST_UTIL.getAdmin();
644    try {
645      admin.disableTable(LABELS_TABLE_NAME);
646      fail("Lables table should not get disabled by user.");
647    } catch (Exception e) {
648    }
649    try {
650      admin.deleteTable(LABELS_TABLE_NAME);
651      fail("Lables table should not get disabled by user.");
652    } catch (Exception e) {
653    }
654    try {
655      ColumnFamilyDescriptor columnFamilyDescriptor =
656        ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("testFamily")).build();
657      admin.addColumnFamily(LABELS_TABLE_NAME, columnFamilyDescriptor);
658      fail("Lables table should not get altered by user.");
659    } catch (Exception e) {
660    }
661    try {
662      admin.deleteColumnFamily(LABELS_TABLE_NAME, VisibilityConstants.LABELS_TABLE_FAMILY);
663      fail("Lables table should not get altered by user.");
664    } catch (Exception e) {
665    }
666    try {
667      ColumnFamilyDescriptor familyDescriptor =
668        ColumnFamilyDescriptorBuilder.newBuilder(VisibilityConstants.LABELS_TABLE_FAMILY)
669          .setBloomFilterType(BloomType.ROWCOL).build();
670      admin.modifyColumnFamily(LABELS_TABLE_NAME, familyDescriptor);
671      fail("Lables table should not get altered by user.");
672    } catch (Exception e) {
673    }
674    try {
675      TableDescriptorBuilder tableDescriptorBuilder =
676        TableDescriptorBuilder.newBuilder(LABELS_TABLE_NAME);
677      ColumnFamilyDescriptor columnFamilyDescriptor =
678        ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("f1")).build();
679      tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
680      columnFamilyDescriptor =
681        ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("f2")).build();
682      tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
683      admin.modifyTable(tableDescriptorBuilder.build());
684      fail("Lables table should not get altered by user.");
685    } catch (Exception e) {
686    }
687  }
688
689  @Test
690  public void testMultipleVersions() throws Exception {
691    final byte[] r1 = Bytes.toBytes("row1");
692    final byte[] r2 = Bytes.toBytes("row2");
693    final byte[] v1 = Bytes.toBytes("100");
694    final byte[] v2 = Bytes.toBytes("101");
695    final byte[] fam2 = Bytes.toBytes("info2");
696    final byte[] qual2 = Bytes.toBytes("qual2");
697    TableName tableName = name.getTableName();
698    // Default max versions is 1.
699    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
700      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam))
701      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam2).setMaxVersions(5).build())
702      .build();
703    TEST_UTIL.getAdmin().createTable(tableDescriptor);
704    try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
705      Put put = new Put(r1);
706      put.addColumn(fam, qual, 3L, v1);
707      put.addColumn(fam, qual2, 3L, v1);
708      put.addColumn(fam2, qual, 3L, v1);
709      put.addColumn(fam2, qual2, 3L, v1);
710      put.setCellVisibility(new CellVisibility(SECRET));
711      table.put(put);
712      put = new Put(r1);
713      put.addColumn(fam, qual, 4L, v2);
714      put.addColumn(fam, qual2, 4L, v2);
715      put.addColumn(fam2, qual, 4L, v2);
716      put.addColumn(fam2, qual2, 4L, v2);
717      put.setCellVisibility(new CellVisibility(PRIVATE));
718      table.put(put);
719
720      put = new Put(r2);
721      put.addColumn(fam, qual, 3L, v1);
722      put.addColumn(fam, qual2, 3L, v1);
723      put.addColumn(fam2, qual, 3L, v1);
724      put.addColumn(fam2, qual2, 3L, v1);
725      put.setCellVisibility(new CellVisibility(SECRET));
726      table.put(put);
727      put = new Put(r2);
728      put.addColumn(fam, qual, 4L, v2);
729      put.addColumn(fam, qual2, 4L, v2);
730      put.addColumn(fam2, qual, 4L, v2);
731      put.addColumn(fam2, qual2, 4L, v2);
732      put.setCellVisibility(new CellVisibility(SECRET));
733      table.put(put);
734
735      Scan s = new Scan();
736      s.readVersions(1);
737      s.setAuthorizations(new Authorizations(SECRET));
738      ResultScanner scanner = table.getScanner(s);
739      Result result = scanner.next();
740      assertTrue(Bytes.equals(r1, result.getRow()));
741      // for cf 'fam' max versions in HCD is 1. So the old version cells, which are having matching
742      // CellVisibility with Authorizations, should not get considered in the label evaluation at
743      // all.
744      assertNull(result.getColumnLatestCell(fam, qual));
745      assertNull(result.getColumnLatestCell(fam, qual2));
746      // for cf 'fam2' max versions in HCD is > 1. So we can consider the old version cells, which
747      // are having matching CellVisibility with Authorizations, in the label evaluation. It can
748      // just skip those recent versions for which visibility is not there as per the new version's
749      // CellVisibility. The old versions which are having visibility can be send back
750      Cell cell = result.getColumnLatestCell(fam2, qual);
751      assertNotNull(cell);
752      assertTrue(Bytes.equals(v1, 0, v1.length, cell.getValueArray(), cell.getValueOffset(),
753        cell.getValueLength()));
754      cell = result.getColumnLatestCell(fam2, qual2);
755      assertNotNull(cell);
756      assertTrue(Bytes.equals(v1, 0, v1.length, cell.getValueArray(), cell.getValueOffset(),
757        cell.getValueLength()));
758
759      result = scanner.next();
760      assertTrue(Bytes.equals(r2, result.getRow()));
761      cell = result.getColumnLatestCell(fam, qual);
762      assertNotNull(cell);
763      assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
764        cell.getValueLength()));
765      cell = result.getColumnLatestCell(fam, qual2);
766      assertNotNull(cell);
767      assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
768        cell.getValueLength()));
769      cell = result.getColumnLatestCell(fam2, qual);
770      assertNotNull(cell);
771      assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
772        cell.getValueLength()));
773      cell = result.getColumnLatestCell(fam2, qual2);
774      assertNotNull(cell);
775      assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
776        cell.getValueLength()));
777    }
778  }
779
780  @Test
781  public void testMutateRow() throws Exception {
782    final byte[] qual2 = Bytes.toBytes("qual2");
783    TableName tableName = name.getTableName();
784    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
785      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam)).build();
786    TEST_UTIL.getAdmin().createTable(tableDescriptor);
787    try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
788      Put p1 = new Put(row1);
789      p1.addColumn(fam, qual, value);
790      p1.setCellVisibility(new CellVisibility(CONFIDENTIAL));
791
792      Put p2 = new Put(row1);
793      p2.addColumn(fam, qual2, value);
794      p2.setCellVisibility(new CellVisibility(SECRET));
795
796      RowMutations rm = new RowMutations(row1);
797      rm.add(p1);
798      rm.add(p2);
799
800      table.mutateRow(rm);
801
802      Get get = new Get(row1);
803      get.setAuthorizations(new Authorizations(CONFIDENTIAL));
804      Result result = table.get(get);
805      assertTrue(result.containsColumn(fam, qual));
806      assertFalse(result.containsColumn(fam, qual2));
807
808      get.setAuthorizations(new Authorizations(SECRET));
809      result = table.get(get);
810      assertFalse(result.containsColumn(fam, qual));
811      assertTrue(result.containsColumn(fam, qual2));
812    }
813  }
814
815  @Test
816  public void testFlushedFileWithVisibilityTags() throws Exception {
817    final byte[] qual2 = Bytes.toBytes("qual2");
818    TableName tableName = name.getTableName();
819    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
820      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam)).build();
821    TEST_UTIL.getAdmin().createTable(tableDescriptor);
822    try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
823      Put p1 = new Put(row1);
824      p1.addColumn(fam, qual, value);
825      p1.setCellVisibility(new CellVisibility(CONFIDENTIAL));
826
827      Put p2 = new Put(row1);
828      p2.addColumn(fam, qual2, value);
829      p2.setCellVisibility(new CellVisibility(SECRET));
830
831      RowMutations rm = new RowMutations(row1);
832      rm.add(p1);
833      rm.add(p2);
834
835      table.mutateRow(rm);
836    }
837    TEST_UTIL.getAdmin().flush(tableName);
838    List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(tableName);
839    HStore store = regions.get(0).getStore(fam);
840    Collection<HStoreFile> storefiles = store.getStorefiles();
841    assertTrue(storefiles.size() > 0);
842    for (HStoreFile storeFile : storefiles) {
843      assertTrue(storeFile.getReader().getHFileReader().getFileContext().isIncludesTags());
844    }
845  }
846
847  static Table createTableAndWriteDataWithLabels(TableName tableName, String... labelExps)
848    throws Exception {
849    List<Put> puts = new ArrayList<>(labelExps.length);
850    for (int i = 0; i < labelExps.length; i++) {
851      Put put = new Put(Bytes.toBytes("row" + (i + 1)));
852      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value);
853      put.setCellVisibility(new CellVisibility(labelExps[i]));
854      puts.add(put);
855    }
856    Table table = TEST_UTIL.createTable(tableName, fam);
857    table.put(puts);
858    return table;
859  }
860
861  public static void addLabels() throws Exception {
862    PrivilegedExceptionAction<VisibilityLabelsResponse> action =
863      new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
864        @Override
865        public VisibilityLabelsResponse run() throws Exception {
866          String[] labels = { SECRET, TOPSECRET, CONFIDENTIAL, PUBLIC, PRIVATE, COPYRIGHT, ACCENT,
867            UNICODE_VIS_TAG, UC1, UC2 };
868          try (Connection conn = ConnectionFactory.createConnection(conf)) {
869            VisibilityClient.addLabels(conn, labels);
870          } catch (Throwable t) {
871            throw new IOException(t);
872          }
873          return null;
874        }
875      };
876    SUPERUSER.runAs(action);
877  }
878}