001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.security.visibility;
019
020import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_FAMILY;
021import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME;
022import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABEL_QUALIFIER;
023import static org.junit.Assert.assertEquals;
024import static org.junit.Assert.assertFalse;
025import static org.junit.Assert.assertNotNull;
026import static org.junit.Assert.assertNull;
027import static org.junit.Assert.assertTrue;
028import static org.junit.Assert.fail;
029
030import com.google.protobuf.ByteString;
031
032import java.io.IOException;
033import java.security.PrivilegedExceptionAction;
034import java.util.ArrayList;
035import java.util.Collection;
036import java.util.List;
037
038import org.apache.hadoop.conf.Configuration;
039import org.apache.hadoop.hbase.Cell;
040import org.apache.hadoop.hbase.CellScanner;
041import org.apache.hadoop.hbase.HBaseTestingUtility;
042import org.apache.hadoop.hbase.HColumnDescriptor;
043import org.apache.hadoop.hbase.HConstants;
044import org.apache.hadoop.hbase.HTableDescriptor;
045import org.apache.hadoop.hbase.TableName;
046import org.apache.hadoop.hbase.client.Admin;
047import org.apache.hadoop.hbase.client.Append;
048import org.apache.hadoop.hbase.client.Connection;
049import org.apache.hadoop.hbase.client.ConnectionFactory;
050import org.apache.hadoop.hbase.client.Get;
051import org.apache.hadoop.hbase.client.Increment;
052import org.apache.hadoop.hbase.client.Put;
053import org.apache.hadoop.hbase.client.Result;
054import org.apache.hadoop.hbase.client.ResultScanner;
055import org.apache.hadoop.hbase.client.RowMutations;
056import org.apache.hadoop.hbase.client.Scan;
057import org.apache.hadoop.hbase.client.Table;
058import org.apache.hadoop.hbase.client.security.SecurityCapability;
059import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
060import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.GetAuthsResponse;
061import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
062import org.apache.hadoop.hbase.regionserver.BloomType;
063import org.apache.hadoop.hbase.regionserver.HRegion;
064import org.apache.hadoop.hbase.regionserver.HRegionServer;
065import org.apache.hadoop.hbase.regionserver.HStore;
066import org.apache.hadoop.hbase.regionserver.HStoreFile;
067import org.apache.hadoop.hbase.security.User;
068import org.apache.hadoop.hbase.util.Bytes;
069import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
070import org.junit.After;
071import org.junit.AfterClass;
072import org.junit.Rule;
073import org.junit.Test;
074import org.junit.rules.TestName;
075
076/**
077 * Base test class for visibility labels basic features
078 */
079public abstract class TestVisibilityLabels {
080
081  public static final String TOPSECRET = "topsecret";
082  public static final String PUBLIC = "public";
083  public static final String PRIVATE = "private";
084  public static final String CONFIDENTIAL = "confidential";
085  public static final String SECRET = "secret";
086  public static final String COPYRIGHT = "\u00A9ABC";
087  public static final String ACCENT = "\u0941";
088  public static final String UNICODE_VIS_TAG = COPYRIGHT + "\"" + ACCENT + "\\" + SECRET + "\""
089      + "\u0027&\\";
090  public static final String UC1 = "\u0027\"\u002b";
091  public static final String UC2 = "\u002d\u003f";
092  public static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
093  public static final byte[] row1 = Bytes.toBytes("row1");
094  public static final byte[] row2 = Bytes.toBytes("row2");
095  public static final byte[] row3 = Bytes.toBytes("row3");
096  public static final byte[] row4 = Bytes.toBytes("row4");
097  public final static byte[] fam = Bytes.toBytes("info");
098  public final static byte[] qual = Bytes.toBytes("qual");
099  public final static byte[] value = Bytes.toBytes("value");
100  public static Configuration conf;
101
102  private volatile boolean killedRS = false;
103  @Rule
104  public final TestName TEST_NAME = new TestName();
105  public static User SUPERUSER, USER1;
106
107  @AfterClass
108  public static void tearDownAfterClass() throws Exception {
109    TEST_UTIL.shutdownMiniCluster();
110  }
111
112  @After
113  public void tearDown() throws Exception {
114    killedRS = false;
115  }
116
117  @Test
118  public void testSecurityCapabilities() throws Exception {
119    List<SecurityCapability> capabilities = TEST_UTIL.getConnection().getAdmin()
120      .getSecurityCapabilities();
121    assertTrue("CELL_VISIBILITY capability is missing",
122      capabilities.contains(SecurityCapability.CELL_VISIBILITY));
123  }
124
125  @Test
126  public void testSimpleVisibilityLabels() throws Exception {
127    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
128    try (Table table = createTableAndWriteDataWithLabels(tableName, SECRET + "|" + CONFIDENTIAL,
129        PRIVATE + "|" + CONFIDENTIAL)) {
130      Scan s = new Scan();
131      s.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL, PRIVATE));
132      ResultScanner scanner = table.getScanner(s);
133      Result[] next = scanner.next(3);
134
135      assertTrue(next.length == 2);
136      CellScanner cellScanner = next[0].cellScanner();
137      cellScanner.advance();
138      Cell current = cellScanner.current();
139      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
140          current.getRowLength(), row1, 0, row1.length));
141      cellScanner = next[1].cellScanner();
142      cellScanner.advance();
143      current = cellScanner.current();
144      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
145          current.getRowLength(), row2, 0, row2.length));
146    }
147  }
148
149  @Test
150  public void testSimpleVisibilityLabelsWithUniCodeCharacters() throws Exception {
151    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
152    try (Table table = createTableAndWriteDataWithLabels(tableName,
153        SECRET + "|" + CellVisibility.quote(COPYRIGHT), "(" + CellVisibility.quote(COPYRIGHT)
154            + "&"  + CellVisibility.quote(ACCENT) + ")|" + CONFIDENTIAL,
155        CellVisibility.quote(UNICODE_VIS_TAG) + "&" + SECRET)) {
156      Scan s = new Scan();
157      s.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL, PRIVATE, COPYRIGHT, ACCENT,
158          UNICODE_VIS_TAG));
159      ResultScanner scanner = table.getScanner(s);
160      Result[] next = scanner.next(3);
161      assertTrue(next.length == 3);
162      CellScanner cellScanner = next[0].cellScanner();
163      cellScanner.advance();
164      Cell current = cellScanner.current();
165      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
166          current.getRowLength(), row1, 0, row1.length));
167      cellScanner = next[1].cellScanner();
168      cellScanner.advance();
169      current = cellScanner.current();
170      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
171          current.getRowLength(), row2, 0, row2.length));
172      cellScanner = next[2].cellScanner();
173      cellScanner.advance();
174      current = cellScanner.current();
175      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
176          current.getRowLength(), row3, 0, row3.length));
177    }
178  }
179
180  @Test
181  public void testAuthorizationsWithSpecialUnicodeCharacters() throws Exception {
182    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
183    try (Table table = createTableAndWriteDataWithLabels(tableName,
184        CellVisibility.quote(UC1) + "|" + CellVisibility.quote(UC2), CellVisibility.quote(UC1),
185        CellVisibility.quote(UNICODE_VIS_TAG))) {
186      Scan s = new Scan();
187      s.setAuthorizations(new Authorizations(UC1, UC2, ACCENT,
188          UNICODE_VIS_TAG));
189      ResultScanner scanner = table.getScanner(s);
190      Result[] next = scanner.next(3);
191      assertTrue(next.length == 3);
192      CellScanner cellScanner = next[0].cellScanner();
193      cellScanner.advance();
194      Cell current = cellScanner.current();
195      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
196          current.getRowLength(), row1, 0, row1.length));
197      cellScanner = next[1].cellScanner();
198      cellScanner.advance();
199      current = cellScanner.current();
200      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
201          current.getRowLength(), row2, 0, row2.length));
202      cellScanner = next[2].cellScanner();
203      cellScanner.advance();
204      current = cellScanner.current();
205      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
206          current.getRowLength(), row3, 0, row3.length));
207    }
208  }
209
210  @Test
211  public void testVisibilityLabelsWithComplexLabels() throws Exception {
212    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
213    try (Table table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|"
214        + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET, "(" + PRIVATE + "&" + CONFIDENTIAL + "&"
215        + SECRET + ")", "(" + PRIVATE + "&" + CONFIDENTIAL + "&" + SECRET + ")", "(" + PRIVATE
216        + "&" + CONFIDENTIAL + "&" + SECRET + ")")) {
217      Scan s = new Scan();
218      s.setAuthorizations(new Authorizations(TOPSECRET, CONFIDENTIAL, PRIVATE, PUBLIC, SECRET));
219      ResultScanner scanner = table.getScanner(s);
220      Result[] next = scanner.next(4);
221      assertEquals(3, next.length);
222      CellScanner cellScanner = next[0].cellScanner();
223      cellScanner.advance();
224      Cell current = cellScanner.current();
225      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
226          current.getRowLength(), row2, 0, row2.length));
227      cellScanner = next[1].cellScanner();
228      cellScanner.advance();
229      current = cellScanner.current();
230      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
231          current.getRowLength(), row3, 0, row3.length));
232      cellScanner = next[2].cellScanner();
233      cellScanner.advance();
234      current = cellScanner.current();
235      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
236          current.getRowLength(), row4, 0, row4.length));
237    }
238  }
239
240  @Test
241  public void testVisibilityLabelsThatDoesNotPassTheCriteria() throws Exception {
242    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
243    try (Table table = createTableAndWriteDataWithLabels(tableName,
244        "(" + SECRET + "|" + CONFIDENTIAL + ")", PRIVATE)){
245      Scan s = new Scan();
246      s.setAuthorizations(new Authorizations(PUBLIC));
247      ResultScanner scanner = table.getScanner(s);
248      Result[] next = scanner.next(3);
249      assertTrue(next.length == 0);
250    }
251  }
252
253  @Test
254  public void testVisibilityLabelsInPutsThatDoesNotMatchAnyDefinedLabels() throws Exception {
255    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
256    try {
257      createTableAndWriteDataWithLabels(tableName, "SAMPLE_LABEL", "TEST");
258      fail("Should have failed with failed sanity check exception");
259    } catch (Exception e) {
260    }
261  }
262
263  @Test
264  public void testVisibilityLabelsInScanThatDoesNotMatchAnyDefinedLabels() throws Exception {
265    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
266    try ( Table table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|"
267        + CONFIDENTIAL + ")", PRIVATE)){
268      Scan s = new Scan();
269      s.setAuthorizations(new Authorizations("SAMPLE"));
270      ResultScanner scanner = table.getScanner(s);
271      Result[] next = scanner.next(3);
272      assertTrue(next.length == 0);
273    }
274  }
275
276  @Test
277  public void testVisibilityLabelsWithGet() throws Exception {
278    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
279    try (Table table = createTableAndWriteDataWithLabels(tableName, SECRET + "&" + CONFIDENTIAL
280        + "&!" + PRIVATE, SECRET + "&" + CONFIDENTIAL + "&" + PRIVATE)) {
281      Get get = new Get(row1);
282      get.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL));
283      Result result = table.get(get);
284      assertTrue(!result.isEmpty());
285      Cell cell = result.getColumnLatestCell(fam, qual);
286      assertTrue(Bytes.equals(value, 0, value.length, cell.getValueArray(), cell.getValueOffset(),
287          cell.getValueLength()));
288    }
289  }
290
291  @Test
292  public void testVisibilityLabelsOnKillingOfRSContainingLabelsTable() throws Exception {
293    List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
294        .getRegionServerThreads();
295    int liveRS = 0;
296    for (RegionServerThread rsThreads : regionServerThreads) {
297      if (!rsThreads.getRegionServer().isAborted()) {
298        liveRS++;
299      }
300    }
301    if (liveRS == 1) {
302      TEST_UTIL.getHBaseCluster().startRegionServer();
303    }
304    Thread t1 = new Thread() {
305      @Override
306      public void run() {
307        List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
308            .getRegionServerThreads();
309        for (RegionServerThread rsThread : regionServerThreads) {
310          List<HRegion> onlineRegions = rsThread.getRegionServer().getRegions(
311              LABELS_TABLE_NAME);
312          if (onlineRegions.size() > 0) {
313            rsThread.getRegionServer().abort("Aborting ");
314            killedRS = true;
315            break;
316          }
317        }
318      }
319
320    };
321    t1.start();
322    final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
323    Thread t = new Thread() {
324      @Override
325      public void run() {
326        try {
327          while (!killedRS) {
328            Thread.sleep(1);
329          }
330          createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL + ")",
331              PRIVATE);
332        } catch (Exception e) {
333        }
334      }
335    };
336    t.start();
337    regionServerThreads = TEST_UTIL.getHBaseCluster().getRegionServerThreads();
338    while (!killedRS) {
339      Thread.sleep(10);
340    }
341    regionServerThreads = TEST_UTIL.getHBaseCluster().getRegionServerThreads();
342    for (RegionServerThread rsThread : regionServerThreads) {
343      while (true) {
344        if (!rsThread.getRegionServer().isAborted()) {
345          List<HRegion> onlineRegions = rsThread.getRegionServer().getRegions(
346              LABELS_TABLE_NAME);
347          if (onlineRegions.size() > 0) {
348            break;
349          } else {
350            Thread.sleep(10);
351          }
352        } else {
353          break;
354        }
355      }
356    }
357    TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000);
358    t.join();
359    try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
360      Scan s = new Scan();
361      s.setAuthorizations(new Authorizations(SECRET));
362      ResultScanner scanner = table.getScanner(s);
363      Result[] next = scanner.next(3);
364      assertTrue(next.length == 1);
365    }
366  }
367
368  @Test
369  public void testVisibilityLabelsOnRSRestart() throws Exception {
370    final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
371    List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
372        .getRegionServerThreads();
373    for (RegionServerThread rsThread : regionServerThreads) {
374      rsThread.getRegionServer().abort("Aborting ");
375    }
376    // Start one new RS
377    RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer();
378    waitForLabelsRegionAvailability(rs.getRegionServer());
379    try (Table table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL
380        + ")", PRIVATE)) {
381      Scan s = new Scan();
382      s.setAuthorizations(new Authorizations(SECRET));
383      ResultScanner scanner = table.getScanner(s);
384      Result[] next = scanner.next(3);
385      assertTrue(next.length == 1);
386    }
387  }
388
389  protected void waitForLabelsRegionAvailability(HRegionServer regionServer) {
390    while (!regionServer.isOnline()) {
391      try {
392        Thread.sleep(10);
393      } catch (InterruptedException e) {
394      }
395    }
396    while (regionServer.getRegions(LABELS_TABLE_NAME).isEmpty()) {
397      try {
398        Thread.sleep(10);
399      } catch (InterruptedException e) {
400      }
401    }
402  }
403
404  @Test
405  public void testVisibilityLabelsInGetThatDoesNotMatchAnyDefinedLabels() throws Exception {
406    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
407    try (Table table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL
408        + ")", PRIVATE)) {
409      Get get = new Get(row1);
410      get.setAuthorizations(new Authorizations("SAMPLE"));
411      Result result = table.get(get);
412      assertTrue(result.isEmpty());
413    }
414  }
415
416  @Test
417  public void testSetAndGetUserAuths() throws Throwable {
418    final String user = "user1";
419    PrivilegedExceptionAction<Void> action = new PrivilegedExceptionAction<Void>() {
420      @Override
421      public Void run() throws Exception {
422        String[] auths = { SECRET, CONFIDENTIAL };
423        try (Connection conn = ConnectionFactory.createConnection(conf)) {
424          VisibilityClient.setAuths(conn, auths, user);
425        } catch (Throwable e) {
426          throw new IOException(e);
427        }
428        return null;
429      }
430    };
431    SUPERUSER.runAs(action);
432    try (Table ht = TEST_UTIL.getConnection().getTable(LABELS_TABLE_NAME)) {
433      Scan scan = new Scan();
434      scan.setAuthorizations(new Authorizations(VisibilityUtils.SYSTEM_LABEL));
435      ResultScanner scanner = ht.getScanner(scan);
436      Result result = null;
437      List<Result> results = new ArrayList<>();
438      while ((result = scanner.next()) != null) {
439        results.add(result);
440      }
441      List<String> auths = extractAuths(user, results);
442      assertTrue(auths.contains(SECRET));
443      assertTrue(auths.contains(CONFIDENTIAL));
444      assertEquals(2, auths.size());
445    }
446
447    action = new PrivilegedExceptionAction<Void>() {
448      @Override
449      public Void run() throws Exception {
450        GetAuthsResponse authsResponse = null;
451        try (Connection conn = ConnectionFactory.createConnection(conf)) {
452          authsResponse = VisibilityClient.getAuths(conn, user);
453        } catch (Throwable e) {
454          throw new IOException(e);
455        }
456        List<String> authsList = new ArrayList<>(authsResponse.getAuthList().size());
457        for (ByteString authBS : authsResponse.getAuthList()) {
458          authsList.add(Bytes.toString(authBS.toByteArray()));
459        }
460        assertEquals(2, authsList.size());
461        assertTrue(authsList.contains(SECRET));
462        assertTrue(authsList.contains(CONFIDENTIAL));
463        return null;
464      }
465    };
466    SUPERUSER.runAs(action);
467
468    // Try doing setAuths once again and there should not be any duplicates
469    action = new PrivilegedExceptionAction<Void>() {
470      @Override
471      public Void run() throws Exception {
472        String[] auths1 = { SECRET, CONFIDENTIAL };
473        GetAuthsResponse authsResponse = null;
474        try (Connection conn = ConnectionFactory.createConnection(conf)) {
475          VisibilityClient.setAuths(conn, auths1, user);
476          try {
477            authsResponse = VisibilityClient.getAuths(conn, user);
478          } catch (Throwable e) {
479            throw new IOException(e);
480          }
481        } catch (Throwable e) {
482        }
483        List<String> authsList = new ArrayList<>(authsResponse.getAuthList().size());
484        for (ByteString authBS : authsResponse.getAuthList()) {
485          authsList.add(Bytes.toString(authBS.toByteArray()));
486        }
487        assertEquals(2, authsList.size());
488        assertTrue(authsList.contains(SECRET));
489        assertTrue(authsList.contains(CONFIDENTIAL));
490        return null;
491      }
492    };
493    SUPERUSER.runAs(action);
494  }
495
496  protected List<String> extractAuths(String user, List<Result> results) {
497    List<String> auths = new ArrayList<>();
498    for (Result result : results) {
499      Cell labelCell = result.getColumnLatestCell(LABELS_TABLE_FAMILY, LABEL_QUALIFIER);
500      Cell userAuthCell = result.getColumnLatestCell(LABELS_TABLE_FAMILY, Bytes.toBytes(user));
501      if (userAuthCell != null) {
502        auths.add(Bytes.toString(labelCell.getValueArray(), labelCell.getValueOffset(),
503            labelCell.getValueLength()));
504      }
505    }
506    return auths;
507  }
508
509  @Test
510  public void testClearUserAuths() throws Throwable {
511    PrivilegedExceptionAction<Void> action = new PrivilegedExceptionAction<Void>() {
512      @Override
513      public Void run() throws Exception {
514        String[] auths = { SECRET, CONFIDENTIAL, PRIVATE };
515        String user = "testUser";
516        try (Connection conn = ConnectionFactory.createConnection(conf)) {
517          VisibilityClient.setAuths(conn, auths, user);
518        } catch (Throwable e) {
519          throw new IOException(e);
520        }
521        // Removing the auths for SECRET and CONFIDENTIAL for the user.
522        // Passing a non existing auth also.
523        auths = new String[] { SECRET, PUBLIC, CONFIDENTIAL };
524        VisibilityLabelsResponse response = null;
525        try (Connection conn = ConnectionFactory.createConnection(conf)) {
526          response = VisibilityClient.clearAuths(conn, auths, user);
527        } catch (Throwable e) {
528          fail("Should not have failed");
529        }
530        List<RegionActionResult> resultList = response.getResultList();
531        assertEquals(3, resultList.size());
532        assertTrue(resultList.get(0).getException().getValue().isEmpty());
533        assertEquals("org.apache.hadoop.hbase.DoNotRetryIOException",
534            resultList.get(1).getException().getName());
535        assertTrue(Bytes.toString(resultList.get(1).getException().getValue().toByteArray())
536            .contains(
537                "org.apache.hadoop.hbase.security.visibility.InvalidLabelException: "
538                    + "Label 'public' is not set for the user testUser"));
539        assertTrue(resultList.get(2).getException().getValue().isEmpty());
540        try (Connection connection = ConnectionFactory.createConnection(conf);
541             Table ht = connection.getTable(LABELS_TABLE_NAME)) {
542          ResultScanner scanner = ht.getScanner(new Scan());
543          Result result = null;
544          List<Result> results = new ArrayList<>();
545          while ((result = scanner.next()) != null) {
546            results.add(result);
547          }
548          List<String> curAuths = extractAuths(user, results);
549          assertTrue(curAuths.contains(PRIVATE));
550          assertEquals(1, curAuths.size());
551        }
552
553        GetAuthsResponse authsResponse = null;
554        try (Connection conn = ConnectionFactory.createConnection(conf)) {
555          authsResponse = VisibilityClient.getAuths(conn, user);
556        } catch (Throwable e) {
557          throw new IOException(e);
558        }
559        List<String> authsList = new ArrayList<>(authsResponse.getAuthList().size());
560        for (ByteString authBS : authsResponse.getAuthList()) {
561          authsList.add(Bytes.toString(authBS.toByteArray()));
562        }
563        assertEquals(1, authsList.size());
564        assertTrue(authsList.contains(PRIVATE));
565        return null;
566      }
567    };
568    SUPERUSER.runAs(action);
569  }
570
571  @Test
572  public void testLabelsWithCheckAndPut() throws Throwable {
573    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
574    try (Table table = TEST_UTIL.createTable(tableName, fam)) {
575      byte[] row1 = Bytes.toBytes("row1");
576      Put put = new Put(row1);
577      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value);
578      put.setCellVisibility(new CellVisibility(SECRET + " & " + CONFIDENTIAL));
579      table.checkAndMutate(row1, fam).qualifier(qual).ifNotExists().thenPut(put);
580      byte[] row2 = Bytes.toBytes("row2");
581      put = new Put(row2);
582      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value);
583      put.setCellVisibility(new CellVisibility(SECRET));
584      table.checkAndMutate(row2, fam).qualifier(qual).ifNotExists().thenPut(put);
585
586      Scan scan = new Scan();
587      scan.setAuthorizations(new Authorizations(SECRET));
588      ResultScanner scanner = table.getScanner(scan);
589      Result result = scanner.next();
590      assertTrue(!result.isEmpty());
591      assertTrue(Bytes.equals(row2, result.getRow()));
592      result = scanner.next();
593      assertNull(result);
594    }
595  }
596
597  @Test
598  public void testLabelsWithIncrement() throws Throwable {
599    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
600    try (Table table = TEST_UTIL.createTable(tableName, fam)) {
601      byte[] row1 = Bytes.toBytes("row1");
602      byte[] val = Bytes.toBytes(1L);
603      Put put = new Put(row1);
604      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, val);
605      put.setCellVisibility(new CellVisibility(SECRET + " & " + CONFIDENTIAL));
606      table.put(put);
607      Get get = new Get(row1);
608      get.setAuthorizations(new Authorizations(SECRET));
609      Result result = table.get(get);
610      assertTrue(result.isEmpty());
611      table.incrementColumnValue(row1, fam, qual, 2L);
612      result = table.get(get);
613      assertTrue(result.isEmpty());
614      Increment increment = new Increment(row1);
615      increment.addColumn(fam, qual, 2L);
616      increment.setCellVisibility(new CellVisibility(SECRET));
617      table.increment(increment);
618      result = table.get(get);
619      assertTrue(!result.isEmpty());
620    }
621  }
622
623  @Test
624  public void testLabelsWithAppend() throws Throwable {
625    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
626    try (Table table = TEST_UTIL.createTable(tableName, fam)) {
627      byte[] row1 = Bytes.toBytes("row1");
628      byte[] val = Bytes.toBytes("a");
629      Put put = new Put(row1);
630      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, val);
631      put.setCellVisibility(new CellVisibility(SECRET + " & " + CONFIDENTIAL));
632      table.put(put);
633      Get get = new Get(row1);
634      get.setAuthorizations(new Authorizations(SECRET));
635      Result result = table.get(get);
636      assertTrue(result.isEmpty());
637      Append append = new Append(row1);
638      append.addColumn(fam, qual, Bytes.toBytes("b"));
639      table.append(append);
640      result = table.get(get);
641      assertTrue(result.isEmpty());
642      append = new Append(row1);
643      append.addColumn(fam, qual, Bytes.toBytes("c"));
644      append.setCellVisibility(new CellVisibility(SECRET));
645      table.append(append);
646      result = table.get(get);
647      assertTrue(!result.isEmpty());
648    }
649  }
650
651  @Test
652  public void testUserShouldNotDoDDLOpOnLabelsTable() throws Exception {
653    Admin admin = TEST_UTIL.getAdmin();
654    try {
655      admin.disableTable(LABELS_TABLE_NAME);
656      fail("Lables table should not get disabled by user.");
657    } catch (Exception e) {
658    }
659    try {
660      admin.deleteTable(LABELS_TABLE_NAME);
661      fail("Lables table should not get disabled by user.");
662    } catch (Exception e) {
663    }
664    try {
665      HColumnDescriptor hcd = new HColumnDescriptor("testFamily");
666      admin.addColumnFamily(LABELS_TABLE_NAME, hcd);
667      fail("Lables table should not get altered by user.");
668    } catch (Exception e) {
669    }
670    try {
671      admin.deleteColumnFamily(LABELS_TABLE_NAME, VisibilityConstants.LABELS_TABLE_FAMILY);
672      fail("Lables table should not get altered by user.");
673    } catch (Exception e) {
674    }
675    try {
676      HColumnDescriptor hcd = new HColumnDescriptor(VisibilityConstants.LABELS_TABLE_FAMILY);
677      hcd.setBloomFilterType(BloomType.ROWCOL);
678      admin.modifyColumnFamily(LABELS_TABLE_NAME, hcd);
679      fail("Lables table should not get altered by user.");
680    } catch (Exception e) {
681    }
682    try {
683      HTableDescriptor htd = new HTableDescriptor(LABELS_TABLE_NAME);
684      htd.addFamily(new HColumnDescriptor("f1"));
685      htd.addFamily(new HColumnDescriptor("f2"));
686      admin.modifyTable(htd);
687      fail("Lables table should not get altered by user.");
688    } catch (Exception e) {
689    }
690  }
691
692  @Test
693  public void testMultipleVersions() throws Exception {
694    final byte[] r1 = Bytes.toBytes("row1");
695    final byte[] r2 = Bytes.toBytes("row2");
696    final byte[] v1 = Bytes.toBytes("100");
697    final byte[] v2 = Bytes.toBytes("101");
698    final byte[] fam2 = Bytes.toBytes("info2");
699    final byte[] qual2 = Bytes.toBytes("qual2");
700    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
701    HTableDescriptor desc = new HTableDescriptor(tableName);
702    HColumnDescriptor col = new HColumnDescriptor(fam);// Default max versions is 1.
703    desc.addFamily(col);
704    col = new HColumnDescriptor(fam2);
705    col.setMaxVersions(5);
706    desc.addFamily(col);
707    TEST_UTIL.getAdmin().createTable(desc);
708    try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
709      Put put = new Put(r1);
710      put.addColumn(fam, qual, 3L, v1);
711      put.addColumn(fam, qual2, 3L, v1);
712      put.addColumn(fam2, qual, 3L, v1);
713      put.addColumn(fam2, qual2, 3L, v1);
714      put.setCellVisibility(new CellVisibility(SECRET));
715      table.put(put);
716      put = new Put(r1);
717      put.addColumn(fam, qual, 4L, v2);
718      put.addColumn(fam, qual2, 4L, v2);
719      put.addColumn(fam2, qual, 4L, v2);
720      put.addColumn(fam2, qual2, 4L, v2);
721      put.setCellVisibility(new CellVisibility(PRIVATE));
722      table.put(put);
723
724      put = new Put(r2);
725      put.addColumn(fam, qual, 3L, v1);
726      put.addColumn(fam, qual2, 3L, v1);
727      put.addColumn(fam2, qual, 3L, v1);
728      put.addColumn(fam2, qual2, 3L, v1);
729      put.setCellVisibility(new CellVisibility(SECRET));
730      table.put(put);
731      put = new Put(r2);
732      put.addColumn(fam, qual, 4L, v2);
733      put.addColumn(fam, qual2, 4L, v2);
734      put.addColumn(fam2, qual, 4L, v2);
735      put.addColumn(fam2, qual2, 4L, v2);
736      put.setCellVisibility(new CellVisibility(SECRET));
737      table.put(put);
738
739      Scan s = new Scan();
740      s.setMaxVersions(1);
741      s.setAuthorizations(new Authorizations(SECRET));
742      ResultScanner scanner = table.getScanner(s);
743      Result result = scanner.next();
744      assertTrue(Bytes.equals(r1, result.getRow()));
745      // for cf 'fam' max versions in HCD is 1. So the old version cells, which are having matching
746      // CellVisibility with Authorizations, should not get considered in the label evaluation at
747      // all.
748      assertNull(result.getColumnLatestCell(fam, qual));
749      assertNull(result.getColumnLatestCell(fam, qual2));
750      // for cf 'fam2' max versions in HCD is > 1. So we can consider the old version cells, which
751      // are having matching CellVisibility with Authorizations, in the label evaluation. It can
752      // just skip those recent versions for which visibility is not there as per the new version's
753      // CellVisibility. The old versions which are having visibility can be send back
754      Cell cell = result.getColumnLatestCell(fam2, qual);
755      assertNotNull(cell);
756      assertTrue(Bytes.equals(v1, 0, v1.length, cell.getValueArray(), cell.getValueOffset(),
757          cell.getValueLength()));
758      cell = result.getColumnLatestCell(fam2, qual2);
759      assertNotNull(cell);
760      assertTrue(Bytes.equals(v1, 0, v1.length, cell.getValueArray(), cell.getValueOffset(),
761          cell.getValueLength()));
762
763      result = scanner.next();
764      assertTrue(Bytes.equals(r2, result.getRow()));
765      cell = result.getColumnLatestCell(fam, qual);
766      assertNotNull(cell);
767      assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
768          cell.getValueLength()));
769      cell = result.getColumnLatestCell(fam, qual2);
770      assertNotNull(cell);
771      assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
772          cell.getValueLength()));
773      cell = result.getColumnLatestCell(fam2, qual);
774      assertNotNull(cell);
775      assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
776          cell.getValueLength()));
777      cell = result.getColumnLatestCell(fam2, qual2);
778      assertNotNull(cell);
779      assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
780          cell.getValueLength()));
781    }
782  }
783
784  @Test
785  public void testMutateRow() throws Exception {
786    final byte[] qual2 = Bytes.toBytes("qual2");
787    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
788    HTableDescriptor desc = new HTableDescriptor(tableName);
789    HColumnDescriptor col = new HColumnDescriptor(fam);
790    desc.addFamily(col);
791    TEST_UTIL.getAdmin().createTable(desc);
792    try (Table table = TEST_UTIL.getConnection().getTable(tableName)){
793      Put p1 = new Put(row1);
794      p1.addColumn(fam, qual, value);
795      p1.setCellVisibility(new CellVisibility(CONFIDENTIAL));
796
797      Put p2 = new Put(row1);
798      p2.addColumn(fam, qual2, value);
799      p2.setCellVisibility(new CellVisibility(SECRET));
800
801      RowMutations rm = new RowMutations(row1);
802      rm.add(p1);
803      rm.add(p2);
804
805      table.mutateRow(rm);
806
807      Get get = new Get(row1);
808      get.setAuthorizations(new Authorizations(CONFIDENTIAL));
809      Result result = table.get(get);
810      assertTrue(result.containsColumn(fam, qual));
811      assertFalse(result.containsColumn(fam, qual2));
812
813      get.setAuthorizations(new Authorizations(SECRET));
814      result = table.get(get);
815      assertFalse(result.containsColumn(fam, qual));
816      assertTrue(result.containsColumn(fam, qual2));
817    }
818  }
819
820  @Test
821  public void testFlushedFileWithVisibilityTags() throws Exception {
822    final byte[] qual2 = Bytes.toBytes("qual2");
823    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
824    HTableDescriptor desc = new HTableDescriptor(tableName);
825    HColumnDescriptor col = new HColumnDescriptor(fam);
826    desc.addFamily(col);
827    TEST_UTIL.getAdmin().createTable(desc);
828    try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
829      Put p1 = new Put(row1);
830      p1.addColumn(fam, qual, value);
831      p1.setCellVisibility(new CellVisibility(CONFIDENTIAL));
832
833      Put p2 = new Put(row1);
834      p2.addColumn(fam, qual2, value);
835      p2.setCellVisibility(new CellVisibility(SECRET));
836
837      RowMutations rm = new RowMutations(row1);
838      rm.add(p1);
839      rm.add(p2);
840
841      table.mutateRow(rm);
842    }
843    TEST_UTIL.getAdmin().flush(tableName);
844    List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(tableName);
845    HStore store = regions.get(0).getStore(fam);
846    Collection<HStoreFile> storefiles = store.getStorefiles();
847    assertTrue(storefiles.size() > 0);
848    for (HStoreFile storeFile : storefiles) {
849      assertTrue(storeFile.getReader().getHFileReader().getFileContext().isIncludesTags());
850    }
851  }
852
853  static Table createTableAndWriteDataWithLabels(TableName tableName, String... labelExps)
854      throws Exception {
855    List<Put> puts = new ArrayList<>(labelExps.length);
856    for (int i = 0; i < labelExps.length; i++) {
857      Put put = new Put(Bytes.toBytes("row" + (i+1)));
858      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value);
859      put.setCellVisibility(new CellVisibility(labelExps[i]));
860      puts.add(put);
861    }
862    Table table = TEST_UTIL.createTable(tableName, fam);
863    table.put(puts);
864    return table;
865  }
866
867  public static void addLabels() throws Exception {
868    PrivilegedExceptionAction<VisibilityLabelsResponse> action =
869        new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
870      @Override
871      public VisibilityLabelsResponse run() throws Exception {
872        String[] labels = { SECRET, TOPSECRET, CONFIDENTIAL, PUBLIC, PRIVATE, COPYRIGHT, ACCENT,
873            UNICODE_VIS_TAG, UC1, UC2 };
874        try (Connection conn = ConnectionFactory.createConnection(conf)) {
875          VisibilityClient.addLabels(conn, labels);
876        } catch (Throwable t) {
877          throw new IOException(t);
878        }
879        return null;
880      }
881    };
882    SUPERUSER.runAs(action);
883  }
884}