Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.security.visibility;
019
020import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_FAMILY;
021import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME;
022import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABEL_QUALIFIER;
023import static org.junit.Assert.assertEquals;
024import static org.junit.Assert.assertFalse;
025import static org.junit.Assert.assertNotNull;
026import static org.junit.Assert.assertNull;
027import static org.junit.Assert.assertTrue;
028import static org.junit.Assert.fail;
029
030import com.google.protobuf.ByteString;
031
032import java.io.IOException;
033import java.security.PrivilegedExceptionAction;
034import java.util.ArrayList;
035import java.util.Collection;
036import java.util.List;
037
038import org.apache.hadoop.conf.Configuration;
039import org.apache.hadoop.hbase.Cell;
040import org.apache.hadoop.hbase.CellScanner;
041import org.apache.hadoop.hbase.HBaseTestingUtility;
042import org.apache.hadoop.hbase.HColumnDescriptor;
043import org.apache.hadoop.hbase.HConstants;
044import org.apache.hadoop.hbase.HTableDescriptor;
045import org.apache.hadoop.hbase.TableName;
046import org.apache.hadoop.hbase.client.Admin;
047import org.apache.hadoop.hbase.client.Append;
048import org.apache.hadoop.hbase.client.Connection;
049import org.apache.hadoop.hbase.client.ConnectionFactory;
050import org.apache.hadoop.hbase.client.Get;
051import org.apache.hadoop.hbase.client.Increment;
052import org.apache.hadoop.hbase.client.Put;
053import org.apache.hadoop.hbase.client.Result;
054import org.apache.hadoop.hbase.client.ResultScanner;
055import org.apache.hadoop.hbase.client.RowMutations;
056import org.apache.hadoop.hbase.client.Scan;
057import org.apache.hadoop.hbase.client.Table;
058import org.apache.hadoop.hbase.client.security.SecurityCapability;
059import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
060import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.GetAuthsResponse;
061import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
062import org.apache.hadoop.hbase.regionserver.BloomType;
063import org.apache.hadoop.hbase.regionserver.HRegion;
064import org.apache.hadoop.hbase.regionserver.HRegionServer;
065import org.apache.hadoop.hbase.regionserver.HStore;
066import org.apache.hadoop.hbase.regionserver.HStoreFile;
067import org.apache.hadoop.hbase.security.User;
068import org.apache.hadoop.hbase.util.Bytes;
069import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
070import org.junit.After;
071import org.junit.AfterClass;
072import org.junit.Rule;
073import org.junit.Test;
074import org.junit.rules.TestName;
075
076/**
077 * Base test class for visibility labels basic features
078 */
079public abstract class TestVisibilityLabels {
080
081  public static final String TOPSECRET = "topsecret";
082  public static final String PUBLIC = "public";
083  public static final String PRIVATE = "private";
084  public static final String CONFIDENTIAL = "confidential";
085  public static final String SECRET = "secret";
086  public static final String COPYRIGHT = "\u00A9ABC";
087  public static final String ACCENT = "\u0941";
088  public static final String UNICODE_VIS_TAG = COPYRIGHT + "\"" + ACCENT + "\\" + SECRET + "\""
089      + "\u0027&\\";
090  public static final String UC1 = "\u0027\"\u002b";
091  public static final String UC2 = "\u002d\u003f";
092  public static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
093  public static final byte[] row1 = Bytes.toBytes("row1");
094  public static final byte[] row2 = Bytes.toBytes("row2");
095  public static final byte[] row3 = Bytes.toBytes("row3");
096  public static final byte[] row4 = Bytes.toBytes("row4");
097  public final static byte[] fam = Bytes.toBytes("info");
098  public final static byte[] qual = Bytes.toBytes("qual");
099  public final static byte[] value = Bytes.toBytes("value");
100  public static Configuration conf;
101
102  private volatile boolean killedRS = false;
103  @Rule
104  public final TestName TEST_NAME = new TestName();
105  public static User SUPERUSER, USER1;
106
107  @AfterClass
108  public static void tearDownAfterClass() throws Exception {
109    TEST_UTIL.shutdownMiniCluster();
110  }
111
112  @After
113  public void tearDown() throws Exception {
114    killedRS = false;
115  }
116
117  @Test
118  public void testSecurityCapabilities() throws Exception {
119    List<SecurityCapability> capabilities = TEST_UTIL.getConnection().getAdmin()
120      .getSecurityCapabilities();
121    assertTrue("CELL_VISIBILITY capability is missing",
122      capabilities.contains(SecurityCapability.CELL_VISIBILITY));
123  }
124
125  @Test
126  public void testSimpleVisibilityLabels() throws Exception {
127    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
128    try (Table table = createTableAndWriteDataWithLabels(tableName, SECRET + "|" + CONFIDENTIAL,
129        PRIVATE + "|" + CONFIDENTIAL)) {
130      Scan s = new Scan();
131      s.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL, PRIVATE));
132      ResultScanner scanner = table.getScanner(s);
133      Result[] next = scanner.next(3);
134
135      assertTrue(next.length == 2);
136      CellScanner cellScanner = next[0].cellScanner();
137      cellScanner.advance();
138      Cell current = cellScanner.current();
139      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
140          current.getRowLength(), row1, 0, row1.length));
141      cellScanner = next[1].cellScanner();
142      cellScanner.advance();
143      current = cellScanner.current();
144      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
145          current.getRowLength(), row2, 0, row2.length));
146    }
147  }
148
149  @Test
150  public void testSimpleVisibilityLabelsWithUniCodeCharacters() throws Exception {
151    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
152    try (Table table = createTableAndWriteDataWithLabels(tableName,
153        SECRET + "|" + CellVisibility.quote(COPYRIGHT), "(" + CellVisibility.quote(COPYRIGHT)
154            + "&"  + CellVisibility.quote(ACCENT) + ")|" + CONFIDENTIAL,
155        CellVisibility.quote(UNICODE_VIS_TAG) + "&" + SECRET)) {
156      Scan s = new Scan();
157      s.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL, PRIVATE, COPYRIGHT, ACCENT,
158          UNICODE_VIS_TAG));
159      ResultScanner scanner = table.getScanner(s);
160      Result[] next = scanner.next(3);
161      assertTrue(next.length == 3);
162      CellScanner cellScanner = next[0].cellScanner();
163      cellScanner.advance();
164      Cell current = cellScanner.current();
165      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
166          current.getRowLength(), row1, 0, row1.length));
167      cellScanner = next[1].cellScanner();
168      cellScanner.advance();
169      current = cellScanner.current();
170      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
171          current.getRowLength(), row2, 0, row2.length));
172      cellScanner = next[2].cellScanner();
173      cellScanner.advance();
174      current = cellScanner.current();
175      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
176          current.getRowLength(), row3, 0, row3.length));
177    }
178  }
179
180  @Test
181  public void testAuthorizationsWithSpecialUnicodeCharacters() throws Exception {
182    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
183    try (Table table = createTableAndWriteDataWithLabels(tableName,
184        CellVisibility.quote(UC1) + "|" + CellVisibility.quote(UC2), CellVisibility.quote(UC1),
185        CellVisibility.quote(UNICODE_VIS_TAG))) {
186      Scan s = new Scan();
187      s.setAuthorizations(new Authorizations(UC1, UC2, ACCENT,
188          UNICODE_VIS_TAG));
189      ResultScanner scanner = table.getScanner(s);
190      Result[] next = scanner.next(3);
191      assertTrue(next.length == 3);
192      CellScanner cellScanner = next[0].cellScanner();
193      cellScanner.advance();
194      Cell current = cellScanner.current();
195      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
196          current.getRowLength(), row1, 0, row1.length));
197      cellScanner = next[1].cellScanner();
198      cellScanner.advance();
199      current = cellScanner.current();
200      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
201          current.getRowLength(), row2, 0, row2.length));
202      cellScanner = next[2].cellScanner();
203      cellScanner.advance();
204      current = cellScanner.current();
205      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
206          current.getRowLength(), row3, 0, row3.length));
207    }
208  }
209
210  @Test
211  public void testVisibilityLabelsWithComplexLabels() throws Exception {
212    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
213    try (Table table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|"
214        + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET, "(" + PRIVATE + "&" + CONFIDENTIAL + "&"
215        + SECRET + ")", "(" + PRIVATE + "&" + CONFIDENTIAL + "&" + SECRET + ")", "(" + PRIVATE
216        + "&" + CONFIDENTIAL + "&" + SECRET + ")")) {
217      Scan s = new Scan();
218      s.setAuthorizations(new Authorizations(TOPSECRET, CONFIDENTIAL, PRIVATE, PUBLIC, SECRET));
219      ResultScanner scanner = table.getScanner(s);
220      Result[] next = scanner.next(4);
221      assertEquals(3, next.length);
222      CellScanner cellScanner = next[0].cellScanner();
223      cellScanner.advance();
224      Cell current = cellScanner.current();
225      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
226          current.getRowLength(), row2, 0, row2.length));
227      cellScanner = next[1].cellScanner();
228      cellScanner.advance();
229      current = cellScanner.current();
230      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
231          current.getRowLength(), row3, 0, row3.length));
232      cellScanner = next[2].cellScanner();
233      cellScanner.advance();
234      current = cellScanner.current();
235      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
236          current.getRowLength(), row4, 0, row4.length));
237    }
238  }
239
240  @Test
241  public void testVisibilityLabelsThatDoesNotPassTheCriteria() throws Exception {
242    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
243    try (Table table = createTableAndWriteDataWithLabels(tableName,
244        "(" + SECRET + "|" + CONFIDENTIAL + ")", PRIVATE)){
245      Scan s = new Scan();
246      s.setAuthorizations(new Authorizations(PUBLIC));
247      ResultScanner scanner = table.getScanner(s);
248      Result[] next = scanner.next(3);
249      assertTrue(next.length == 0);
250    }
251  }
252
253  @Test
254  public void testVisibilityLabelsInPutsThatDoesNotMatchAnyDefinedLabels() throws Exception {
255    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
256    try {
257      createTableAndWriteDataWithLabels(tableName, "SAMPLE_LABEL", "TEST");
258      fail("Should have failed with failed sanity check exception");
259    } catch (Exception e) {
260    }
261  }
262
263  @Test
264  public void testVisibilityLabelsInScanThatDoesNotMatchAnyDefinedLabels() throws Exception {
265    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
266    try ( Table table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|"
267        + CONFIDENTIAL + ")", PRIVATE)){
268      Scan s = new Scan();
269      s.setAuthorizations(new Authorizations("SAMPLE"));
270      ResultScanner scanner = table.getScanner(s);
271      Result[] next = scanner.next(3);
272      assertTrue(next.length == 0);
273    }
274  }
275
276  @Test
277  public void testVisibilityLabelsWithGet() throws Exception {
278    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
279    try (Table table = createTableAndWriteDataWithLabels(tableName, SECRET + "&" + CONFIDENTIAL
280        + "&!" + PRIVATE, SECRET + "&" + CONFIDENTIAL + "&" + PRIVATE)) {
281      Get get = new Get(row1);
282      get.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL));
283      Result result = table.get(get);
284      assertTrue(!result.isEmpty());
285      Cell cell = result.getColumnLatestCell(fam, qual);
286      assertTrue(Bytes.equals(value, 0, value.length, cell.getValueArray(), cell.getValueOffset(),
287          cell.getValueLength()));
288    }
289  }
290
291  @Test
292  public void testVisibilityLabelsOnKillingOfRSContainingLabelsTable() throws Exception {
293    List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
294        .getRegionServerThreads();
295    int liveRS = 0;
296    for (RegionServerThread rsThreads : regionServerThreads) {
297      if (!rsThreads.getRegionServer().isAborted()) {
298        liveRS++;
299      }
300    }
301    if (liveRS == 1) {
302      TEST_UTIL.getHBaseCluster().startRegionServer();
303    }
304    Thread t1 = new Thread() {
305      @Override
306      public void run() {
307        List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
308            .getRegionServerThreads();
309        for (RegionServerThread rsThread : regionServerThreads) {
310          List<HRegion> onlineRegions = rsThread.getRegionServer().getRegions(
311              LABELS_TABLE_NAME);
312          if (onlineRegions.size() > 0) {
313            rsThread.getRegionServer().abort("Aborting ");
314            killedRS = true;
315            break;
316          }
317        }
318      }
319
320    };
321    t1.start();
322    final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
323    Thread t = new Thread() {
324      @Override
325      public void run() {
326        try {
327          while (!killedRS) {
328            Thread.sleep(1);
329          }
330          createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL + ")",
331              PRIVATE);
332        } catch (Exception e) {
333        }
334      }
335    };
336    t.start();
337    regionServerThreads = TEST_UTIL.getHBaseCluster().getRegionServerThreads();
338    while (!killedRS) {
339      Thread.sleep(10);
340    }
341    regionServerThreads = TEST_UTIL.getHBaseCluster().getRegionServerThreads();
342    for (RegionServerThread rsThread : regionServerThreads) {
343      while (true) {
344        if (!rsThread.getRegionServer().isAborted()) {
345          List<HRegion> onlineRegions = rsThread.getRegionServer().getRegions(
346              LABELS_TABLE_NAME);
347          if (onlineRegions.size() > 0) {
348            break;
349          } else {
350            Thread.sleep(10);
351          }
352        } else {
353          break;
354        }
355      }
356    }
357    TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000);
358    t.join();
359    try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
360      Scan s = new Scan();
361      s.setAuthorizations(new Authorizations(SECRET));
362      ResultScanner scanner = table.getScanner(s);
363      Result[] next = scanner.next(3);
364      assertTrue(next.length == 1);
365    }
366  }
367
368  @Test
369  public void testVisibilityLabelsOnRSRestart() throws Exception {
370    final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
371    List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
372        .getRegionServerThreads();
373    for (RegionServerThread rsThread : regionServerThreads) {
374      rsThread.getRegionServer().abort("Aborting ");
375    }
376    // Start one new RS
377    RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer();
378    waitForLabelsRegionAvailability(rs.getRegionServer());
379    try (Table table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL
380        + ")", PRIVATE);) {
381      Scan s = new Scan();
382      s.setAuthorizations(new Authorizations(SECRET));
383      ResultScanner scanner = table.getScanner(s);
384      Result[] next = scanner.next(3);
385      assertTrue(next.length == 1);
386    }
387  }
388
389  protected void waitForLabelsRegionAvailability(HRegionServer regionServer) {
390    while (!regionServer.isOnline()) {
391      try {
392        Thread.sleep(10);
393      } catch (InterruptedException e) {
394      }
395    }
396    while (regionServer.getRegions(LABELS_TABLE_NAME).isEmpty()) {
397      try {
398        Thread.sleep(10);
399      } catch (InterruptedException e) {
400      }
401    }
402  }
403
404  @Test
405  public void testVisibilityLabelsInGetThatDoesNotMatchAnyDefinedLabels() throws Exception {
406    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
407    try (Table table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL
408        + ")", PRIVATE)) {
409      Get get = new Get(row1);
410      get.setAuthorizations(new Authorizations("SAMPLE"));
411      Result result = table.get(get);
412      assertTrue(result.isEmpty());
413    }
414  }
415
416  @Test
417  public void testSetAndGetUserAuths() throws Throwable {
418    final String user = "user1";
419    PrivilegedExceptionAction<Void> action = new PrivilegedExceptionAction<Void>() {
420      @Override
421      public Void run() throws Exception {
422        String[] auths = { SECRET, CONFIDENTIAL };
423        try (Connection conn = ConnectionFactory.createConnection(conf)) {
424          VisibilityClient.setAuths(conn, auths, user);
425        } catch (Throwable e) {
426        }
427        return null;
428      }
429    };
430    SUPERUSER.runAs(action);
431    try (Table ht = TEST_UTIL.getConnection().getTable(LABELS_TABLE_NAME);) {
432      Scan scan = new Scan();
433      scan.setAuthorizations(new Authorizations(VisibilityUtils.SYSTEM_LABEL));
434      ResultScanner scanner = ht.getScanner(scan);
435      Result result = null;
436      List<Result> results = new ArrayList<>();
437      while ((result = scanner.next()) != null) {
438        results.add(result);
439      }
440      List<String> auths = extractAuths(user, results);
441      assertTrue(auths.contains(SECRET));
442      assertTrue(auths.contains(CONFIDENTIAL));
443      assertEquals(2, auths.size());
444    }
445
446    action = new PrivilegedExceptionAction<Void>() {
447      @Override
448      public Void run() throws Exception {
449        GetAuthsResponse authsResponse = null;
450        try (Connection conn = ConnectionFactory.createConnection(conf)) {
451          authsResponse = VisibilityClient.getAuths(conn, user);
452        } catch (Throwable e) {
453          fail("Should not have failed");
454        }
455        List<String> authsList = new ArrayList<>(authsResponse.getAuthList().size());
456        for (ByteString authBS : authsResponse.getAuthList()) {
457          authsList.add(Bytes.toString(authBS.toByteArray()));
458        }
459        assertEquals(2, authsList.size());
460        assertTrue(authsList.contains(SECRET));
461        assertTrue(authsList.contains(CONFIDENTIAL));
462        return null;
463      }
464    };
465    SUPERUSER.runAs(action);
466
467    // Try doing setAuths once again and there should not be any duplicates
468    action = new PrivilegedExceptionAction<Void>() {
469      @Override
470      public Void run() throws Exception {
471        String[] auths1 = { SECRET, CONFIDENTIAL };
472        GetAuthsResponse authsResponse = null;
473        try (Connection conn = ConnectionFactory.createConnection(conf)) {
474          VisibilityClient.setAuths(conn, auths1, user);
475          try {
476            authsResponse = VisibilityClient.getAuths(conn, user);
477          } catch (Throwable e) {
478            fail("Should not have failed");
479          }
480        } catch (Throwable e) {
481        }
482        List<String> authsList = new ArrayList<>(authsResponse.getAuthList().size());
483        for (ByteString authBS : authsResponse.getAuthList()) {
484          authsList.add(Bytes.toString(authBS.toByteArray()));
485        }
486        assertEquals(2, authsList.size());
487        assertTrue(authsList.contains(SECRET));
488        assertTrue(authsList.contains(CONFIDENTIAL));
489        return null;
490      }
491    };
492    SUPERUSER.runAs(action);
493  }
494
495  protected List<String> extractAuths(String user, List<Result> results) {
496    List<String> auths = new ArrayList<>();
497    for (Result result : results) {
498      Cell labelCell = result.getColumnLatestCell(LABELS_TABLE_FAMILY, LABEL_QUALIFIER);
499      Cell userAuthCell = result.getColumnLatestCell(LABELS_TABLE_FAMILY, Bytes.toBytes(user));
500      if (userAuthCell != null) {
501        auths.add(Bytes.toString(labelCell.getValueArray(), labelCell.getValueOffset(),
502            labelCell.getValueLength()));
503      }
504    }
505    return auths;
506  }
507
508  @Test
509  public void testClearUserAuths() throws Throwable {
510    PrivilegedExceptionAction<Void> action = new PrivilegedExceptionAction<Void>() {
511      @Override
512      public Void run() throws Exception {
513        String[] auths = { SECRET, CONFIDENTIAL, PRIVATE };
514        String user = "testUser";
515        try (Connection conn = ConnectionFactory.createConnection(conf)) {
516          VisibilityClient.setAuths(conn, auths, user);
517        } catch (Throwable e) {
518          fail("Should not have failed");
519        }
520        // Removing the auths for SECRET and CONFIDENTIAL for the user.
521        // Passing a non existing auth also.
522        auths = new String[] { SECRET, PUBLIC, CONFIDENTIAL };
523        VisibilityLabelsResponse response = null;
524        try (Connection conn = ConnectionFactory.createConnection(conf)) {
525          response = VisibilityClient.clearAuths(conn, auths, user);
526        } catch (Throwable e) {
527          fail("Should not have failed");
528        }
529        List<RegionActionResult> resultList = response.getResultList();
530        assertEquals(3, resultList.size());
531        assertTrue(resultList.get(0).getException().getValue().isEmpty());
532        assertEquals("org.apache.hadoop.hbase.DoNotRetryIOException",
533            resultList.get(1).getException().getName());
534        assertTrue(Bytes.toString(resultList.get(1).getException().getValue().toByteArray())
535            .contains(
536                "org.apache.hadoop.hbase.security.visibility.InvalidLabelException: "
537                    + "Label 'public' is not set for the user testUser"));
538        assertTrue(resultList.get(2).getException().getValue().isEmpty());
539        try (Connection connection = ConnectionFactory.createConnection(conf);
540             Table ht = connection.getTable(LABELS_TABLE_NAME)) {
541          ResultScanner scanner = ht.getScanner(new Scan());
542          Result result = null;
543          List<Result> results = new ArrayList<>();
544          while ((result = scanner.next()) != null) {
545            results.add(result);
546          }
547          List<String> curAuths = extractAuths(user, results);
548          assertTrue(curAuths.contains(PRIVATE));
549          assertEquals(1, curAuths.size());
550        }
551
552        GetAuthsResponse authsResponse = null;
553        try (Connection conn = ConnectionFactory.createConnection(conf)) {
554          authsResponse = VisibilityClient.getAuths(conn, user);
555        } catch (Throwable e) {
556          fail("Should not have failed");
557        }
558        List<String> authsList = new ArrayList<>(authsResponse.getAuthList().size());
559        for (ByteString authBS : authsResponse.getAuthList()) {
560          authsList.add(Bytes.toString(authBS.toByteArray()));
561        }
562        assertEquals(1, authsList.size());
563        assertTrue(authsList.contains(PRIVATE));
564        return null;
565      }
566    };
567    SUPERUSER.runAs(action);
568  }
569
570  @Test
571  public void testLabelsWithCheckAndPut() throws Throwable {
572    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
573    try (Table table = TEST_UTIL.createTable(tableName, fam)) {
574      byte[] row1 = Bytes.toBytes("row1");
575      Put put = new Put(row1);
576      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value);
577      put.setCellVisibility(new CellVisibility(SECRET + " & " + CONFIDENTIAL));
578      table.checkAndMutate(row1, fam).qualifier(qual).ifNotExists().thenPut(put);
579      byte[] row2 = Bytes.toBytes("row2");
580      put = new Put(row2);
581      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value);
582      put.setCellVisibility(new CellVisibility(SECRET));
583      table.checkAndMutate(row2, fam).qualifier(qual).ifNotExists().thenPut(put);
584
585      Scan scan = new Scan();
586      scan.setAuthorizations(new Authorizations(SECRET));
587      ResultScanner scanner = table.getScanner(scan);
588      Result result = scanner.next();
589      assertTrue(!result.isEmpty());
590      assertTrue(Bytes.equals(row2, result.getRow()));
591      result = scanner.next();
592      assertNull(result);
593    }
594  }
595
596  @Test
597  public void testLabelsWithIncrement() throws Throwable {
598    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
599    try (Table table = TEST_UTIL.createTable(tableName, fam)) {
600      byte[] row1 = Bytes.toBytes("row1");
601      byte[] val = Bytes.toBytes(1L);
602      Put put = new Put(row1);
603      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, val);
604      put.setCellVisibility(new CellVisibility(SECRET + " & " + CONFIDENTIAL));
605      table.put(put);
606      Get get = new Get(row1);
607      get.setAuthorizations(new Authorizations(SECRET));
608      Result result = table.get(get);
609      assertTrue(result.isEmpty());
610      table.incrementColumnValue(row1, fam, qual, 2L);
611      result = table.get(get);
612      assertTrue(result.isEmpty());
613      Increment increment = new Increment(row1);
614      increment.addColumn(fam, qual, 2L);
615      increment.setCellVisibility(new CellVisibility(SECRET));
616      table.increment(increment);
617      result = table.get(get);
618      assertTrue(!result.isEmpty());
619    }
620  }
621
622  @Test
623  public void testLabelsWithAppend() throws Throwable {
624    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
625    try (Table table = TEST_UTIL.createTable(tableName, fam);) {
626      byte[] row1 = Bytes.toBytes("row1");
627      byte[] val = Bytes.toBytes("a");
628      Put put = new Put(row1);
629      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, val);
630      put.setCellVisibility(new CellVisibility(SECRET + " & " + CONFIDENTIAL));
631      table.put(put);
632      Get get = new Get(row1);
633      get.setAuthorizations(new Authorizations(SECRET));
634      Result result = table.get(get);
635      assertTrue(result.isEmpty());
636      Append append = new Append(row1);
637      append.addColumn(fam, qual, Bytes.toBytes("b"));
638      table.append(append);
639      result = table.get(get);
640      assertTrue(result.isEmpty());
641      append = new Append(row1);
642      append.addColumn(fam, qual, Bytes.toBytes("c"));
643      append.setCellVisibility(new CellVisibility(SECRET));
644      table.append(append);
645      result = table.get(get);
646      assertTrue(!result.isEmpty());
647    }
648  }
649
650  @Test
651  public void testUserShouldNotDoDDLOpOnLabelsTable() throws Exception {
652    Admin admin = TEST_UTIL.getAdmin();
653    try {
654      admin.disableTable(LABELS_TABLE_NAME);
655      fail("Lables table should not get disabled by user.");
656    } catch (Exception e) {
657    }
658    try {
659      admin.deleteTable(LABELS_TABLE_NAME);
660      fail("Lables table should not get disabled by user.");
661    } catch (Exception e) {
662    }
663    try {
664      HColumnDescriptor hcd = new HColumnDescriptor("testFamily");
665      admin.addColumnFamily(LABELS_TABLE_NAME, hcd);
666      fail("Lables table should not get altered by user.");
667    } catch (Exception e) {
668    }
669    try {
670      admin.deleteColumnFamily(LABELS_TABLE_NAME, VisibilityConstants.LABELS_TABLE_FAMILY);
671      fail("Lables table should not get altered by user.");
672    } catch (Exception e) {
673    }
674    try {
675      HColumnDescriptor hcd = new HColumnDescriptor(VisibilityConstants.LABELS_TABLE_FAMILY);
676      hcd.setBloomFilterType(BloomType.ROWCOL);
677      admin.modifyColumnFamily(LABELS_TABLE_NAME, hcd);
678      fail("Lables table should not get altered by user.");
679    } catch (Exception e) {
680    }
681    try {
682      HTableDescriptor htd = new HTableDescriptor(LABELS_TABLE_NAME);
683      htd.addFamily(new HColumnDescriptor("f1"));
684      htd.addFamily(new HColumnDescriptor("f2"));
685      admin.modifyTable(LABELS_TABLE_NAME, htd);
686      fail("Lables table should not get altered by user.");
687    } catch (Exception e) {
688    }
689  }
690
691  @Test
692  public void testMultipleVersions() throws Exception {
693    final byte[] r1 = Bytes.toBytes("row1");
694    final byte[] r2 = Bytes.toBytes("row2");
695    final byte[] v1 = Bytes.toBytes("100");
696    final byte[] v2 = Bytes.toBytes("101");
697    final byte[] fam2 = Bytes.toBytes("info2");
698    final byte[] qual2 = Bytes.toBytes("qual2");
699    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
700    HTableDescriptor desc = new HTableDescriptor(tableName);
701    HColumnDescriptor col = new HColumnDescriptor(fam);// Default max versions is 1.
702    desc.addFamily(col);
703    col = new HColumnDescriptor(fam2);
704    col.setMaxVersions(5);
705    desc.addFamily(col);
706    TEST_UTIL.getAdmin().createTable(desc);
707    try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
708      Put put = new Put(r1);
709      put.addColumn(fam, qual, 3L, v1);
710      put.addColumn(fam, qual2, 3L, v1);
711      put.addColumn(fam2, qual, 3L, v1);
712      put.addColumn(fam2, qual2, 3L, v1);
713      put.setCellVisibility(new CellVisibility(SECRET));
714      table.put(put);
715      put = new Put(r1);
716      put.addColumn(fam, qual, 4L, v2);
717      put.addColumn(fam, qual2, 4L, v2);
718      put.addColumn(fam2, qual, 4L, v2);
719      put.addColumn(fam2, qual2, 4L, v2);
720      put.setCellVisibility(new CellVisibility(PRIVATE));
721      table.put(put);
722
723      put = new Put(r2);
724      put.addColumn(fam, qual, 3L, v1);
725      put.addColumn(fam, qual2, 3L, v1);
726      put.addColumn(fam2, qual, 3L, v1);
727      put.addColumn(fam2, qual2, 3L, v1);
728      put.setCellVisibility(new CellVisibility(SECRET));
729      table.put(put);
730      put = new Put(r2);
731      put.addColumn(fam, qual, 4L, v2);
732      put.addColumn(fam, qual2, 4L, v2);
733      put.addColumn(fam2, qual, 4L, v2);
734      put.addColumn(fam2, qual2, 4L, v2);
735      put.setCellVisibility(new CellVisibility(SECRET));
736      table.put(put);
737
738      Scan s = new Scan();
739      s.setMaxVersions(1);
740      s.setAuthorizations(new Authorizations(SECRET));
741      ResultScanner scanner = table.getScanner(s);
742      Result result = scanner.next();
743      assertTrue(Bytes.equals(r1, result.getRow()));
744      // for cf 'fam' max versions in HCD is 1. So the old version cells, which are having matching
745      // CellVisibility with Authorizations, should not get considered in the label evaluation at
746      // all.
747      assertNull(result.getColumnLatestCell(fam, qual));
748      assertNull(result.getColumnLatestCell(fam, qual2));
749      // for cf 'fam2' max versions in HCD is > 1. So we can consider the old version cells, which
750      // are having matching CellVisibility with Authorizations, in the label evaluation. It can
751      // just skip those recent versions for which visibility is not there as per the new version's
752      // CellVisibility. The old versions which are having visibility can be send back
753      Cell cell = result.getColumnLatestCell(fam2, qual);
754      assertNotNull(cell);
755      assertTrue(Bytes.equals(v1, 0, v1.length, cell.getValueArray(), cell.getValueOffset(),
756          cell.getValueLength()));
757      cell = result.getColumnLatestCell(fam2, qual2);
758      assertNotNull(cell);
759      assertTrue(Bytes.equals(v1, 0, v1.length, cell.getValueArray(), cell.getValueOffset(),
760          cell.getValueLength()));
761
762      result = scanner.next();
763      assertTrue(Bytes.equals(r2, result.getRow()));
764      cell = result.getColumnLatestCell(fam, qual);
765      assertNotNull(cell);
766      assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
767          cell.getValueLength()));
768      cell = result.getColumnLatestCell(fam, qual2);
769      assertNotNull(cell);
770      assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
771          cell.getValueLength()));
772      cell = result.getColumnLatestCell(fam2, qual);
773      assertNotNull(cell);
774      assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
775          cell.getValueLength()));
776      cell = result.getColumnLatestCell(fam2, qual2);
777      assertNotNull(cell);
778      assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
779          cell.getValueLength()));
780    }
781  }
782
783  @Test
784  public void testMutateRow() throws Exception {
785    final byte[] qual2 = Bytes.toBytes("qual2");
786    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
787    HTableDescriptor desc = new HTableDescriptor(tableName);
788    HColumnDescriptor col = new HColumnDescriptor(fam);
789    desc.addFamily(col);
790    TEST_UTIL.getAdmin().createTable(desc);
791    try (Table table = TEST_UTIL.getConnection().getTable(tableName)){
792      Put p1 = new Put(row1);
793      p1.addColumn(fam, qual, value);
794      p1.setCellVisibility(new CellVisibility(CONFIDENTIAL));
795
796      Put p2 = new Put(row1);
797      p2.addColumn(fam, qual2, value);
798      p2.setCellVisibility(new CellVisibility(SECRET));
799
800      RowMutations rm = new RowMutations(row1);
801      rm.add(p1);
802      rm.add(p2);
803
804      table.mutateRow(rm);
805
806      Get get = new Get(row1);
807      get.setAuthorizations(new Authorizations(CONFIDENTIAL));
808      Result result = table.get(get);
809      assertTrue(result.containsColumn(fam, qual));
810      assertFalse(result.containsColumn(fam, qual2));
811
812      get.setAuthorizations(new Authorizations(SECRET));
813      result = table.get(get);
814      assertFalse(result.containsColumn(fam, qual));
815      assertTrue(result.containsColumn(fam, qual2));
816    }
817  }
818
819  @Test
820  public void testFlushedFileWithVisibilityTags() throws Exception {
821    final byte[] qual2 = Bytes.toBytes("qual2");
822    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
823    HTableDescriptor desc = new HTableDescriptor(tableName);
824    HColumnDescriptor col = new HColumnDescriptor(fam);
825    desc.addFamily(col);
826    TEST_UTIL.getAdmin().createTable(desc);
827    try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
828      Put p1 = new Put(row1);
829      p1.addColumn(fam, qual, value);
830      p1.setCellVisibility(new CellVisibility(CONFIDENTIAL));
831
832      Put p2 = new Put(row1);
833      p2.addColumn(fam, qual2, value);
834      p2.setCellVisibility(new CellVisibility(SECRET));
835
836      RowMutations rm = new RowMutations(row1);
837      rm.add(p1);
838      rm.add(p2);
839
840      table.mutateRow(rm);
841    }
842    TEST_UTIL.getAdmin().flush(tableName);
843    List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(tableName);
844    HStore store = regions.get(0).getStore(fam);
845    Collection<HStoreFile> storefiles = store.getStorefiles();
846    assertTrue(storefiles.size() > 0);
847    for (HStoreFile storeFile : storefiles) {
848      assertTrue(storeFile.getReader().getHFileReader().getFileContext().isIncludesTags());
849    }
850  }
851
852  static Table createTableAndWriteDataWithLabels(TableName tableName, String... labelExps)
853      throws Exception {
854    List<Put> puts = new ArrayList<>(labelExps.length);
855    for (int i = 0; i < labelExps.length; i++) {
856      Put put = new Put(Bytes.toBytes("row" + (i+1)));
857      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value);
858      put.setCellVisibility(new CellVisibility(labelExps[i]));
859      puts.add(put);
860    }
861    Table table = TEST_UTIL.createTable(tableName, fam);
862    table.put(puts);
863    return table;
864  }
865
866  public static void addLabels() throws Exception {
867    PrivilegedExceptionAction<VisibilityLabelsResponse> action =
868        new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
869      @Override
870      public VisibilityLabelsResponse run() throws Exception {
871        String[] labels = { SECRET, TOPSECRET, CONFIDENTIAL, PUBLIC, PRIVATE, COPYRIGHT, ACCENT,
872            UNICODE_VIS_TAG, UC1, UC2 };
873        try (Connection conn = ConnectionFactory.createConnection(conf)) {
874          VisibilityClient.addLabels(conn, labels);
875        } catch (Throwable t) {
876          throw new IOException(t);
877        }
878        return null;
879      }
880    };
881    SUPERUSER.runAs(action);
882  }
883}