001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.security.visibility;
019
020import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_FAMILY;
021import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME;
022import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABEL_QUALIFIER;
023import static org.junit.Assert.assertEquals;
024import static org.junit.Assert.assertFalse;
025import static org.junit.Assert.assertNotNull;
026import static org.junit.Assert.assertNull;
027import static org.junit.Assert.assertTrue;
028import static org.junit.Assert.fail;
029
030import java.io.IOException;
031import java.security.PrivilegedExceptionAction;
032import java.util.ArrayList;
033import java.util.Collection;
034import java.util.List;
035import org.apache.hadoop.conf.Configuration;
036import org.apache.hadoop.hbase.Cell;
037import org.apache.hadoop.hbase.CellScanner;
038import org.apache.hadoop.hbase.HBaseTestingUtil;
039import org.apache.hadoop.hbase.HConstants;
040import org.apache.hadoop.hbase.TableName;
041import org.apache.hadoop.hbase.client.Admin;
042import org.apache.hadoop.hbase.client.Append;
043import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
044import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
045import org.apache.hadoop.hbase.client.Connection;
046import org.apache.hadoop.hbase.client.ConnectionFactory;
047import org.apache.hadoop.hbase.client.Get;
048import org.apache.hadoop.hbase.client.Increment;
049import org.apache.hadoop.hbase.client.Put;
050import org.apache.hadoop.hbase.client.Result;
051import org.apache.hadoop.hbase.client.ResultScanner;
052import org.apache.hadoop.hbase.client.RowMutations;
053import org.apache.hadoop.hbase.client.Scan;
054import org.apache.hadoop.hbase.client.Table;
055import org.apache.hadoop.hbase.client.TableDescriptor;
056import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
057import org.apache.hadoop.hbase.client.security.SecurityCapability;
058import org.apache.hadoop.hbase.regionserver.BloomType;
059import org.apache.hadoop.hbase.regionserver.HRegion;
060import org.apache.hadoop.hbase.regionserver.HRegionServer;
061import org.apache.hadoop.hbase.regionserver.HStore;
062import org.apache.hadoop.hbase.regionserver.HStoreFile;
063import org.apache.hadoop.hbase.security.User;
064import org.apache.hadoop.hbase.util.Bytes;
065import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
066import org.junit.After;
067import org.junit.AfterClass;
068import org.junit.Rule;
069import org.junit.Test;
070import org.junit.rules.TestName;
071
072import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
073
074import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.RegionActionResult;
075import org.apache.hadoop.hbase.shaded.protobuf.generated.VisibilityLabelsProtos.GetAuthsResponse;
076import org.apache.hadoop.hbase.shaded.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
077
078/**
079 * Base test class for visibility labels basic features
080 */
081public abstract class TestVisibilityLabels {
082
083  public static final String TOPSECRET = "topsecret";
084  public static final String PUBLIC = "public";
085  public static final String PRIVATE = "private";
086  public static final String CONFIDENTIAL = "confidential";
087  public static final String SECRET = "secret";
088  public static final String COPYRIGHT = "\u00A9ABC";
089  public static final String ACCENT = "\u0941";
090  public static final String UNICODE_VIS_TAG = COPYRIGHT + "\"" + ACCENT + "\\" + SECRET + "\""
091      + "\u0027&\\";
092  public static final String UC1 = "\u0027\"\u002b";
093  public static final String UC2 = "\u002d\u003f";
094  public static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
095  public static final byte[] row1 = Bytes.toBytes("row1");
096  public static final byte[] row2 = Bytes.toBytes("row2");
097  public static final byte[] row3 = Bytes.toBytes("row3");
098  public static final byte[] row4 = Bytes.toBytes("row4");
099  public final static byte[] fam = Bytes.toBytes("info");
100  public final static byte[] qual = Bytes.toBytes("qual");
101  public final static byte[] value = Bytes.toBytes("value");
102  public static Configuration conf;
103
104  private volatile boolean killedRS = false;
105  @Rule
106  public final TestName TEST_NAME = new TestName();
107  public static User SUPERUSER, USER1;
108
109  @AfterClass
110  public static void tearDownAfterClass() throws Exception {
111    TEST_UTIL.shutdownMiniCluster();
112  }
113
114  @After
115  public void tearDown() throws Exception {
116    killedRS = false;
117  }
118
119  @Test
120  public void testSecurityCapabilities() throws Exception {
121    List<SecurityCapability> capabilities = TEST_UTIL.getConnection().getAdmin()
122      .getSecurityCapabilities();
123    assertTrue("CELL_VISIBILITY capability is missing",
124      capabilities.contains(SecurityCapability.CELL_VISIBILITY));
125  }
126
127  @Test
128  public void testSimpleVisibilityLabels() throws Exception {
129    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
130    try (Table table = createTableAndWriteDataWithLabels(tableName, SECRET + "|" + CONFIDENTIAL,
131        PRIVATE + "|" + CONFIDENTIAL)) {
132      Scan s = new Scan();
133      s.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL, PRIVATE));
134      ResultScanner scanner = table.getScanner(s);
135      Result[] next = scanner.next(3);
136
137      assertTrue(next.length == 2);
138      CellScanner cellScanner = next[0].cellScanner();
139      cellScanner.advance();
140      Cell current = cellScanner.current();
141      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
142          current.getRowLength(), row1, 0, row1.length));
143      cellScanner = next[1].cellScanner();
144      cellScanner.advance();
145      current = cellScanner.current();
146      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
147          current.getRowLength(), row2, 0, row2.length));
148    }
149  }
150
151  @Test
152  public void testSimpleVisibilityLabelsWithUniCodeCharacters() throws Exception {
153    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
154    try (Table table = createTableAndWriteDataWithLabels(tableName,
155        SECRET + "|" + CellVisibility.quote(COPYRIGHT), "(" + CellVisibility.quote(COPYRIGHT)
156            + "&"  + CellVisibility.quote(ACCENT) + ")|" + CONFIDENTIAL,
157        CellVisibility.quote(UNICODE_VIS_TAG) + "&" + SECRET)) {
158      Scan s = new Scan();
159      s.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL, PRIVATE, COPYRIGHT, ACCENT,
160          UNICODE_VIS_TAG));
161      ResultScanner scanner = table.getScanner(s);
162      Result[] next = scanner.next(3);
163      assertTrue(next.length == 3);
164      CellScanner cellScanner = next[0].cellScanner();
165      cellScanner.advance();
166      Cell current = cellScanner.current();
167      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
168          current.getRowLength(), row1, 0, row1.length));
169      cellScanner = next[1].cellScanner();
170      cellScanner.advance();
171      current = cellScanner.current();
172      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
173          current.getRowLength(), row2, 0, row2.length));
174      cellScanner = next[2].cellScanner();
175      cellScanner.advance();
176      current = cellScanner.current();
177      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
178          current.getRowLength(), row3, 0, row3.length));
179    }
180  }
181
182  @Test
183  public void testAuthorizationsWithSpecialUnicodeCharacters() throws Exception {
184    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
185    try (Table table = createTableAndWriteDataWithLabels(tableName,
186        CellVisibility.quote(UC1) + "|" + CellVisibility.quote(UC2), CellVisibility.quote(UC1),
187        CellVisibility.quote(UNICODE_VIS_TAG))) {
188      Scan s = new Scan();
189      s.setAuthorizations(new Authorizations(UC1, UC2, ACCENT,
190          UNICODE_VIS_TAG));
191      ResultScanner scanner = table.getScanner(s);
192      Result[] next = scanner.next(3);
193      assertTrue(next.length == 3);
194      CellScanner cellScanner = next[0].cellScanner();
195      cellScanner.advance();
196      Cell current = cellScanner.current();
197      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
198          current.getRowLength(), row1, 0, row1.length));
199      cellScanner = next[1].cellScanner();
200      cellScanner.advance();
201      current = cellScanner.current();
202      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
203          current.getRowLength(), row2, 0, row2.length));
204      cellScanner = next[2].cellScanner();
205      cellScanner.advance();
206      current = cellScanner.current();
207      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
208          current.getRowLength(), row3, 0, row3.length));
209    }
210  }
211
212  @Test
213  public void testVisibilityLabelsWithComplexLabels() throws Exception {
214    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
215    try (Table table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|"
216        + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET, "(" + PRIVATE + "&" + CONFIDENTIAL + "&"
217        + SECRET + ")", "(" + PRIVATE + "&" + CONFIDENTIAL + "&" + SECRET + ")", "(" + PRIVATE
218        + "&" + CONFIDENTIAL + "&" + SECRET + ")")) {
219      Scan s = new Scan();
220      s.setAuthorizations(new Authorizations(TOPSECRET, CONFIDENTIAL, PRIVATE, PUBLIC, SECRET));
221      ResultScanner scanner = table.getScanner(s);
222      Result[] next = scanner.next(4);
223      assertEquals(3, next.length);
224      CellScanner cellScanner = next[0].cellScanner();
225      cellScanner.advance();
226      Cell current = cellScanner.current();
227      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
228          current.getRowLength(), row2, 0, row2.length));
229      cellScanner = next[1].cellScanner();
230      cellScanner.advance();
231      current = cellScanner.current();
232      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
233          current.getRowLength(), row3, 0, row3.length));
234      cellScanner = next[2].cellScanner();
235      cellScanner.advance();
236      current = cellScanner.current();
237      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
238          current.getRowLength(), row4, 0, row4.length));
239    }
240  }
241
242  @Test
243  public void testVisibilityLabelsThatDoesNotPassTheCriteria() throws Exception {
244    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
245    try (Table table = createTableAndWriteDataWithLabels(tableName,
246        "(" + SECRET + "|" + CONFIDENTIAL + ")", PRIVATE)){
247      Scan s = new Scan();
248      s.setAuthorizations(new Authorizations(PUBLIC));
249      ResultScanner scanner = table.getScanner(s);
250      Result[] next = scanner.next(3);
251      assertTrue(next.length == 0);
252    }
253  }
254
255  @Test
256  public void testVisibilityLabelsInPutsThatDoesNotMatchAnyDefinedLabels() throws Exception {
257    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
258    try {
259      createTableAndWriteDataWithLabels(tableName, "SAMPLE_LABEL", "TEST");
260      fail("Should have failed with failed sanity check exception");
261    } catch (Exception e) {
262    }
263  }
264
265  @Test
266  public void testVisibilityLabelsInScanThatDoesNotMatchAnyDefinedLabels() throws Exception {
267    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
268    try ( Table table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|"
269        + CONFIDENTIAL + ")", PRIVATE)){
270      Scan s = new Scan();
271      s.setAuthorizations(new Authorizations("SAMPLE"));
272      ResultScanner scanner = table.getScanner(s);
273      Result[] next = scanner.next(3);
274      assertTrue(next.length == 0);
275    }
276  }
277
278  @Test
279  public void testVisibilityLabelsWithGet() throws Exception {
280    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
281    try (Table table = createTableAndWriteDataWithLabels(tableName, SECRET + "&" + CONFIDENTIAL
282        + "&!" + PRIVATE, SECRET + "&" + CONFIDENTIAL + "&" + PRIVATE)) {
283      Get get = new Get(row1);
284      get.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL));
285      Result result = table.get(get);
286      assertTrue(!result.isEmpty());
287      Cell cell = result.getColumnLatestCell(fam, qual);
288      assertTrue(Bytes.equals(value, 0, value.length, cell.getValueArray(), cell.getValueOffset(),
289          cell.getValueLength()));
290    }
291  }
292
293  @Test
294  public void testVisibilityLabelsOnKillingOfRSContainingLabelsTable() throws Exception {
295    List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
296        .getRegionServerThreads();
297    int liveRS = 0;
298    for (RegionServerThread rsThreads : regionServerThreads) {
299      if (!rsThreads.getRegionServer().isAborted()) {
300        liveRS++;
301      }
302    }
303    if (liveRS == 1) {
304      TEST_UTIL.getHBaseCluster().startRegionServer();
305    }
306    Thread t1 = new Thread() {
307      @Override
308      public void run() {
309        List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
310            .getRegionServerThreads();
311        for (RegionServerThread rsThread : regionServerThreads) {
312          List<HRegion> onlineRegions = rsThread.getRegionServer().getRegions(
313              LABELS_TABLE_NAME);
314          if (onlineRegions.size() > 0) {
315            rsThread.getRegionServer().abort("Aborting ");
316            killedRS = true;
317            break;
318          }
319        }
320      }
321
322    };
323    t1.start();
324    final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
325    Thread t = new Thread() {
326      @Override
327      public void run() {
328        try {
329          while (!killedRS) {
330            Thread.sleep(1);
331          }
332          createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL + ")",
333              PRIVATE);
334        } catch (Exception e) {
335        }
336      }
337    };
338    t.start();
339    regionServerThreads = TEST_UTIL.getHBaseCluster().getRegionServerThreads();
340    while (!killedRS) {
341      Thread.sleep(10);
342    }
343    regionServerThreads = TEST_UTIL.getHBaseCluster().getRegionServerThreads();
344    for (RegionServerThread rsThread : regionServerThreads) {
345      while (true) {
346        if (!rsThread.getRegionServer().isAborted()) {
347          List<HRegion> onlineRegions = rsThread.getRegionServer().getRegions(
348              LABELS_TABLE_NAME);
349          if (onlineRegions.size() > 0) {
350            break;
351          } else {
352            Thread.sleep(10);
353          }
354        } else {
355          break;
356        }
357      }
358    }
359    TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000);
360    t.join();
361    try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
362      Scan s = new Scan();
363      s.setAuthorizations(new Authorizations(SECRET));
364      ResultScanner scanner = table.getScanner(s);
365      Result[] next = scanner.next(3);
366      assertTrue(next.length == 1);
367    }
368  }
369
370  @Test
371  public void testVisibilityLabelsOnRSRestart() throws Exception {
372    final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
373    List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
374        .getRegionServerThreads();
375    for (RegionServerThread rsThread : regionServerThreads) {
376      rsThread.getRegionServer().abort("Aborting ");
377    }
378    // Start one new RS
379    RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer();
380    waitForLabelsRegionAvailability(rs.getRegionServer());
381    try (Table table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL
382        + ")", PRIVATE)) {
383      Scan s = new Scan();
384      s.setAuthorizations(new Authorizations(SECRET));
385      ResultScanner scanner = table.getScanner(s);
386      Result[] next = scanner.next(3);
387      assertTrue(next.length == 1);
388    }
389  }
390
391  protected void waitForLabelsRegionAvailability(HRegionServer regionServer) {
392    while (!regionServer.isOnline()) {
393      try {
394        Thread.sleep(10);
395      } catch (InterruptedException e) {
396      }
397    }
398    while (regionServer.getRegions(LABELS_TABLE_NAME).isEmpty()) {
399      try {
400        Thread.sleep(10);
401      } catch (InterruptedException e) {
402      }
403    }
404  }
405
406  @Test
407  public void testVisibilityLabelsInGetThatDoesNotMatchAnyDefinedLabels() throws Exception {
408    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
409    try (Table table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL
410        + ")", PRIVATE)) {
411      Get get = new Get(row1);
412      get.setAuthorizations(new Authorizations("SAMPLE"));
413      Result result = table.get(get);
414      assertTrue(result.isEmpty());
415    }
416  }
417
418  @Test
419  public void testSetAndGetUserAuths() throws Throwable {
420    final String user = "user1";
421    PrivilegedExceptionAction<Void> action = new PrivilegedExceptionAction<Void>() {
422      @Override
423      public Void run() throws Exception {
424        String[] auths = { SECRET, CONFIDENTIAL };
425        try (Connection conn = ConnectionFactory.createConnection(conf)) {
426          VisibilityClient.setAuths(conn, auths, user);
427        } catch (Throwable e) {
428          throw new IOException(e);
429        }
430        return null;
431      }
432    };
433    SUPERUSER.runAs(action);
434    try (Table ht = TEST_UTIL.getConnection().getTable(LABELS_TABLE_NAME)) {
435      Scan scan = new Scan();
436      scan.setAuthorizations(new Authorizations(VisibilityUtils.SYSTEM_LABEL));
437      ResultScanner scanner = ht.getScanner(scan);
438      Result result = null;
439      List<Result> results = new ArrayList<>();
440      while ((result = scanner.next()) != null) {
441        results.add(result);
442      }
443      List<String> auths = extractAuths(user, results);
444      assertTrue(auths.contains(SECRET));
445      assertTrue(auths.contains(CONFIDENTIAL));
446      assertEquals(2, auths.size());
447    }
448
449    action = new PrivilegedExceptionAction<Void>() {
450      @Override
451      public Void run() throws Exception {
452        GetAuthsResponse authsResponse = null;
453        try (Connection conn = ConnectionFactory.createConnection(conf)) {
454          authsResponse = VisibilityClient.getAuths(conn, user);
455        } catch (Throwable e) {
456          throw new IOException(e);
457        }
458        List<String> authsList = new ArrayList<>(authsResponse.getAuthList().size());
459        for (ByteString authBS : authsResponse.getAuthList()) {
460          authsList.add(Bytes.toString(authBS.toByteArray()));
461        }
462        assertEquals(2, authsList.size());
463        assertTrue(authsList.contains(SECRET));
464        assertTrue(authsList.contains(CONFIDENTIAL));
465        return null;
466      }
467    };
468    SUPERUSER.runAs(action);
469
470    // Try doing setAuths once again and there should not be any duplicates
471    action = new PrivilegedExceptionAction<Void>() {
472      @Override
473      public Void run() throws Exception {
474        String[] auths1 = { SECRET, CONFIDENTIAL };
475        GetAuthsResponse authsResponse = null;
476        try (Connection conn = ConnectionFactory.createConnection(conf)) {
477          VisibilityClient.setAuths(conn, auths1, user);
478          try {
479            authsResponse = VisibilityClient.getAuths(conn, user);
480          } catch (Throwable e) {
481            throw new IOException(e);
482          }
483        } catch (Throwable e) {
484        }
485        List<String> authsList = new ArrayList<>(authsResponse.getAuthList().size());
486        for (ByteString authBS : authsResponse.getAuthList()) {
487          authsList.add(Bytes.toString(authBS.toByteArray()));
488        }
489        assertEquals(2, authsList.size());
490        assertTrue(authsList.contains(SECRET));
491        assertTrue(authsList.contains(CONFIDENTIAL));
492        return null;
493      }
494    };
495    SUPERUSER.runAs(action);
496  }
497
498  protected List<String> extractAuths(String user, List<Result> results) {
499    List<String> auths = new ArrayList<>();
500    for (Result result : results) {
501      Cell labelCell = result.getColumnLatestCell(LABELS_TABLE_FAMILY, LABEL_QUALIFIER);
502      Cell userAuthCell = result.getColumnLatestCell(LABELS_TABLE_FAMILY, Bytes.toBytes(user));
503      if (userAuthCell != null) {
504        auths.add(Bytes.toString(labelCell.getValueArray(), labelCell.getValueOffset(),
505            labelCell.getValueLength()));
506      }
507    }
508    return auths;
509  }
510
511  @Test
512  public void testClearUserAuths() throws Throwable {
513    PrivilegedExceptionAction<Void> action = new PrivilegedExceptionAction<Void>() {
514      @Override
515      public Void run() throws Exception {
516        String[] auths = { SECRET, CONFIDENTIAL, PRIVATE };
517        String user = "testUser";
518        try (Connection conn = ConnectionFactory.createConnection(conf)) {
519          VisibilityClient.setAuths(conn, auths, user);
520        } catch (Throwable e) {
521          throw new IOException(e);
522        }
523        // Removing the auths for SECRET and CONFIDENTIAL for the user.
524        // Passing a non existing auth also.
525        auths = new String[] { SECRET, PUBLIC, CONFIDENTIAL };
526        VisibilityLabelsResponse response = null;
527        try (Connection conn = ConnectionFactory.createConnection(conf)) {
528          response = VisibilityClient.clearAuths(conn, auths, user);
529        } catch (Throwable e) {
530          fail("Should not have failed");
531        }
532        List<RegionActionResult> resultList = response.getResultList();
533        assertEquals(3, resultList.size());
534        assertTrue(resultList.get(0).getException().getValue().isEmpty());
535        assertEquals("org.apache.hadoop.hbase.DoNotRetryIOException",
536            resultList.get(1).getException().getName());
537        assertTrue(Bytes.toString(resultList.get(1).getException().getValue().toByteArray())
538            .contains(
539                "org.apache.hadoop.hbase.security.visibility.InvalidLabelException: "
540                    + "Label 'public' is not set for the user testUser"));
541        assertTrue(resultList.get(2).getException().getValue().isEmpty());
542        try (Connection connection = ConnectionFactory.createConnection(conf);
543             Table ht = connection.getTable(LABELS_TABLE_NAME)) {
544          ResultScanner scanner = ht.getScanner(new Scan());
545          Result result = null;
546          List<Result> results = new ArrayList<>();
547          while ((result = scanner.next()) != null) {
548            results.add(result);
549          }
550          List<String> curAuths = extractAuths(user, results);
551          assertTrue(curAuths.contains(PRIVATE));
552          assertEquals(1, curAuths.size());
553        }
554
555        GetAuthsResponse authsResponse = null;
556        try (Connection conn = ConnectionFactory.createConnection(conf)) {
557          authsResponse = VisibilityClient.getAuths(conn, user);
558        } catch (Throwable e) {
559          throw new IOException(e);
560        }
561        List<String> authsList = new ArrayList<>(authsResponse.getAuthList().size());
562        for (ByteString authBS : authsResponse.getAuthList()) {
563          authsList.add(Bytes.toString(authBS.toByteArray()));
564        }
565        assertEquals(1, authsList.size());
566        assertTrue(authsList.contains(PRIVATE));
567        return null;
568      }
569    };
570    SUPERUSER.runAs(action);
571  }
572
573  @Test
574  public void testLabelsWithCheckAndPut() throws Throwable {
575    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
576    try (Table table = TEST_UTIL.createTable(tableName, fam)) {
577      byte[] row1 = Bytes.toBytes("row1");
578      Put put = new Put(row1);
579      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value);
580      put.setCellVisibility(new CellVisibility(SECRET + " & " + CONFIDENTIAL));
581      table.checkAndMutate(row1, fam).qualifier(qual).ifNotExists().thenPut(put);
582      byte[] row2 = Bytes.toBytes("row2");
583      put = new Put(row2);
584      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value);
585      put.setCellVisibility(new CellVisibility(SECRET));
586      table.checkAndMutate(row2, fam).qualifier(qual).ifNotExists().thenPut(put);
587
588      Scan scan = new Scan();
589      scan.setAuthorizations(new Authorizations(SECRET));
590      ResultScanner scanner = table.getScanner(scan);
591      Result result = scanner.next();
592      assertTrue(!result.isEmpty());
593      assertTrue(Bytes.equals(row2, result.getRow()));
594      result = scanner.next();
595      assertNull(result);
596    }
597  }
598
599  @Test
600  public void testLabelsWithIncrement() throws Throwable {
601    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
602    try (Table table = TEST_UTIL.createTable(tableName, fam)) {
603      byte[] row1 = Bytes.toBytes("row1");
604      byte[] val = Bytes.toBytes(1L);
605      Put put = new Put(row1);
606      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, val);
607      put.setCellVisibility(new CellVisibility(SECRET + " & " + CONFIDENTIAL));
608      table.put(put);
609      Get get = new Get(row1);
610      get.setAuthorizations(new Authorizations(SECRET));
611      Result result = table.get(get);
612      assertTrue(result.isEmpty());
613      table.incrementColumnValue(row1, fam, qual, 2L);
614      result = table.get(get);
615      assertTrue(result.isEmpty());
616      Increment increment = new Increment(row1);
617      increment.addColumn(fam, qual, 2L);
618      increment.setCellVisibility(new CellVisibility(SECRET));
619      table.increment(increment);
620      result = table.get(get);
621      assertTrue(!result.isEmpty());
622    }
623  }
624
625  @Test
626  public void testLabelsWithAppend() throws Throwable {
627    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
628    try (Table table = TEST_UTIL.createTable(tableName, fam)) {
629      byte[] row1 = Bytes.toBytes("row1");
630      byte[] val = Bytes.toBytes("a");
631      Put put = new Put(row1);
632      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, val);
633      put.setCellVisibility(new CellVisibility(SECRET + " & " + CONFIDENTIAL));
634      table.put(put);
635      Get get = new Get(row1);
636      get.setAuthorizations(new Authorizations(SECRET));
637      Result result = table.get(get);
638      assertTrue(result.isEmpty());
639      Append append = new Append(row1);
640      append.addColumn(fam, qual, Bytes.toBytes("b"));
641      table.append(append);
642      result = table.get(get);
643      assertTrue(result.isEmpty());
644      append = new Append(row1);
645      append.addColumn(fam, qual, Bytes.toBytes("c"));
646      append.setCellVisibility(new CellVisibility(SECRET));
647      table.append(append);
648      result = table.get(get);
649      assertTrue(!result.isEmpty());
650    }
651  }
652
653  @Test
654  public void testUserShouldNotDoDDLOpOnLabelsTable() throws Exception {
655    Admin admin = TEST_UTIL.getAdmin();
656    try {
657      admin.disableTable(LABELS_TABLE_NAME);
658      fail("Lables table should not get disabled by user.");
659    } catch (Exception e) {
660    }
661    try {
662      admin.deleteTable(LABELS_TABLE_NAME);
663      fail("Lables table should not get disabled by user.");
664    } catch (Exception e) {
665    }
666    try {
667      ColumnFamilyDescriptor columnFamilyDescriptor =
668        ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("testFamily")).build();
669      admin.addColumnFamily(LABELS_TABLE_NAME, columnFamilyDescriptor);
670      fail("Lables table should not get altered by user.");
671    } catch (Exception e) {
672    }
673    try {
674      admin.deleteColumnFamily(LABELS_TABLE_NAME, VisibilityConstants.LABELS_TABLE_FAMILY);
675      fail("Lables table should not get altered by user.");
676    } catch (Exception e) {
677    }
678    try {
679      ColumnFamilyDescriptor familyDescriptor =
680        ColumnFamilyDescriptorBuilder.newBuilder(VisibilityConstants.LABELS_TABLE_FAMILY)
681          .setBloomFilterType(BloomType.ROWCOL).build();
682      admin.modifyColumnFamily(LABELS_TABLE_NAME, familyDescriptor);
683      fail("Lables table should not get altered by user.");
684    } catch (Exception e) {
685    }
686    try {
687      TableDescriptorBuilder tableDescriptorBuilder =
688        TableDescriptorBuilder.newBuilder(LABELS_TABLE_NAME);
689      ColumnFamilyDescriptor columnFamilyDescriptor =
690        ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("f1")).build();
691      tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
692      columnFamilyDescriptor = ColumnFamilyDescriptorBuilder
693        .newBuilder(Bytes.toBytes("f2")).build();
694      tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
695      admin.modifyTable(tableDescriptorBuilder.build());
696      fail("Lables table should not get altered by user.");
697    } catch (Exception e) {
698    }
699  }
700
701  @Test
702  public void testMultipleVersions() throws Exception {
703    final byte[] r1 = Bytes.toBytes("row1");
704    final byte[] r2 = Bytes.toBytes("row2");
705    final byte[] v1 = Bytes.toBytes("100");
706    final byte[] v2 = Bytes.toBytes("101");
707    final byte[] fam2 = Bytes.toBytes("info2");
708    final byte[] qual2 = Bytes.toBytes("qual2");
709    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
710    // Default max versions is 1.
711    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
712      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam))
713      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam2).setMaxVersions(5).build())
714      .build();
715    TEST_UTIL.getAdmin().createTable(tableDescriptor);
716    try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
717      Put put = new Put(r1);
718      put.addColumn(fam, qual, 3L, v1);
719      put.addColumn(fam, qual2, 3L, v1);
720      put.addColumn(fam2, qual, 3L, v1);
721      put.addColumn(fam2, qual2, 3L, v1);
722      put.setCellVisibility(new CellVisibility(SECRET));
723      table.put(put);
724      put = new Put(r1);
725      put.addColumn(fam, qual, 4L, v2);
726      put.addColumn(fam, qual2, 4L, v2);
727      put.addColumn(fam2, qual, 4L, v2);
728      put.addColumn(fam2, qual2, 4L, v2);
729      put.setCellVisibility(new CellVisibility(PRIVATE));
730      table.put(put);
731
732      put = new Put(r2);
733      put.addColumn(fam, qual, 3L, v1);
734      put.addColumn(fam, qual2, 3L, v1);
735      put.addColumn(fam2, qual, 3L, v1);
736      put.addColumn(fam2, qual2, 3L, v1);
737      put.setCellVisibility(new CellVisibility(SECRET));
738      table.put(put);
739      put = new Put(r2);
740      put.addColumn(fam, qual, 4L, v2);
741      put.addColumn(fam, qual2, 4L, v2);
742      put.addColumn(fam2, qual, 4L, v2);
743      put.addColumn(fam2, qual2, 4L, v2);
744      put.setCellVisibility(new CellVisibility(SECRET));
745      table.put(put);
746
747      Scan s = new Scan();
748      s.readVersions(1);
749      s.setAuthorizations(new Authorizations(SECRET));
750      ResultScanner scanner = table.getScanner(s);
751      Result result = scanner.next();
752      assertTrue(Bytes.equals(r1, result.getRow()));
753      // for cf 'fam' max versions in HCD is 1. So the old version cells, which are having matching
754      // CellVisibility with Authorizations, should not get considered in the label evaluation at
755      // all.
756      assertNull(result.getColumnLatestCell(fam, qual));
757      assertNull(result.getColumnLatestCell(fam, qual2));
758      // for cf 'fam2' max versions in HCD is > 1. So we can consider the old version cells, which
759      // are having matching CellVisibility with Authorizations, in the label evaluation. It can
760      // just skip those recent versions for which visibility is not there as per the new version's
761      // CellVisibility. The old versions which are having visibility can be send back
762      Cell cell = result.getColumnLatestCell(fam2, qual);
763      assertNotNull(cell);
764      assertTrue(Bytes.equals(v1, 0, v1.length, cell.getValueArray(), cell.getValueOffset(),
765          cell.getValueLength()));
766      cell = result.getColumnLatestCell(fam2, qual2);
767      assertNotNull(cell);
768      assertTrue(Bytes.equals(v1, 0, v1.length, cell.getValueArray(), cell.getValueOffset(),
769          cell.getValueLength()));
770
771      result = scanner.next();
772      assertTrue(Bytes.equals(r2, result.getRow()));
773      cell = result.getColumnLatestCell(fam, qual);
774      assertNotNull(cell);
775      assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
776          cell.getValueLength()));
777      cell = result.getColumnLatestCell(fam, qual2);
778      assertNotNull(cell);
779      assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
780          cell.getValueLength()));
781      cell = result.getColumnLatestCell(fam2, qual);
782      assertNotNull(cell);
783      assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
784          cell.getValueLength()));
785      cell = result.getColumnLatestCell(fam2, qual2);
786      assertNotNull(cell);
787      assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
788          cell.getValueLength()));
789    }
790  }
791
792  @Test
793  public void testMutateRow() throws Exception {
794    final byte[] qual2 = Bytes.toBytes("qual2");
795    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
796    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
797      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam)).build();
798    TEST_UTIL.getAdmin().createTable(tableDescriptor);
799    try (Table table = TEST_UTIL.getConnection().getTable(tableName)){
800      Put p1 = new Put(row1);
801      p1.addColumn(fam, qual, value);
802      p1.setCellVisibility(new CellVisibility(CONFIDENTIAL));
803
804      Put p2 = new Put(row1);
805      p2.addColumn(fam, qual2, value);
806      p2.setCellVisibility(new CellVisibility(SECRET));
807
808      RowMutations rm = new RowMutations(row1);
809      rm.add(p1);
810      rm.add(p2);
811
812      table.mutateRow(rm);
813
814      Get get = new Get(row1);
815      get.setAuthorizations(new Authorizations(CONFIDENTIAL));
816      Result result = table.get(get);
817      assertTrue(result.containsColumn(fam, qual));
818      assertFalse(result.containsColumn(fam, qual2));
819
820      get.setAuthorizations(new Authorizations(SECRET));
821      result = table.get(get);
822      assertFalse(result.containsColumn(fam, qual));
823      assertTrue(result.containsColumn(fam, qual2));
824    }
825  }
826
827  @Test
828  public void testFlushedFileWithVisibilityTags() throws Exception {
829    final byte[] qual2 = Bytes.toBytes("qual2");
830    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
831    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
832      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam)).build();
833    TEST_UTIL.getAdmin().createTable(tableDescriptor);
834    try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
835      Put p1 = new Put(row1);
836      p1.addColumn(fam, qual, value);
837      p1.setCellVisibility(new CellVisibility(CONFIDENTIAL));
838
839      Put p2 = new Put(row1);
840      p2.addColumn(fam, qual2, value);
841      p2.setCellVisibility(new CellVisibility(SECRET));
842
843      RowMutations rm = new RowMutations(row1);
844      rm.add(p1);
845      rm.add(p2);
846
847      table.mutateRow(rm);
848    }
849    TEST_UTIL.getAdmin().flush(tableName);
850    List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(tableName);
851    HStore store = regions.get(0).getStore(fam);
852    Collection<HStoreFile> storefiles = store.getStorefiles();
853    assertTrue(storefiles.size() > 0);
854    for (HStoreFile storeFile : storefiles) {
855      assertTrue(storeFile.getReader().getHFileReader().getFileContext().isIncludesTags());
856    }
857  }
858
859  static Table createTableAndWriteDataWithLabels(TableName tableName, String... labelExps)
860      throws Exception {
861    List<Put> puts = new ArrayList<>(labelExps.length);
862    for (int i = 0; i < labelExps.length; i++) {
863      Put put = new Put(Bytes.toBytes("row" + (i+1)));
864      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value);
865      put.setCellVisibility(new CellVisibility(labelExps[i]));
866      puts.add(put);
867    }
868    Table table = TEST_UTIL.createTable(tableName, fam);
869    table.put(puts);
870    return table;
871  }
872
873  public static void addLabels() throws Exception {
874    PrivilegedExceptionAction<VisibilityLabelsResponse> action =
875        new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
876      @Override
877      public VisibilityLabelsResponse run() throws Exception {
878        String[] labels = { SECRET, TOPSECRET, CONFIDENTIAL, PUBLIC, PRIVATE, COPYRIGHT, ACCENT,
879            UNICODE_VIS_TAG, UC1, UC2 };
880        try (Connection conn = ConnectionFactory.createConnection(conf)) {
881          VisibilityClient.addLabels(conn, labels);
882        } catch (Throwable t) {
883          throw new IOException(t);
884        }
885        return null;
886      }
887    };
888    SUPERUSER.runAs(action);
889  }
890}