Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.security.visibility;
019
020import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_FAMILY;
021import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME;
022import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABEL_QUALIFIER;
023import static org.junit.Assert.assertEquals;
024import static org.junit.Assert.assertFalse;
025import static org.junit.Assert.assertNotNull;
026import static org.junit.Assert.assertNull;
027import static org.junit.Assert.assertTrue;
028import static org.junit.Assert.fail;
029
030import com.google.protobuf.ByteString;
031import java.io.IOException;
032import java.security.PrivilegedExceptionAction;
033import java.util.ArrayList;
034import java.util.Collection;
035import java.util.List;
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.hbase.Cell;
038import org.apache.hadoop.hbase.CellScanner;
039import org.apache.hadoop.hbase.HBaseTestingUtility;
040import org.apache.hadoop.hbase.HColumnDescriptor;
041import org.apache.hadoop.hbase.HConstants;
042import org.apache.hadoop.hbase.HTableDescriptor;
043import org.apache.hadoop.hbase.TableName;
044import org.apache.hadoop.hbase.client.Admin;
045import org.apache.hadoop.hbase.client.Append;
046import org.apache.hadoop.hbase.client.Connection;
047import org.apache.hadoop.hbase.client.ConnectionFactory;
048import org.apache.hadoop.hbase.client.Get;
049import org.apache.hadoop.hbase.client.Increment;
050import org.apache.hadoop.hbase.client.Put;
051import org.apache.hadoop.hbase.client.Result;
052import org.apache.hadoop.hbase.client.ResultScanner;
053import org.apache.hadoop.hbase.client.RowMutations;
054import org.apache.hadoop.hbase.client.Scan;
055import org.apache.hadoop.hbase.client.Table;
056import org.apache.hadoop.hbase.client.security.SecurityCapability;
057import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
058import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.GetAuthsResponse;
059import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
060import org.apache.hadoop.hbase.regionserver.BloomType;
061import org.apache.hadoop.hbase.regionserver.HRegion;
062import org.apache.hadoop.hbase.regionserver.HRegionServer;
063import org.apache.hadoop.hbase.regionserver.HStore;
064import org.apache.hadoop.hbase.regionserver.HStoreFile;
065import org.apache.hadoop.hbase.security.User;
066import org.apache.hadoop.hbase.util.Bytes;
067import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
068import org.junit.After;
069import org.junit.AfterClass;
070import org.junit.Rule;
071import org.junit.Test;
072import org.junit.rules.TestName;
073
074/**
075 * Base test class for visibility labels basic features
076 */
077public abstract class TestVisibilityLabels {
078
079  public static final String TOPSECRET = "topsecret";
080  public static final String PUBLIC = "public";
081  public static final String PRIVATE = "private";
082  public static final String CONFIDENTIAL = "confidential";
083  public static final String SECRET = "secret";
084  public static final String COPYRIGHT = "\u00A9ABC";
085  public static final String ACCENT = "\u0941";
086  public static final String UNICODE_VIS_TAG =
087    COPYRIGHT + "\"" + ACCENT + "\\" + SECRET + "\"" + "\u0027&\\";
088  public static final String UC1 = "\u0027\"\u002b";
089  public static final String UC2 = "\u002d\u003f";
090  public static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
091  public static final byte[] row1 = Bytes.toBytes("row1");
092  public static final byte[] row2 = Bytes.toBytes("row2");
093  public static final byte[] row3 = Bytes.toBytes("row3");
094  public static final byte[] row4 = Bytes.toBytes("row4");
095  public final static byte[] fam = Bytes.toBytes("info");
096  public final static byte[] qual = Bytes.toBytes("qual");
097  public final static byte[] value = Bytes.toBytes("value");
098  public static Configuration conf;
099
100  private volatile boolean killedRS = false;
101  @Rule
102  public final TestName TEST_NAME = new TestName();
103  public static User SUPERUSER, USER1;
104
105  @AfterClass
106  public static void tearDownAfterClass() throws Exception {
107    TEST_UTIL.shutdownMiniCluster();
108  }
109
110  @After
111  public void tearDown() throws Exception {
112    killedRS = false;
113  }
114
115  @Test
116  public void testSecurityCapabilities() throws Exception {
117    List<SecurityCapability> capabilities =
118      TEST_UTIL.getConnection().getAdmin().getSecurityCapabilities();
119    assertTrue("CELL_VISIBILITY capability is missing",
120      capabilities.contains(SecurityCapability.CELL_VISIBILITY));
121  }
122
123  @Test
124  public void testSimpleVisibilityLabels() throws Exception {
125    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
126    try (Table table = createTableAndWriteDataWithLabels(tableName, SECRET + "|" + CONFIDENTIAL,
127      PRIVATE + "|" + CONFIDENTIAL)) {
128      Scan s = new Scan();
129      s.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL, PRIVATE));
130      ResultScanner scanner = table.getScanner(s);
131      Result[] next = scanner.next(3);
132
133      assertTrue(next.length == 2);
134      CellScanner cellScanner = next[0].cellScanner();
135      cellScanner.advance();
136      Cell current = cellScanner.current();
137      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
138        row1, 0, row1.length));
139      cellScanner = next[1].cellScanner();
140      cellScanner.advance();
141      current = cellScanner.current();
142      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
143        row2, 0, row2.length));
144    }
145  }
146
147  @Test
148  public void testSimpleVisibilityLabelsWithUniCodeCharacters() throws Exception {
149    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
150    try (Table table = createTableAndWriteDataWithLabels(tableName,
151      SECRET + "|" + CellVisibility.quote(COPYRIGHT), "(" + CellVisibility.quote(COPYRIGHT) + "&"
152        + CellVisibility.quote(ACCENT) + ")|" + CONFIDENTIAL,
153      CellVisibility.quote(UNICODE_VIS_TAG) + "&" + SECRET)) {
154      Scan s = new Scan();
155      s.setAuthorizations(
156        new Authorizations(SECRET, CONFIDENTIAL, PRIVATE, COPYRIGHT, ACCENT, UNICODE_VIS_TAG));
157      ResultScanner scanner = table.getScanner(s);
158      Result[] next = scanner.next(3);
159      assertTrue(next.length == 3);
160      CellScanner cellScanner = next[0].cellScanner();
161      cellScanner.advance();
162      Cell current = cellScanner.current();
163      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
164        row1, 0, row1.length));
165      cellScanner = next[1].cellScanner();
166      cellScanner.advance();
167      current = cellScanner.current();
168      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
169        row2, 0, row2.length));
170      cellScanner = next[2].cellScanner();
171      cellScanner.advance();
172      current = cellScanner.current();
173      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
174        row3, 0, row3.length));
175    }
176  }
177
178  @Test
179  public void testAuthorizationsWithSpecialUnicodeCharacters() throws Exception {
180    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
181    try (Table table = createTableAndWriteDataWithLabels(tableName,
182      CellVisibility.quote(UC1) + "|" + CellVisibility.quote(UC2), CellVisibility.quote(UC1),
183      CellVisibility.quote(UNICODE_VIS_TAG))) {
184      Scan s = new Scan();
185      s.setAuthorizations(new Authorizations(UC1, UC2, ACCENT, UNICODE_VIS_TAG));
186      ResultScanner scanner = table.getScanner(s);
187      Result[] next = scanner.next(3);
188      assertTrue(next.length == 3);
189      CellScanner cellScanner = next[0].cellScanner();
190      cellScanner.advance();
191      Cell current = cellScanner.current();
192      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
193        row1, 0, row1.length));
194      cellScanner = next[1].cellScanner();
195      cellScanner.advance();
196      current = cellScanner.current();
197      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
198        row2, 0, row2.length));
199      cellScanner = next[2].cellScanner();
200      cellScanner.advance();
201      current = cellScanner.current();
202      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
203        row3, 0, row3.length));
204    }
205  }
206
207  @Test
208  public void testVisibilityLabelsWithComplexLabels() throws Exception {
209    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
210    try (Table table = createTableAndWriteDataWithLabels(tableName,
211      "(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET,
212      "(" + PRIVATE + "&" + CONFIDENTIAL + "&" + SECRET + ")",
213      "(" + PRIVATE + "&" + CONFIDENTIAL + "&" + SECRET + ")",
214      "(" + PRIVATE + "&" + CONFIDENTIAL + "&" + SECRET + ")")) {
215      Scan s = new Scan();
216      s.setAuthorizations(new Authorizations(TOPSECRET, CONFIDENTIAL, PRIVATE, PUBLIC, SECRET));
217      ResultScanner scanner = table.getScanner(s);
218      Result[] next = scanner.next(4);
219      assertEquals(3, next.length);
220      CellScanner cellScanner = next[0].cellScanner();
221      cellScanner.advance();
222      Cell current = cellScanner.current();
223      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
224        row2, 0, row2.length));
225      cellScanner = next[1].cellScanner();
226      cellScanner.advance();
227      current = cellScanner.current();
228      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
229        row3, 0, row3.length));
230      cellScanner = next[2].cellScanner();
231      cellScanner.advance();
232      current = cellScanner.current();
233      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
234        row4, 0, row4.length));
235    }
236  }
237
238  @Test
239  public void testVisibilityLabelsThatDoesNotPassTheCriteria() throws Exception {
240    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
241    try (Table table = createTableAndWriteDataWithLabels(tableName,
242      "(" + SECRET + "|" + CONFIDENTIAL + ")", PRIVATE)) {
243      Scan s = new Scan();
244      s.setAuthorizations(new Authorizations(PUBLIC));
245      ResultScanner scanner = table.getScanner(s);
246      Result[] next = scanner.next(3);
247      assertTrue(next.length == 0);
248    }
249  }
250
251  @Test
252  public void testVisibilityLabelsInPutsThatDoesNotMatchAnyDefinedLabels() throws Exception {
253    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
254    try {
255      createTableAndWriteDataWithLabels(tableName, "SAMPLE_LABEL", "TEST");
256      fail("Should have failed with failed sanity check exception");
257    } catch (Exception e) {
258    }
259  }
260
261  @Test
262  public void testVisibilityLabelsInScanThatDoesNotMatchAnyDefinedLabels() throws Exception {
263    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
264    try (Table table = createTableAndWriteDataWithLabels(tableName,
265      "(" + SECRET + "|" + CONFIDENTIAL + ")", PRIVATE)) {
266      Scan s = new Scan();
267      s.setAuthorizations(new Authorizations("SAMPLE"));
268      ResultScanner scanner = table.getScanner(s);
269      Result[] next = scanner.next(3);
270      assertTrue(next.length == 0);
271    }
272  }
273
274  @Test
275  public void testVisibilityLabelsWithGet() throws Exception {
276    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
277    try (Table table = createTableAndWriteDataWithLabels(tableName,
278      SECRET + "&" + CONFIDENTIAL + "&!" + PRIVATE, SECRET + "&" + CONFIDENTIAL + "&" + PRIVATE)) {
279      Get get = new Get(row1);
280      get.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL));
281      Result result = table.get(get);
282      assertTrue(!result.isEmpty());
283      Cell cell = result.getColumnLatestCell(fam, qual);
284      assertTrue(Bytes.equals(value, 0, value.length, cell.getValueArray(), cell.getValueOffset(),
285        cell.getValueLength()));
286    }
287  }
288
289  @Test
290  public void testVisibilityLabelsOnKillingOfRSContainingLabelsTable() throws Exception {
291    List<RegionServerThread> regionServerThreads =
292      TEST_UTIL.getHBaseCluster().getRegionServerThreads();
293    int liveRS = 0;
294    for (RegionServerThread rsThreads : regionServerThreads) {
295      if (!rsThreads.getRegionServer().isAborted()) {
296        liveRS++;
297      }
298    }
299    if (liveRS == 1) {
300      TEST_UTIL.getHBaseCluster().startRegionServer();
301    }
302    Thread t1 = new Thread() {
303      @Override
304      public void run() {
305        List<RegionServerThread> regionServerThreads =
306          TEST_UTIL.getHBaseCluster().getRegionServerThreads();
307        for (RegionServerThread rsThread : regionServerThreads) {
308          List<HRegion> onlineRegions = rsThread.getRegionServer().getRegions(LABELS_TABLE_NAME);
309          if (onlineRegions.size() > 0) {
310            rsThread.getRegionServer().abort("Aborting ");
311            killedRS = true;
312            break;
313          }
314        }
315      }
316
317    };
318    t1.start();
319    final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
320    Thread t = new Thread() {
321      @Override
322      public void run() {
323        try {
324          while (!killedRS) {
325            Thread.sleep(1);
326          }
327          createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL + ")",
328            PRIVATE);
329        } catch (Exception e) {
330        }
331      }
332    };
333    t.start();
334    regionServerThreads = TEST_UTIL.getHBaseCluster().getRegionServerThreads();
335    while (!killedRS) {
336      Thread.sleep(10);
337    }
338    regionServerThreads = TEST_UTIL.getHBaseCluster().getRegionServerThreads();
339    for (RegionServerThread rsThread : regionServerThreads) {
340      while (true) {
341        if (!rsThread.getRegionServer().isAborted()) {
342          List<HRegion> onlineRegions = rsThread.getRegionServer().getRegions(LABELS_TABLE_NAME);
343          if (onlineRegions.size() > 0) {
344            break;
345          } else {
346            Thread.sleep(10);
347          }
348        } else {
349          break;
350        }
351      }
352    }
353    TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000);
354    t.join();
355    try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
356      Scan s = new Scan();
357      s.setAuthorizations(new Authorizations(SECRET));
358      ResultScanner scanner = table.getScanner(s);
359      Result[] next = scanner.next(3);
360      assertTrue(next.length == 1);
361    }
362  }
363
364  @Test
365  public void testVisibilityLabelsOnRSRestart() throws Exception {
366    final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
367    List<RegionServerThread> regionServerThreads =
368      TEST_UTIL.getHBaseCluster().getRegionServerThreads();
369    for (RegionServerThread rsThread : regionServerThreads) {
370      rsThread.getRegionServer().abort("Aborting ");
371    }
372    // Start one new RS
373    RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer();
374    waitForLabelsRegionAvailability(rs.getRegionServer());
375    try (Table table = createTableAndWriteDataWithLabels(tableName,
376      "(" + SECRET + "|" + CONFIDENTIAL + ")", PRIVATE);) {
377      Scan s = new Scan();
378      s.setAuthorizations(new Authorizations(SECRET));
379      ResultScanner scanner = table.getScanner(s);
380      Result[] next = scanner.next(3);
381      assertTrue(next.length == 1);
382    }
383  }
384
385  protected void waitForLabelsRegionAvailability(HRegionServer regionServer) {
386    while (!regionServer.isOnline()) {
387      try {
388        Thread.sleep(10);
389      } catch (InterruptedException e) {
390      }
391    }
392    while (regionServer.getRegions(LABELS_TABLE_NAME).isEmpty()) {
393      try {
394        Thread.sleep(10);
395      } catch (InterruptedException e) {
396      }
397    }
398  }
399
400  @Test
401  public void testVisibilityLabelsInGetThatDoesNotMatchAnyDefinedLabels() throws Exception {
402    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
403    try (Table table = createTableAndWriteDataWithLabels(tableName,
404      "(" + SECRET + "|" + CONFIDENTIAL + ")", PRIVATE)) {
405      Get get = new Get(row1);
406      get.setAuthorizations(new Authorizations("SAMPLE"));
407      Result result = table.get(get);
408      assertTrue(result.isEmpty());
409    }
410  }
411
412  @Test
413  public void testSetAndGetUserAuths() throws Throwable {
414    final String user = "user1";
415    PrivilegedExceptionAction<Void> action = new PrivilegedExceptionAction<Void>() {
416      @Override
417      public Void run() throws Exception {
418        String[] auths = { SECRET, CONFIDENTIAL };
419        try (Connection conn = ConnectionFactory.createConnection(conf)) {
420          VisibilityClient.setAuths(conn, auths, user);
421        } catch (Throwable e) {
422        }
423        return null;
424      }
425    };
426    SUPERUSER.runAs(action);
427    try (Table ht = TEST_UTIL.getConnection().getTable(LABELS_TABLE_NAME);) {
428      Scan scan = new Scan();
429      scan.setAuthorizations(new Authorizations(VisibilityUtils.SYSTEM_LABEL));
430      ResultScanner scanner = ht.getScanner(scan);
431      Result result = null;
432      List<Result> results = new ArrayList<>();
433      while ((result = scanner.next()) != null) {
434        results.add(result);
435      }
436      List<String> auths = extractAuths(user, results);
437      assertTrue(auths.contains(SECRET));
438      assertTrue(auths.contains(CONFIDENTIAL));
439      assertEquals(2, auths.size());
440    }
441
442    action = new PrivilegedExceptionAction<Void>() {
443      @Override
444      public Void run() throws Exception {
445        GetAuthsResponse authsResponse = null;
446        try (Connection conn = ConnectionFactory.createConnection(conf)) {
447          authsResponse = VisibilityClient.getAuths(conn, user);
448        } catch (Throwable e) {
449          fail("Should not have failed");
450        }
451        List<String> authsList = new ArrayList<>(authsResponse.getAuthList().size());
452        for (ByteString authBS : authsResponse.getAuthList()) {
453          authsList.add(Bytes.toString(authBS.toByteArray()));
454        }
455        assertEquals(2, authsList.size());
456        assertTrue(authsList.contains(SECRET));
457        assertTrue(authsList.contains(CONFIDENTIAL));
458        return null;
459      }
460    };
461    SUPERUSER.runAs(action);
462
463    // Try doing setAuths once again and there should not be any duplicates
464    action = new PrivilegedExceptionAction<Void>() {
465      @Override
466      public Void run() throws Exception {
467        String[] auths1 = { SECRET, CONFIDENTIAL };
468        GetAuthsResponse authsResponse = null;
469        try (Connection conn = ConnectionFactory.createConnection(conf)) {
470          VisibilityClient.setAuths(conn, auths1, user);
471          try {
472            authsResponse = VisibilityClient.getAuths(conn, user);
473          } catch (Throwable e) {
474            fail("Should not have failed");
475          }
476        } catch (Throwable e) {
477        }
478        List<String> authsList = new ArrayList<>(authsResponse.getAuthList().size());
479        for (ByteString authBS : authsResponse.getAuthList()) {
480          authsList.add(Bytes.toString(authBS.toByteArray()));
481        }
482        assertEquals(2, authsList.size());
483        assertTrue(authsList.contains(SECRET));
484        assertTrue(authsList.contains(CONFIDENTIAL));
485        return null;
486      }
487    };
488    SUPERUSER.runAs(action);
489  }
490
491  protected List<String> extractAuths(String user, List<Result> results) {
492    List<String> auths = new ArrayList<>();
493    for (Result result : results) {
494      Cell labelCell = result.getColumnLatestCell(LABELS_TABLE_FAMILY, LABEL_QUALIFIER);
495      Cell userAuthCell = result.getColumnLatestCell(LABELS_TABLE_FAMILY, Bytes.toBytes(user));
496      if (userAuthCell != null) {
497        auths.add(Bytes.toString(labelCell.getValueArray(), labelCell.getValueOffset(),
498          labelCell.getValueLength()));
499      }
500    }
501    return auths;
502  }
503
504  @Test
505  public void testClearUserAuths() throws Throwable {
506    PrivilegedExceptionAction<Void> action = new PrivilegedExceptionAction<Void>() {
507      @Override
508      public Void run() throws Exception {
509        String[] auths = { SECRET, CONFIDENTIAL, PRIVATE };
510        String user = "testUser";
511        try (Connection conn = ConnectionFactory.createConnection(conf)) {
512          VisibilityClient.setAuths(conn, auths, user);
513        } catch (Throwable e) {
514          fail("Should not have failed");
515        }
516        // Removing the auths for SECRET and CONFIDENTIAL for the user.
517        // Passing a non existing auth also.
518        auths = new String[] { SECRET, PUBLIC, CONFIDENTIAL };
519        VisibilityLabelsResponse response = null;
520        try (Connection conn = ConnectionFactory.createConnection(conf)) {
521          response = VisibilityClient.clearAuths(conn, auths, user);
522        } catch (Throwable e) {
523          fail("Should not have failed");
524        }
525        List<RegionActionResult> resultList = response.getResultList();
526        assertEquals(3, resultList.size());
527        assertTrue(resultList.get(0).getException().getValue().isEmpty());
528        assertEquals("org.apache.hadoop.hbase.DoNotRetryIOException",
529          resultList.get(1).getException().getName());
530        assertTrue(Bytes.toString(resultList.get(1).getException().getValue().toByteArray())
531          .contains("org.apache.hadoop.hbase.security.visibility.InvalidLabelException: "
532            + "Label 'public' is not set for the user testUser"));
533        assertTrue(resultList.get(2).getException().getValue().isEmpty());
534        try (Connection connection = ConnectionFactory.createConnection(conf);
535          Table ht = connection.getTable(LABELS_TABLE_NAME)) {
536          ResultScanner scanner = ht.getScanner(new Scan());
537          Result result = null;
538          List<Result> results = new ArrayList<>();
539          while ((result = scanner.next()) != null) {
540            results.add(result);
541          }
542          List<String> curAuths = extractAuths(user, results);
543          assertTrue(curAuths.contains(PRIVATE));
544          assertEquals(1, curAuths.size());
545        }
546
547        GetAuthsResponse authsResponse = null;
548        try (Connection conn = ConnectionFactory.createConnection(conf)) {
549          authsResponse = VisibilityClient.getAuths(conn, user);
550        } catch (Throwable e) {
551          fail("Should not have failed");
552        }
553        List<String> authsList = new ArrayList<>(authsResponse.getAuthList().size());
554        for (ByteString authBS : authsResponse.getAuthList()) {
555          authsList.add(Bytes.toString(authBS.toByteArray()));
556        }
557        assertEquals(1, authsList.size());
558        assertTrue(authsList.contains(PRIVATE));
559        return null;
560      }
561    };
562    SUPERUSER.runAs(action);
563  }
564
565  @Test
566  public void testLabelsWithCheckAndPut() throws Throwable {
567    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
568    try (Table table = TEST_UTIL.createTable(tableName, fam)) {
569      byte[] row1 = Bytes.toBytes("row1");
570      Put put = new Put(row1);
571      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value);
572      put.setCellVisibility(new CellVisibility(SECRET + " & " + CONFIDENTIAL));
573      table.checkAndMutate(row1, fam).qualifier(qual).ifNotExists().thenPut(put);
574      byte[] row2 = Bytes.toBytes("row2");
575      put = new Put(row2);
576      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value);
577      put.setCellVisibility(new CellVisibility(SECRET));
578      table.checkAndMutate(row2, fam).qualifier(qual).ifNotExists().thenPut(put);
579
580      Scan scan = new Scan();
581      scan.setAuthorizations(new Authorizations(SECRET));
582      ResultScanner scanner = table.getScanner(scan);
583      Result result = scanner.next();
584      assertTrue(!result.isEmpty());
585      assertTrue(Bytes.equals(row2, result.getRow()));
586      result = scanner.next();
587      assertNull(result);
588    }
589  }
590
591  @Test
592  public void testLabelsWithIncrement() throws Throwable {
593    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
594    try (Table table = TEST_UTIL.createTable(tableName, fam)) {
595      byte[] row1 = Bytes.toBytes("row1");
596      byte[] val = Bytes.toBytes(1L);
597      Put put = new Put(row1);
598      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, val);
599      put.setCellVisibility(new CellVisibility(SECRET + " & " + CONFIDENTIAL));
600      table.put(put);
601      Get get = new Get(row1);
602      get.setAuthorizations(new Authorizations(SECRET));
603      Result result = table.get(get);
604      assertTrue(result.isEmpty());
605      table.incrementColumnValue(row1, fam, qual, 2L);
606      result = table.get(get);
607      assertTrue(result.isEmpty());
608      Increment increment = new Increment(row1);
609      increment.addColumn(fam, qual, 2L);
610      increment.setCellVisibility(new CellVisibility(SECRET));
611      table.increment(increment);
612      result = table.get(get);
613      assertTrue(!result.isEmpty());
614    }
615  }
616
617  @Test
618  public void testLabelsWithAppend() throws Throwable {
619    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
620    try (Table table = TEST_UTIL.createTable(tableName, fam);) {
621      byte[] row1 = Bytes.toBytes("row1");
622      byte[] val = Bytes.toBytes("a");
623      Put put = new Put(row1);
624      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, val);
625      put.setCellVisibility(new CellVisibility(SECRET + " & " + CONFIDENTIAL));
626      table.put(put);
627      Get get = new Get(row1);
628      get.setAuthorizations(new Authorizations(SECRET));
629      Result result = table.get(get);
630      assertTrue(result.isEmpty());
631      Append append = new Append(row1);
632      append.addColumn(fam, qual, Bytes.toBytes("b"));
633      table.append(append);
634      result = table.get(get);
635      assertTrue(result.isEmpty());
636      append = new Append(row1);
637      append.addColumn(fam, qual, Bytes.toBytes("c"));
638      append.setCellVisibility(new CellVisibility(SECRET));
639      table.append(append);
640      result = table.get(get);
641      assertTrue(!result.isEmpty());
642    }
643  }
644
645  @Test
646  public void testUserShouldNotDoDDLOpOnLabelsTable() throws Exception {
647    Admin admin = TEST_UTIL.getAdmin();
648    try {
649      admin.disableTable(LABELS_TABLE_NAME);
650      fail("Lables table should not get disabled by user.");
651    } catch (Exception e) {
652    }
653    try {
654      admin.deleteTable(LABELS_TABLE_NAME);
655      fail("Lables table should not get disabled by user.");
656    } catch (Exception e) {
657    }
658    try {
659      HColumnDescriptor hcd = new HColumnDescriptor("testFamily");
660      admin.addColumnFamily(LABELS_TABLE_NAME, hcd);
661      fail("Lables table should not get altered by user.");
662    } catch (Exception e) {
663    }
664    try {
665      admin.deleteColumnFamily(LABELS_TABLE_NAME, VisibilityConstants.LABELS_TABLE_FAMILY);
666      fail("Lables table should not get altered by user.");
667    } catch (Exception e) {
668    }
669    try {
670      HColumnDescriptor hcd = new HColumnDescriptor(VisibilityConstants.LABELS_TABLE_FAMILY);
671      hcd.setBloomFilterType(BloomType.ROWCOL);
672      admin.modifyColumnFamily(LABELS_TABLE_NAME, hcd);
673      fail("Lables table should not get altered by user.");
674    } catch (Exception e) {
675    }
676    try {
677      HTableDescriptor htd = new HTableDescriptor(LABELS_TABLE_NAME);
678      htd.addFamily(new HColumnDescriptor("f1"));
679      htd.addFamily(new HColumnDescriptor("f2"));
680      admin.modifyTable(LABELS_TABLE_NAME, htd);
681      fail("Lables table should not get altered by user.");
682    } catch (Exception e) {
683    }
684  }
685
686  @Test
687  public void testMultipleVersions() throws Exception {
688    final byte[] r1 = Bytes.toBytes("row1");
689    final byte[] r2 = Bytes.toBytes("row2");
690    final byte[] v1 = Bytes.toBytes("100");
691    final byte[] v2 = Bytes.toBytes("101");
692    final byte[] fam2 = Bytes.toBytes("info2");
693    final byte[] qual2 = Bytes.toBytes("qual2");
694    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
695    HTableDescriptor desc = new HTableDescriptor(tableName);
696    HColumnDescriptor col = new HColumnDescriptor(fam);// Default max versions is 1.
697    desc.addFamily(col);
698    col = new HColumnDescriptor(fam2);
699    col.setMaxVersions(5);
700    desc.addFamily(col);
701    TEST_UTIL.getAdmin().createTable(desc);
702    try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
703      Put put = new Put(r1);
704      put.addColumn(fam, qual, 3L, v1);
705      put.addColumn(fam, qual2, 3L, v1);
706      put.addColumn(fam2, qual, 3L, v1);
707      put.addColumn(fam2, qual2, 3L, v1);
708      put.setCellVisibility(new CellVisibility(SECRET));
709      table.put(put);
710      put = new Put(r1);
711      put.addColumn(fam, qual, 4L, v2);
712      put.addColumn(fam, qual2, 4L, v2);
713      put.addColumn(fam2, qual, 4L, v2);
714      put.addColumn(fam2, qual2, 4L, v2);
715      put.setCellVisibility(new CellVisibility(PRIVATE));
716      table.put(put);
717
718      put = new Put(r2);
719      put.addColumn(fam, qual, 3L, v1);
720      put.addColumn(fam, qual2, 3L, v1);
721      put.addColumn(fam2, qual, 3L, v1);
722      put.addColumn(fam2, qual2, 3L, v1);
723      put.setCellVisibility(new CellVisibility(SECRET));
724      table.put(put);
725      put = new Put(r2);
726      put.addColumn(fam, qual, 4L, v2);
727      put.addColumn(fam, qual2, 4L, v2);
728      put.addColumn(fam2, qual, 4L, v2);
729      put.addColumn(fam2, qual2, 4L, v2);
730      put.setCellVisibility(new CellVisibility(SECRET));
731      table.put(put);
732
733      Scan s = new Scan();
734      s.setMaxVersions(1);
735      s.setAuthorizations(new Authorizations(SECRET));
736      ResultScanner scanner = table.getScanner(s);
737      Result result = scanner.next();
738      assertTrue(Bytes.equals(r1, result.getRow()));
739      // for cf 'fam' max versions in HCD is 1. So the old version cells, which are having matching
740      // CellVisibility with Authorizations, should not get considered in the label evaluation at
741      // all.
742      assertNull(result.getColumnLatestCell(fam, qual));
743      assertNull(result.getColumnLatestCell(fam, qual2));
744      // for cf 'fam2' max versions in HCD is > 1. So we can consider the old version cells, which
745      // are having matching CellVisibility with Authorizations, in the label evaluation. It can
746      // just skip those recent versions for which visibility is not there as per the new version's
747      // CellVisibility. The old versions which are having visibility can be send back
748      Cell cell = result.getColumnLatestCell(fam2, qual);
749      assertNotNull(cell);
750      assertTrue(Bytes.equals(v1, 0, v1.length, cell.getValueArray(), cell.getValueOffset(),
751        cell.getValueLength()));
752      cell = result.getColumnLatestCell(fam2, qual2);
753      assertNotNull(cell);
754      assertTrue(Bytes.equals(v1, 0, v1.length, cell.getValueArray(), cell.getValueOffset(),
755        cell.getValueLength()));
756
757      result = scanner.next();
758      assertTrue(Bytes.equals(r2, result.getRow()));
759      cell = result.getColumnLatestCell(fam, qual);
760      assertNotNull(cell);
761      assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
762        cell.getValueLength()));
763      cell = result.getColumnLatestCell(fam, qual2);
764      assertNotNull(cell);
765      assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
766        cell.getValueLength()));
767      cell = result.getColumnLatestCell(fam2, qual);
768      assertNotNull(cell);
769      assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
770        cell.getValueLength()));
771      cell = result.getColumnLatestCell(fam2, qual2);
772      assertNotNull(cell);
773      assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
774        cell.getValueLength()));
775    }
776  }
777
778  @Test
779  public void testMutateRow() throws Exception {
780    final byte[] qual2 = Bytes.toBytes("qual2");
781    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
782    HTableDescriptor desc = new HTableDescriptor(tableName);
783    HColumnDescriptor col = new HColumnDescriptor(fam);
784    desc.addFamily(col);
785    TEST_UTIL.getAdmin().createTable(desc);
786    try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
787      Put p1 = new Put(row1);
788      p1.addColumn(fam, qual, value);
789      p1.setCellVisibility(new CellVisibility(CONFIDENTIAL));
790
791      Put p2 = new Put(row1);
792      p2.addColumn(fam, qual2, value);
793      p2.setCellVisibility(new CellVisibility(SECRET));
794
795      RowMutations rm = new RowMutations(row1);
796      rm.add(p1);
797      rm.add(p2);
798
799      table.mutateRow(rm);
800
801      Get get = new Get(row1);
802      get.setAuthorizations(new Authorizations(CONFIDENTIAL));
803      Result result = table.get(get);
804      assertTrue(result.containsColumn(fam, qual));
805      assertFalse(result.containsColumn(fam, qual2));
806
807      get.setAuthorizations(new Authorizations(SECRET));
808      result = table.get(get);
809      assertFalse(result.containsColumn(fam, qual));
810      assertTrue(result.containsColumn(fam, qual2));
811    }
812  }
813
814  @Test
815  public void testFlushedFileWithVisibilityTags() throws Exception {
816    final byte[] qual2 = Bytes.toBytes("qual2");
817    TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
818    HTableDescriptor desc = new HTableDescriptor(tableName);
819    HColumnDescriptor col = new HColumnDescriptor(fam);
820    desc.addFamily(col);
821    TEST_UTIL.getAdmin().createTable(desc);
822    try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
823      Put p1 = new Put(row1);
824      p1.addColumn(fam, qual, value);
825      p1.setCellVisibility(new CellVisibility(CONFIDENTIAL));
826
827      Put p2 = new Put(row1);
828      p2.addColumn(fam, qual2, value);
829      p2.setCellVisibility(new CellVisibility(SECRET));
830
831      RowMutations rm = new RowMutations(row1);
832      rm.add(p1);
833      rm.add(p2);
834
835      table.mutateRow(rm);
836    }
837    TEST_UTIL.getAdmin().flush(tableName);
838    List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(tableName);
839    HStore store = regions.get(0).getStore(fam);
840    Collection<HStoreFile> storefiles = store.getStorefiles();
841    assertTrue(storefiles.size() > 0);
842    for (HStoreFile storeFile : storefiles) {
843      assertTrue(storeFile.getReader().getHFileReader().getFileContext().isIncludesTags());
844    }
845  }
846
847  static Table createTableAndWriteDataWithLabels(TableName tableName, String... labelExps)
848    throws Exception {
849    List<Put> puts = new ArrayList<>(labelExps.length);
850    for (int i = 0; i < labelExps.length; i++) {
851      Put put = new Put(Bytes.toBytes("row" + (i + 1)));
852      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value);
853      put.setCellVisibility(new CellVisibility(labelExps[i]));
854      puts.add(put);
855    }
856    Table table = TEST_UTIL.createTable(tableName, fam);
857    table.put(puts);
858    return table;
859  }
860
861  public static void addLabels() throws Exception {
862    PrivilegedExceptionAction<VisibilityLabelsResponse> action =
863      new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
864        @Override
865        public VisibilityLabelsResponse run() throws Exception {
866          String[] labels = { SECRET, TOPSECRET, CONFIDENTIAL, PUBLIC, PRIVATE, COPYRIGHT, ACCENT,
867            UNICODE_VIS_TAG, UC1, UC2 };
868          try (Connection conn = ConnectionFactory.createConnection(conf)) {
869            VisibilityClient.addLabels(conn, labels);
870          } catch (Throwable t) {
871            throw new IOException(t);
872          }
873          return null;
874        }
875      };
876    SUPERUSER.runAs(action);
877  }
878}