001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.security.visibility;
019
020import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME;
021import static org.apache.hadoop.hbase.security.visibility.VisibilityUtils.SYSTEM_LABEL;
022import static org.junit.jupiter.api.Assertions.assertEquals;
023import static org.junit.jupiter.api.Assertions.assertFalse;
024import static org.junit.jupiter.api.Assertions.assertThrows;
025import static org.junit.jupiter.api.Assertions.assertTrue;
026
027import java.io.IOException;
028import java.security.PrivilegedExceptionAction;
029import java.util.List;
030import java.util.concurrent.atomic.AtomicBoolean;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.TableNameTestExtension;
033import org.apache.hadoop.hbase.client.Connection;
034import org.apache.hadoop.hbase.client.ConnectionFactory;
035import org.apache.hadoop.hbase.client.Result;
036import org.apache.hadoop.hbase.client.ResultScanner;
037import org.apache.hadoop.hbase.client.Scan;
038import org.apache.hadoop.hbase.client.Table;
039import org.apache.hadoop.hbase.security.User;
040import org.apache.hadoop.hbase.testclassification.MediumTests;
041import org.apache.hadoop.hbase.testclassification.SecurityTests;
042import org.apache.hadoop.hbase.util.Bytes;
043import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
044import org.apache.hadoop.hbase.util.Threads;
045import org.junit.jupiter.api.Assertions;
046import org.junit.jupiter.api.BeforeAll;
047import org.junit.jupiter.api.Tag;
048import org.junit.jupiter.api.Test;
049import org.junit.jupiter.api.TestInfo;
050import org.slf4j.Logger;
051import org.slf4j.LoggerFactory;
052
053import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
054
055import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
056import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.RegionActionResult;
057import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.NameBytesPair;
058import org.apache.hadoop.hbase.shaded.protobuf.generated.VisibilityLabelsProtos.ListLabelsResponse;
059import org.apache.hadoop.hbase.shaded.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
060
061@Tag(SecurityTests.TAG)
062@Tag(MediumTests.TAG)
063public class TestVisibilityLabelsWithDefaultVisLabelService extends VisibilityLabelsTestBase {
064
065  private static final Logger LOG =
066    LoggerFactory.getLogger(TestVisibilityLabelsWithDefaultVisLabelService.class);
067
068  @BeforeAll
069  public static void setupBeforeClass() throws Exception {
070    // setup configuration
071    conf = TEST_UTIL.getConfiguration();
072    VisibilityTestUtil.enableVisiblityLabels(conf);
073    conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS, SimpleScanLabelGenerator.class,
074      ScanLabelGenerator.class);
075    conf.set("hbase.superuser", "admin");
076    TEST_UTIL.startMiniCluster(2);
077    SUPERUSER = User.createUserForTesting(conf, "admin", new String[] { "supergroup" });
078    USER1 = User.createUserForTesting(conf, "user1", new String[] {});
079
080    // Wait for the labels table to become available
081    TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000);
082    addLabels();
083  }
084
085  @Test
086  public void testVisibilityLabelsInPutsThatDoesNotMatchAnyDefinedLabels() throws Exception {
087    TableName tableName = name.getTableName();
088    assertThrows(Exception.class,
089      () -> createTableAndWriteDataWithLabels(tableName, "SAMPLE_LABEL", "TEST"),
090      "Should have failed with failed sanity check exception");
091  }
092
093  @Test
094  public void testAddLabels() throws Throwable {
095    PrivilegedExceptionAction<VisibilityLabelsResponse> action =
096      new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
097        @Override
098        public VisibilityLabelsResponse run() throws Exception {
099          String[] labels = { "L1", SECRET, "L2", "invalid~", "L3" };
100          VisibilityLabelsResponse response = null;
101          try (Connection conn = ConnectionFactory.createConnection(conf)) {
102            response = VisibilityClient.addLabels(conn, labels);
103          } catch (Throwable e) {
104            throw new IOException(e);
105          }
106          List<RegionActionResult> resultList = response.getResultList();
107          assertEquals(5, resultList.size());
108          assertTrue(resultList.get(0).getException().getValue().isEmpty());
109          assertEquals("org.apache.hadoop.hbase.DoNotRetryIOException",
110            resultList.get(1).getException().getName());
111          assertTrue(Bytes.toString(resultList.get(1).getException().getValue().toByteArray())
112            .contains("org.apache.hadoop.hbase.security.visibility.LabelAlreadyExistsException: "
113              + "Label 'secret' already exists"));
114          assertTrue(resultList.get(2).getException().getValue().isEmpty());
115          assertTrue(resultList.get(3).getException().getValue().isEmpty());
116          assertTrue(resultList.get(4).getException().getValue().isEmpty());
117          return null;
118        }
119      };
120    SUPERUSER.runAs(action);
121  }
122
123  @Test
124  public void testAddVisibilityLabelsOnRSRestart() throws Exception {
125    List<RegionServerThread> regionServerThreads =
126      TEST_UTIL.getHBaseCluster().getRegionServerThreads();
127    for (RegionServerThread rsThread : regionServerThreads) {
128      rsThread.getRegionServer().abort("Aborting ");
129    }
130    // Start one new RS
131    RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer();
132    waitForLabelsRegionAvailability(rs.getRegionServer());
133    final AtomicBoolean vcInitialized = new AtomicBoolean(true);
134    do {
135      PrivilegedExceptionAction<VisibilityLabelsResponse> action =
136        new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
137          @Override
138          public VisibilityLabelsResponse run() throws Exception {
139            String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, "ABC", "XYZ" };
140            try (Connection conn = ConnectionFactory.createConnection(conf)) {
141              VisibilityLabelsResponse resp = VisibilityClient.addLabels(conn, labels);
142              List<RegionActionResult> results = resp.getResultList();
143              if (results.get(0).hasException()) {
144                NameBytesPair pair = results.get(0).getException();
145                Throwable t = ProtobufUtil.toException(pair);
146                LOG.debug("Got exception writing labels", t);
147                if (t instanceof VisibilityControllerNotReadyException) {
148                  vcInitialized.set(false);
149                  LOG.warn("VisibilityController was not yet initialized");
150                  Threads.sleep(10);
151                } else {
152                  vcInitialized.set(true);
153                }
154              } else LOG.debug("new labels added: " + resp);
155            } catch (Throwable t) {
156              throw new IOException(t);
157            }
158            return null;
159          }
160        };
161      SUPERUSER.runAs(action);
162    } while (!vcInitialized.get());
163    // Scan the visibility label
164    Scan s = new Scan();
165    s.setAuthorizations(new Authorizations(VisibilityUtils.SYSTEM_LABEL));
166
167    int i = 0;
168    try (Table ht = TEST_UTIL.getConnection().getTable(LABELS_TABLE_NAME);
169      ResultScanner scanner = ht.getScanner(s)) {
170      while (true) {
171        Result next = scanner.next();
172        if (next == null) {
173          break;
174        }
175        i++;
176      }
177    }
178    // One label is the "system" label.
179    Assertions.assertEquals(13, i, "The count should be 13");
180  }
181
182  @Test
183  public void testListLabels() throws Throwable {
184    PrivilegedExceptionAction<ListLabelsResponse> action =
185      new PrivilegedExceptionAction<ListLabelsResponse>() {
186        @Override
187        public ListLabelsResponse run() throws Exception {
188          ListLabelsResponse response = null;
189          try (Connection conn = ConnectionFactory.createConnection(conf)) {
190            response = VisibilityClient.listLabels(conn, null);
191          } catch (Throwable e) {
192            throw new IOException(e);
193          }
194          // The addLabels() in setup added:
195          // { SECRET, TOPSECRET, CONFIDENTIAL, PUBLIC, PRIVATE, COPYRIGHT, ACCENT,
196          // UNICODE_VIS_TAG, UC1, UC2 };
197          // The previous tests added 2 more labels: ABC, XYZ
198          // The 'system' label is excluded.
199          List<ByteString> labels = response.getLabelList();
200          assertEquals(12, labels.size());
201          assertTrue(labels.contains(ByteString.copyFrom(Bytes.toBytes(SECRET))));
202          assertTrue(labels.contains(ByteString.copyFrom(Bytes.toBytes(TOPSECRET))));
203          assertTrue(labels.contains(ByteString.copyFrom(Bytes.toBytes(CONFIDENTIAL))));
204          assertTrue(labels.contains(ByteString.copyFrom(Bytes.toBytes("ABC"))));
205          assertTrue(labels.contains(ByteString.copyFrom(Bytes.toBytes("XYZ"))));
206          assertFalse(labels.contains(ByteString.copyFrom(Bytes.toBytes(SYSTEM_LABEL))));
207          return null;
208        }
209      };
210    SUPERUSER.runAs(action);
211  }
212
213  @Test
214  public void testListLabelsWithRegEx() throws Throwable {
215    PrivilegedExceptionAction<ListLabelsResponse> action =
216      new PrivilegedExceptionAction<ListLabelsResponse>() {
217        @Override
218        public ListLabelsResponse run() throws Exception {
219          ListLabelsResponse response = null;
220          try (Connection conn = ConnectionFactory.createConnection(conf)) {
221            response = VisibilityClient.listLabels(conn, ".*secret");
222          } catch (Throwable e) {
223            throw new IOException(e);
224          }
225          // Only return the labels that end with 'secret'
226          List<ByteString> labels = response.getLabelList();
227          assertEquals(2, labels.size());
228          assertTrue(labels.contains(ByteString.copyFrom(Bytes.toBytes(SECRET))));
229          assertTrue(labels.contains(ByteString.copyFrom(Bytes.toBytes(TOPSECRET))));
230          return null;
231        }
232      };
233    SUPERUSER.runAs(action);
234  }
235
236  @Test
237  public void testVisibilityLabelsOnWALReplay(TestInfo testInfo) throws Exception {
238    final TableName tableName = TableName
239      .valueOf(TableNameTestExtension.cleanUpTestName(testInfo.getTestMethod().get().getName()));
240    try (Table table = createTableAndWriteDataWithLabels(tableName,
241      "(" + SECRET + "|" + CONFIDENTIAL + ")", PRIVATE)) {
242      List<RegionServerThread> regionServerThreads =
243        TEST_UTIL.getHBaseCluster().getRegionServerThreads();
244      for (RegionServerThread rsThread : regionServerThreads) {
245        rsThread.getRegionServer().abort("Aborting ");
246      }
247      // Start one new RS
248      RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer();
249      waitForLabelsRegionAvailability(rs.getRegionServer());
250      Scan s = new Scan();
251      s.setAuthorizations(new Authorizations(SECRET));
252      ResultScanner scanner = table.getScanner(s);
253      Result[] next = scanner.next(3);
254      assertTrue(next.length == 1);
255    }
256  }
257}