001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.security.visibility;
019
020import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME;
021import static org.junit.jupiter.api.Assertions.assertArrayEquals;
022import static org.junit.jupiter.api.Assertions.assertEquals;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024
025import java.io.IOException;
026import java.security.PrivilegedExceptionAction;
027import java.util.ArrayList;
028import java.util.List;
029import java.util.Optional;
030import java.util.concurrent.atomic.AtomicInteger;
031import org.apache.hadoop.conf.Configuration;
032import org.apache.hadoop.hbase.ArrayBackedTag;
033import org.apache.hadoop.hbase.Cell;
034import org.apache.hadoop.hbase.CellScanner;
035import org.apache.hadoop.hbase.CellUtil;
036import org.apache.hadoop.hbase.ExtendedCell;
037import org.apache.hadoop.hbase.HBaseConfiguration;
038import org.apache.hadoop.hbase.HBaseTestingUtil;
039import org.apache.hadoop.hbase.HConstants;
040import org.apache.hadoop.hbase.PrivateCellUtil;
041import org.apache.hadoop.hbase.TableName;
042import org.apache.hadoop.hbase.Tag;
043import org.apache.hadoop.hbase.TagType;
044import org.apache.hadoop.hbase.client.Admin;
045import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
046import org.apache.hadoop.hbase.client.Connection;
047import org.apache.hadoop.hbase.client.ConnectionFactory;
048import org.apache.hadoop.hbase.client.Durability;
049import org.apache.hadoop.hbase.client.Get;
050import org.apache.hadoop.hbase.client.Put;
051import org.apache.hadoop.hbase.client.Result;
052import org.apache.hadoop.hbase.client.ResultScanner;
053import org.apache.hadoop.hbase.client.Scan;
054import org.apache.hadoop.hbase.client.Table;
055import org.apache.hadoop.hbase.client.TableDescriptor;
056import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
057import org.apache.hadoop.hbase.codec.KeyValueCodecWithTags;
058import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
059import org.apache.hadoop.hbase.coprocessor.ObserverContext;
060import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
061import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
062import org.apache.hadoop.hbase.coprocessor.RegionObserver;
063import org.apache.hadoop.hbase.replication.ReplicationEndpoint;
064import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
065import org.apache.hadoop.hbase.security.User;
066import org.apache.hadoop.hbase.testclassification.MediumTests;
067import org.apache.hadoop.hbase.testclassification.SecurityTests;
068import org.apache.hadoop.hbase.util.Bytes;
069import org.apache.hadoop.hbase.wal.WAL.Entry;
070import org.apache.hadoop.hbase.wal.WALEdit;
071import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
072import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
073import org.junit.jupiter.api.Assertions;
074import org.junit.jupiter.api.BeforeEach;
075import org.junit.jupiter.api.Test;
076import org.slf4j.Logger;
077import org.slf4j.LoggerFactory;
078
079import org.apache.hadoop.hbase.shaded.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
080
081@org.junit.jupiter.api.Tag(SecurityTests.TAG)
082@org.junit.jupiter.api.Tag(MediumTests.TAG)
083public class TestVisibilityLabelsReplication {
084
085  private static final Logger LOG = LoggerFactory.getLogger(TestVisibilityLabelsReplication.class);
086  protected static final int NON_VIS_TAG_TYPE = 100;
087  protected static final String TEMP = "temp";
088  protected static Configuration conf;
089  protected static Configuration conf1;
090  protected static TableName TABLE_NAME = TableName.valueOf("TABLE_NAME");
091  protected static Admin admin;
092  public static final String TOPSECRET = "topsecret";
093  public static final String PUBLIC = "public";
094  public static final String PRIVATE = "private";
095  public static final String CONFIDENTIAL = "confidential";
096  public static final String COPYRIGHT = "\u00A9ABC";
097  public static final String ACCENT = "\u0941";
098  public static final String SECRET = "secret";
099  public static final String UNICODE_VIS_TAG =
100    COPYRIGHT + "\"" + ACCENT + "\\" + SECRET + "\"" + "\u0027&\\";
101  public static HBaseTestingUtil TEST_UTIL;
102  public static HBaseTestingUtil TEST_UTIL1;
103  public static final byte[] row1 = Bytes.toBytes("row1");
104  public static final byte[] row2 = Bytes.toBytes("row2");
105  public static final byte[] row3 = Bytes.toBytes("row3");
106  public static final byte[] row4 = Bytes.toBytes("row4");
107  public final static byte[] fam = Bytes.toBytes("info");
108  public final static byte[] qual = Bytes.toBytes("qual");
109  public final static byte[] value = Bytes.toBytes("value");
110  protected static ZKWatcher zkw1;
111  protected static ZKWatcher zkw2;
112  protected static int expected[] = { 4, 6, 4, 0, 3 };
113  private static final String NON_VISIBILITY = "non-visibility";
114  protected static String[] expectedVisString =
115    { "(\"secret\"&\"topsecret\"&\"public\")|(\"topsecret\"&\"confidential\")",
116      "(\"public\"&\"private\")|(\"topsecret\"&\"private\")|"
117        + "(\"confidential\"&\"public\")|(\"topsecret\"&\"confidential\")",
118      "(!\"topsecret\"&\"secret\")|(!\"topsecret\"&\"confidential\")", "(\"secret\"&\"" + COPYRIGHT
119        + "\\\"" + ACCENT + "\\\\" + SECRET + "\\\"" + "\u0027&\\\\" + "\")" };
120
121  public static User SUPERUSER, USER1;
122
123  protected void setUpTest() throws Exception {
124    // setup configuration
125    conf = HBaseConfiguration.create();
126    conf.setInt("hfile.format.version", 3);
127    conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
128    conf.setInt("replication.source.size.capacity", 10240);
129    conf.setLong("replication.source.sleepforretries", 100);
130    conf.setInt("hbase.regionserver.maxlogs", 10);
131    conf.setLong("hbase.master.logcleaner.ttl", 10);
132    conf.setInt("zookeeper.recovery.retry", 1);
133    conf.setInt("zookeeper.recovery.retry.intervalmill", 10);
134    conf.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
135    conf.setInt("replication.stats.thread.period.seconds", 5);
136    conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
137    setVisibilityLabelServiceImpl(conf);
138    conf.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName());
139    VisibilityTestUtil.enableVisiblityLabels(conf);
140    conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY,
141      VisibilityReplication.class.getName());
142    conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, SimpleCP.class.getName());
143    // Have to reset conf1 in case zk cluster location different
144    // than default
145    conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS, SimpleScanLabelGenerator.class,
146      ScanLabelGenerator.class);
147    conf.set("hbase.superuser", User.getCurrent().getShortName());
148    SUPERUSER = User.createUserForTesting(conf, User.getCurrent().getShortName(),
149      new String[] { "supergroup" });
150    // User.createUserForTesting(conf, User.getCurrent().getShortName(), new
151    // String[] { "supergroup" });
152    USER1 = User.createUserForTesting(conf, "user1", new String[] {});
153    TEST_UTIL = new HBaseTestingUtil(conf);
154    TEST_UTIL.startMiniZKCluster();
155    MiniZooKeeperCluster miniZK = TEST_UTIL.getZkCluster();
156    zkw1 = new ZKWatcher(conf, "cluster1", null, true);
157
158    // Base conf2 on conf1 so it gets the right zk cluster.
159    conf1 = HBaseConfiguration.create(conf);
160    conf1.setInt("hfile.format.version", 3);
161    conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
162    conf1.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
163    conf1.setBoolean("hbase.tests.use.shortcircuit.reads", false);
164    conf1.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName());
165    conf1.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
166      TestCoprocessorForTagsAtSink.class.getName());
167    // setVisibilityLabelServiceImpl(conf1);
168    USER1 = User.createUserForTesting(conf1, "user1", new String[] {});
169    TEST_UTIL1 = new HBaseTestingUtil(conf1);
170    TEST_UTIL1.setZkCluster(miniZK);
171    zkw2 = new ZKWatcher(conf1, "cluster2", null, true);
172
173    TEST_UTIL.startMiniCluster(1);
174    // Wait for the labels table to become available
175    TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000);
176    TEST_UTIL1.startMiniCluster(1);
177
178    admin = TEST_UTIL.getAdmin();
179    ReplicationPeerConfig rpc =
180      ReplicationPeerConfig.newBuilder().setClusterKey(TEST_UTIL1.getRpcConnnectionURI()).build();
181    admin.addReplicationPeer("2", rpc);
182
183    Admin hBaseAdmin = TEST_UTIL.getAdmin();
184    TableDescriptor tableDescriptor =
185      TableDescriptorBuilder.newBuilder(TABLE_NAME).setColumnFamily(ColumnFamilyDescriptorBuilder
186        .newBuilder(fam).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()).build();
187    try {
188      hBaseAdmin.createTable(tableDescriptor);
189    } finally {
190      if (hBaseAdmin != null) {
191        hBaseAdmin.close();
192      }
193    }
194    Admin hBaseAdmin1 = TEST_UTIL1.getAdmin();
195    try {
196      hBaseAdmin1.createTable(tableDescriptor);
197    } finally {
198      if (hBaseAdmin1 != null) {
199        hBaseAdmin1.close();
200      }
201    }
202    addLabels();
203    setAuths(conf);
204    setAuths(conf1);
205  }
206
207  @BeforeEach
208  public void setUp() throws Exception {
209    setUpTest();
210  }
211
212  protected static void setVisibilityLabelServiceImpl(Configuration conf) {
213    conf.setClass(VisibilityLabelServiceManager.VISIBILITY_LABEL_SERVICE_CLASS,
214      DefaultVisibilityLabelServiceImpl.class, VisibilityLabelService.class);
215  }
216
217  @Test
218  public void testVisibilityReplication() throws Exception {
219    int retry = 0;
220    try (Table table = writeData(TABLE_NAME,
221      "(" + SECRET + "&" + PUBLIC + ")" + "|(" + CONFIDENTIAL + ")&(" + TOPSECRET + ")",
222      "(" + PRIVATE + "|" + CONFIDENTIAL + ")&(" + PUBLIC + "|" + TOPSECRET + ")",
223      "(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET,
224      CellVisibility.quote(UNICODE_VIS_TAG) + "&" + SECRET)) {
225      Scan s = new Scan();
226      s.setAuthorizations(
227        new Authorizations(SECRET, CONFIDENTIAL, PRIVATE, TOPSECRET, UNICODE_VIS_TAG));
228      ResultScanner scanner = table.getScanner(s);
229      Result[] next = scanner.next(4);
230
231      assertTrue(next.length == 4);
232      CellScanner cellScanner = next[0].cellScanner();
233      cellScanner.advance();
234      Cell current = cellScanner.current();
235      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
236        row1, 0, row1.length));
237      cellScanner = next[1].cellScanner();
238      cellScanner.advance();
239      current = cellScanner.current();
240      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
241        row2, 0, row2.length));
242      cellScanner = next[2].cellScanner();
243      cellScanner.advance();
244      current = cellScanner.current();
245      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
246        row3, 0, row3.length));
247      cellScanner = next[3].cellScanner();
248      cellScanner.advance();
249      current = cellScanner.current();
250      assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
251        row4, 0, row4.length));
252      try (Table table2 = TEST_UTIL1.getConnection().getTable(TABLE_NAME)) {
253        s = new Scan();
254        // Ensure both rows are replicated
255        scanner = table2.getScanner(s);
256        next = scanner.next(4);
257        while (next.length == 0 && retry <= 10) {
258          scanner = table2.getScanner(s);
259          next = scanner.next(4);
260          Thread.sleep(2000);
261          retry++;
262        }
263        assertTrue(next.length == 4);
264        verifyGet(row1, expectedVisString[0], expected[0], false, TOPSECRET, CONFIDENTIAL);
265        TestCoprocessorForTagsAtSink.tags.clear();
266        verifyGet(row2, expectedVisString[1], expected[1], false, CONFIDENTIAL, PUBLIC);
267        TestCoprocessorForTagsAtSink.tags.clear();
268        verifyGet(row3, expectedVisString[2], expected[2], false, PRIVATE, SECRET);
269        verifyGet(row3, "", expected[3], true, TOPSECRET, SECRET);
270        verifyGet(row4, expectedVisString[3], expected[4], false, UNICODE_VIS_TAG, SECRET);
271      }
272    }
273  }
274
275  protected static void doAssert(byte[] row, String visTag) throws Exception {
276    if (VisibilityReplicationEndPointForTest.lastEntries == null) {
277      return; // first call
278    }
279    Assertions.assertEquals(1, VisibilityReplicationEndPointForTest.lastEntries.size());
280    List<Cell> cells = VisibilityReplicationEndPointForTest.lastEntries.get(0).getEdit().getCells();
281    Assertions.assertEquals(4, cells.size());
282    boolean tagFound = false;
283    for (Cell cell : cells) {
284      if (
285        (Bytes.equals(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), row, 0,
286          row.length))
287      ) {
288        List<Tag> tags = PrivateCellUtil.getTags((ExtendedCell) cell);
289        for (Tag tag : tags) {
290          if (tag.getType() == TagType.STRING_VIS_TAG_TYPE) {
291            assertEquals(visTag, Tag.getValueAsString(tag));
292            tagFound = true;
293            break;
294          }
295        }
296      }
297    }
298    assertTrue(tagFound);
299  }
300
301  protected void verifyGet(final byte[] row, final String visString, final int expected,
302    final boolean nullExpected, final String... auths) throws IOException, InterruptedException {
303    PrivilegedExceptionAction<Void> scanAction = new PrivilegedExceptionAction<Void>() {
304      @Override
305      public Void run() throws Exception {
306        try (Connection connection = ConnectionFactory.createConnection(conf1);
307          Table table2 = connection.getTable(TABLE_NAME)) {
308          CellScanner cellScanner;
309          Cell current;
310          Get get = new Get(row);
311          get.setAuthorizations(new Authorizations(auths));
312          Result result = table2.get(get);
313          cellScanner = result.cellScanner();
314          boolean advance = cellScanner.advance();
315          if (nullExpected) {
316            assertTrue(!advance);
317            return null;
318          }
319          current = cellScanner.current();
320          assertArrayEquals(CellUtil.cloneRow(current), row);
321          for (Tag tag : TestCoprocessorForTagsAtSink.tags) {
322            LOG.info("The tag type is " + tag.getType());
323          }
324          assertEquals(expected, TestCoprocessorForTagsAtSink.tags.size());
325          Tag tag = TestCoprocessorForTagsAtSink.tags.get(1);
326          if (tag.getType() != NON_VIS_TAG_TYPE) {
327            assertEquals(TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE, tag.getType());
328          }
329          tag = TestCoprocessorForTagsAtSink.tags.get(0);
330          boolean foundNonVisTag = false;
331          for (Tag t : TestCoprocessorForTagsAtSink.tags) {
332            if (t.getType() == NON_VIS_TAG_TYPE) {
333              assertEquals(TEMP, Tag.getValueAsString(t));
334              foundNonVisTag = true;
335              break;
336            }
337          }
338          doAssert(row, visString);
339          assertTrue(foundNonVisTag);
340          return null;
341        }
342      }
343    };
344    USER1.runAs(scanAction);
345  }
346
347  public static void addLabels() throws Exception {
348    PrivilegedExceptionAction<VisibilityLabelsResponse> action =
349      new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
350        @Override
351        public VisibilityLabelsResponse run() throws Exception {
352          String[] labels = { SECRET, TOPSECRET, CONFIDENTIAL, PUBLIC, PRIVATE, UNICODE_VIS_TAG };
353          try (Connection conn = ConnectionFactory.createConnection(conf)) {
354            VisibilityClient.addLabels(conn, labels);
355          } catch (Throwable t) {
356            throw new IOException(t);
357          }
358          return null;
359        }
360      };
361    SUPERUSER.runAs(action);
362  }
363
364  public static void setAuths(final Configuration conf) throws Exception {
365    PrivilegedExceptionAction<VisibilityLabelsResponse> action =
366      new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
367        @Override
368        public VisibilityLabelsResponse run() throws Exception {
369          try (Connection conn = ConnectionFactory.createConnection(conf)) {
370            return VisibilityClient.setAuths(conn,
371              new String[] { SECRET, CONFIDENTIAL, PRIVATE, TOPSECRET, UNICODE_VIS_TAG }, "user1");
372          } catch (Throwable e) {
373            throw new Exception(e);
374          }
375        }
376      };
377    SUPERUSER.runAs(action);
378  }
379
380  static Table writeData(TableName tableName, String... labelExps) throws Exception {
381    Table table = TEST_UTIL.getConnection().getTable(TABLE_NAME);
382    int i = 1;
383    List<Put> puts = new ArrayList<>(labelExps.length);
384    for (String labelExp : labelExps) {
385      Put put = new Put(Bytes.toBytes("row" + i));
386      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value);
387      put.setCellVisibility(new CellVisibility(labelExp));
388      put.setAttribute(NON_VISIBILITY, Bytes.toBytes(TEMP));
389      puts.add(put);
390      i++;
391    }
392    table.put(puts);
393    return table;
394  }
395
396  // A simple BaseRegionbserver impl that allows to add a non-visibility tag from the
397  // attributes of the Put mutation. The existing cells in the put mutation is overwritten
398  // with a new cell that has the visibility tags and the non visibility tag
399  public static class SimpleCP implements RegionCoprocessor, RegionObserver {
400    @Override
401    public Optional<RegionObserver> getRegionObserver() {
402      return Optional.of(this);
403    }
404
405    @Override
406    public void prePut(ObserverContext<? extends RegionCoprocessorEnvironment> e, Put m,
407      WALEdit edit, Durability durability) throws IOException {
408      byte[] attribute = m.getAttribute(NON_VISIBILITY);
409      byte[] cf = null;
410      List<Cell> updatedCells = new ArrayList<>();
411      if (attribute != null) {
412        for (List<? extends Cell> edits : m.getFamilyCellMap().values()) {
413          for (Cell cell : edits) {
414            if (cf == null) {
415              cf = CellUtil.cloneFamily(cell);
416            }
417            Tag tag = new ArrayBackedTag((byte) NON_VIS_TAG_TYPE, attribute);
418            List<Tag> tagList =
419              new ArrayList<>(PrivateCellUtil.getTags((ExtendedCell) cell).size() + 1);
420            tagList.add(tag);
421            tagList.addAll(PrivateCellUtil.getTags((ExtendedCell) cell));
422            Cell newcell = PrivateCellUtil.createCell((ExtendedCell) cell, tagList);
423            ((List<Cell>) updatedCells).add(newcell);
424          }
425        }
426        m.getFamilyCellMap().remove(cf);
427        // Update the family map
428        m.getFamilyCellMap().put(cf, updatedCells);
429      }
430    }
431  }
432
433  public static class TestCoprocessorForTagsAtSink implements RegionCoprocessor, RegionObserver {
434    public static List<Tag> tags = null;
435
436    @Override
437    public Optional<RegionObserver> getRegionObserver() {
438      return Optional.of(this);
439    }
440
441    @Override
442    public void postGetOp(ObserverContext<? extends RegionCoprocessorEnvironment> e, Get get,
443      List<Cell> results) throws IOException {
444      if (results.size() > 0) {
445        // Check tag presence in the 1st cell in 1st Result
446        if (!results.isEmpty()) {
447          Cell cell = results.get(0);
448          tags = PrivateCellUtil.getTags((ExtendedCell) cell);
449        }
450      }
451    }
452  }
453
454  /**
455   * An extn of VisibilityReplicationEndpoint to verify the tags that are replicated
456   */
457  public static class VisibilityReplicationEndPointForTest extends VisibilityReplicationEndpoint {
458    static AtomicInteger replicateCount = new AtomicInteger();
459    static volatile List<Entry> lastEntries = null;
460
461    public VisibilityReplicationEndPointForTest(ReplicationEndpoint endpoint,
462      VisibilityLabelService visibilityLabelsService) {
463      super(endpoint, visibilityLabelsService);
464    }
465
466    @Override
467    public boolean replicate(ReplicateContext replicateContext) {
468      boolean ret = super.replicate(replicateContext);
469      lastEntries = replicateContext.getEntries();
470      replicateCount.incrementAndGet();
471      return ret;
472    }
473  }
474}