001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.security.visibility; 019 020import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME; 021import static org.junit.Assert.assertArrayEquals; 022import static org.junit.Assert.assertEquals; 023import static org.junit.Assert.assertTrue; 024 025import java.io.IOException; 026import java.security.PrivilegedExceptionAction; 027import java.util.ArrayList; 028import java.util.List; 029import java.util.Optional; 030import java.util.concurrent.atomic.AtomicInteger; 031import org.apache.hadoop.conf.Configuration; 032import org.apache.hadoop.hbase.ArrayBackedTag; 033import org.apache.hadoop.hbase.Cell; 034import org.apache.hadoop.hbase.CellScanner; 035import org.apache.hadoop.hbase.CellUtil; 036import org.apache.hadoop.hbase.HBaseClassTestRule; 037import org.apache.hadoop.hbase.HBaseConfiguration; 038import org.apache.hadoop.hbase.HBaseTestingUtil; 039import org.apache.hadoop.hbase.HConstants; 040import org.apache.hadoop.hbase.KeyValue; 041import org.apache.hadoop.hbase.KeyValueUtil; 042import org.apache.hadoop.hbase.PrivateCellUtil; 043import org.apache.hadoop.hbase.TableName; 044import org.apache.hadoop.hbase.Tag; 045import org.apache.hadoop.hbase.TagType; 046import org.apache.hadoop.hbase.client.Admin; 047import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 048import org.apache.hadoop.hbase.client.Connection; 049import org.apache.hadoop.hbase.client.ConnectionFactory; 050import org.apache.hadoop.hbase.client.Durability; 051import org.apache.hadoop.hbase.client.Get; 052import org.apache.hadoop.hbase.client.Put; 053import org.apache.hadoop.hbase.client.Result; 054import org.apache.hadoop.hbase.client.ResultScanner; 055import org.apache.hadoop.hbase.client.Scan; 056import org.apache.hadoop.hbase.client.Table; 057import org.apache.hadoop.hbase.client.TableDescriptor; 058import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 059import org.apache.hadoop.hbase.codec.KeyValueCodecWithTags; 060import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; 061import org.apache.hadoop.hbase.coprocessor.ObserverContext; 062import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor; 063import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; 064import org.apache.hadoop.hbase.coprocessor.RegionObserver; 065import org.apache.hadoop.hbase.replication.ReplicationEndpoint; 066import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; 067import org.apache.hadoop.hbase.security.User; 068import org.apache.hadoop.hbase.testclassification.MediumTests; 069import org.apache.hadoop.hbase.testclassification.SecurityTests; 070import org.apache.hadoop.hbase.util.Bytes; 071import org.apache.hadoop.hbase.wal.WAL.Entry; 072import org.apache.hadoop.hbase.wal.WALEdit; 073import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; 074import org.apache.hadoop.hbase.zookeeper.ZKWatcher; 075import org.junit.Assert; 076import org.junit.Before; 077import org.junit.ClassRule; 078import org.junit.Rule; 079import org.junit.Test; 080import org.junit.experimental.categories.Category; 081import org.junit.rules.TestName; 082import org.slf4j.Logger; 083import org.slf4j.LoggerFactory; 084 085import org.apache.hadoop.hbase.shaded.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse; 086 087@Category({ SecurityTests.class, MediumTests.class }) 088public class TestVisibilityLabelsReplication { 089 090 @ClassRule 091 public static final HBaseClassTestRule CLASS_RULE = 092 HBaseClassTestRule.forClass(TestVisibilityLabelsReplication.class); 093 094 private static final Logger LOG = LoggerFactory.getLogger(TestVisibilityLabelsReplication.class); 095 protected static final int NON_VIS_TAG_TYPE = 100; 096 protected static final String TEMP = "temp"; 097 protected static Configuration conf; 098 protected static Configuration conf1; 099 protected static TableName TABLE_NAME = TableName.valueOf("TABLE_NAME"); 100 protected static Admin admin; 101 public static final String TOPSECRET = "topsecret"; 102 public static final String PUBLIC = "public"; 103 public static final String PRIVATE = "private"; 104 public static final String CONFIDENTIAL = "confidential"; 105 public static final String COPYRIGHT = "\u00A9ABC"; 106 public static final String ACCENT = "\u0941"; 107 public static final String SECRET = "secret"; 108 public static final String UNICODE_VIS_TAG = 109 COPYRIGHT + "\"" + ACCENT + "\\" + SECRET + "\"" + "\u0027&\\"; 110 public static HBaseTestingUtil TEST_UTIL; 111 public static HBaseTestingUtil TEST_UTIL1; 112 public static final byte[] row1 = Bytes.toBytes("row1"); 113 public static final byte[] row2 = Bytes.toBytes("row2"); 114 public static final byte[] row3 = Bytes.toBytes("row3"); 115 public static final byte[] row4 = Bytes.toBytes("row4"); 116 public final static byte[] fam = Bytes.toBytes("info"); 117 public final static byte[] qual = Bytes.toBytes("qual"); 118 public final static byte[] value = Bytes.toBytes("value"); 119 protected static ZKWatcher zkw1; 120 protected static ZKWatcher zkw2; 121 protected static int expected[] = { 4, 6, 4, 0, 3 }; 122 private static final String NON_VISIBILITY = "non-visibility"; 123 protected static String[] expectedVisString = 124 { "(\"secret\"&\"topsecret\"&\"public\")|(\"topsecret\"&\"confidential\")", 125 "(\"public\"&\"private\")|(\"topsecret\"&\"private\")|" 126 + "(\"confidential\"&\"public\")|(\"topsecret\"&\"confidential\")", 127 "(!\"topsecret\"&\"secret\")|(!\"topsecret\"&\"confidential\")", "(\"secret\"&\"" + COPYRIGHT 128 + "\\\"" + ACCENT + "\\\\" + SECRET + "\\\"" + "\u0027&\\\\" + "\")" }; 129 130 @Rule 131 public final TestName TEST_NAME = new TestName(); 132 public static User SUPERUSER, USER1; 133 134 @Before 135 public void setup() throws Exception { 136 // setup configuration 137 conf = HBaseConfiguration.create(); 138 conf.setInt("hfile.format.version", 3); 139 conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1"); 140 conf.setInt("replication.source.size.capacity", 10240); 141 conf.setLong("replication.source.sleepforretries", 100); 142 conf.setInt("hbase.regionserver.maxlogs", 10); 143 conf.setLong("hbase.master.logcleaner.ttl", 10); 144 conf.setInt("zookeeper.recovery.retry", 1); 145 conf.setInt("zookeeper.recovery.retry.intervalmill", 10); 146 conf.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100); 147 conf.setInt("replication.stats.thread.period.seconds", 5); 148 conf.setBoolean("hbase.tests.use.shortcircuit.reads", false); 149 setVisibilityLabelServiceImpl(conf); 150 conf.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName()); 151 VisibilityTestUtil.enableVisiblityLabels(conf); 152 conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, 153 VisibilityReplication.class.getName()); 154 conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, SimpleCP.class.getName()); 155 // Have to reset conf1 in case zk cluster location different 156 // than default 157 conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS, SimpleScanLabelGenerator.class, 158 ScanLabelGenerator.class); 159 conf.set("hbase.superuser", User.getCurrent().getShortName()); 160 SUPERUSER = User.createUserForTesting(conf, User.getCurrent().getShortName(), 161 new String[] { "supergroup" }); 162 // User.createUserForTesting(conf, User.getCurrent().getShortName(), new 163 // String[] { "supergroup" }); 164 USER1 = User.createUserForTesting(conf, "user1", new String[] {}); 165 TEST_UTIL = new HBaseTestingUtil(conf); 166 TEST_UTIL.startMiniZKCluster(); 167 MiniZooKeeperCluster miniZK = TEST_UTIL.getZkCluster(); 168 zkw1 = new ZKWatcher(conf, "cluster1", null, true); 169 170 // Base conf2 on conf1 so it gets the right zk cluster. 171 conf1 = HBaseConfiguration.create(conf); 172 conf1.setInt("hfile.format.version", 3); 173 conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2"); 174 conf1.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); 175 conf1.setBoolean("hbase.tests.use.shortcircuit.reads", false); 176 conf1.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName()); 177 conf1.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, 178 TestCoprocessorForTagsAtSink.class.getName()); 179 // setVisibilityLabelServiceImpl(conf1); 180 USER1 = User.createUserForTesting(conf1, "user1", new String[] {}); 181 TEST_UTIL1 = new HBaseTestingUtil(conf1); 182 TEST_UTIL1.setZkCluster(miniZK); 183 zkw2 = new ZKWatcher(conf1, "cluster2", null, true); 184 185 TEST_UTIL.startMiniCluster(1); 186 // Wait for the labels table to become available 187 TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000); 188 TEST_UTIL1.startMiniCluster(1); 189 190 admin = TEST_UTIL.getAdmin(); 191 ReplicationPeerConfig rpc = 192 ReplicationPeerConfig.newBuilder().setClusterKey(TEST_UTIL1.getClusterKey()).build(); 193 admin.addReplicationPeer("2", rpc); 194 195 Admin hBaseAdmin = TEST_UTIL.getAdmin(); 196 TableDescriptor tableDescriptor = 197 TableDescriptorBuilder.newBuilder(TABLE_NAME).setColumnFamily(ColumnFamilyDescriptorBuilder 198 .newBuilder(fam).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()).build(); 199 try { 200 hBaseAdmin.createTable(tableDescriptor); 201 } finally { 202 if (hBaseAdmin != null) { 203 hBaseAdmin.close(); 204 } 205 } 206 Admin hBaseAdmin1 = TEST_UTIL1.getAdmin(); 207 try { 208 hBaseAdmin1.createTable(tableDescriptor); 209 } finally { 210 if (hBaseAdmin1 != null) { 211 hBaseAdmin1.close(); 212 } 213 } 214 addLabels(); 215 setAuths(conf); 216 setAuths(conf1); 217 } 218 219 protected static void setVisibilityLabelServiceImpl(Configuration conf) { 220 conf.setClass(VisibilityLabelServiceManager.VISIBILITY_LABEL_SERVICE_CLASS, 221 DefaultVisibilityLabelServiceImpl.class, VisibilityLabelService.class); 222 } 223 224 @Test 225 public void testVisibilityReplication() throws Exception { 226 int retry = 0; 227 try (Table table = writeData(TABLE_NAME, 228 "(" + SECRET + "&" + PUBLIC + ")" + "|(" + CONFIDENTIAL + ")&(" + TOPSECRET + ")", 229 "(" + PRIVATE + "|" + CONFIDENTIAL + ")&(" + PUBLIC + "|" + TOPSECRET + ")", 230 "(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET, 231 CellVisibility.quote(UNICODE_VIS_TAG) + "&" + SECRET)) { 232 Scan s = new Scan(); 233 s.setAuthorizations( 234 new Authorizations(SECRET, CONFIDENTIAL, PRIVATE, TOPSECRET, UNICODE_VIS_TAG)); 235 ResultScanner scanner = table.getScanner(s); 236 Result[] next = scanner.next(4); 237 238 assertTrue(next.length == 4); 239 CellScanner cellScanner = next[0].cellScanner(); 240 cellScanner.advance(); 241 Cell current = cellScanner.current(); 242 assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(), 243 row1, 0, row1.length)); 244 cellScanner = next[1].cellScanner(); 245 cellScanner.advance(); 246 current = cellScanner.current(); 247 assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(), 248 row2, 0, row2.length)); 249 cellScanner = next[2].cellScanner(); 250 cellScanner.advance(); 251 current = cellScanner.current(); 252 assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(), 253 row3, 0, row3.length)); 254 cellScanner = next[3].cellScanner(); 255 cellScanner.advance(); 256 current = cellScanner.current(); 257 assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(), 258 row4, 0, row4.length)); 259 try (Table table2 = TEST_UTIL1.getConnection().getTable(TABLE_NAME)) { 260 s = new Scan(); 261 // Ensure both rows are replicated 262 scanner = table2.getScanner(s); 263 next = scanner.next(4); 264 while (next.length == 0 && retry <= 10) { 265 scanner = table2.getScanner(s); 266 next = scanner.next(4); 267 Thread.sleep(2000); 268 retry++; 269 } 270 assertTrue(next.length == 4); 271 verifyGet(row1, expectedVisString[0], expected[0], false, TOPSECRET, CONFIDENTIAL); 272 TestCoprocessorForTagsAtSink.tags.clear(); 273 verifyGet(row2, expectedVisString[1], expected[1], false, CONFIDENTIAL, PUBLIC); 274 TestCoprocessorForTagsAtSink.tags.clear(); 275 verifyGet(row3, expectedVisString[2], expected[2], false, PRIVATE, SECRET); 276 verifyGet(row3, "", expected[3], true, TOPSECRET, SECRET); 277 verifyGet(row4, expectedVisString[3], expected[4], false, UNICODE_VIS_TAG, SECRET); 278 } 279 } 280 } 281 282 protected static void doAssert(byte[] row, String visTag) throws Exception { 283 if (VisibilityReplicationEndPointForTest.lastEntries == null) { 284 return; // first call 285 } 286 Assert.assertEquals(1, VisibilityReplicationEndPointForTest.lastEntries.size()); 287 List<Cell> cells = VisibilityReplicationEndPointForTest.lastEntries.get(0).getEdit().getCells(); 288 Assert.assertEquals(4, cells.size()); 289 boolean tagFound = false; 290 for (Cell cell : cells) { 291 if ( 292 (Bytes.equals(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), row, 0, 293 row.length)) 294 ) { 295 List<Tag> tags = PrivateCellUtil.getTags(cell); 296 for (Tag tag : tags) { 297 if (tag.getType() == TagType.STRING_VIS_TAG_TYPE) { 298 assertEquals(visTag, Tag.getValueAsString(tag)); 299 tagFound = true; 300 break; 301 } 302 } 303 } 304 } 305 assertTrue(tagFound); 306 } 307 308 protected void verifyGet(final byte[] row, final String visString, final int expected, 309 final boolean nullExpected, final String... auths) throws IOException, InterruptedException { 310 PrivilegedExceptionAction<Void> scanAction = new PrivilegedExceptionAction<Void>() { 311 @Override 312 public Void run() throws Exception { 313 try (Connection connection = ConnectionFactory.createConnection(conf1); 314 Table table2 = connection.getTable(TABLE_NAME)) { 315 CellScanner cellScanner; 316 Cell current; 317 Get get = new Get(row); 318 get.setAuthorizations(new Authorizations(auths)); 319 Result result = table2.get(get); 320 cellScanner = result.cellScanner(); 321 boolean advance = cellScanner.advance(); 322 if (nullExpected) { 323 assertTrue(!advance); 324 return null; 325 } 326 current = cellScanner.current(); 327 assertArrayEquals(CellUtil.cloneRow(current), row); 328 for (Tag tag : TestCoprocessorForTagsAtSink.tags) { 329 LOG.info("The tag type is " + tag.getType()); 330 } 331 assertEquals(expected, TestCoprocessorForTagsAtSink.tags.size()); 332 Tag tag = TestCoprocessorForTagsAtSink.tags.get(1); 333 if (tag.getType() != NON_VIS_TAG_TYPE) { 334 assertEquals(TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE, tag.getType()); 335 } 336 tag = TestCoprocessorForTagsAtSink.tags.get(0); 337 boolean foundNonVisTag = false; 338 for (Tag t : TestCoprocessorForTagsAtSink.tags) { 339 if (t.getType() == NON_VIS_TAG_TYPE) { 340 assertEquals(TEMP, Tag.getValueAsString(t)); 341 foundNonVisTag = true; 342 break; 343 } 344 } 345 doAssert(row, visString); 346 assertTrue(foundNonVisTag); 347 return null; 348 } 349 } 350 }; 351 USER1.runAs(scanAction); 352 } 353 354 public static void addLabels() throws Exception { 355 PrivilegedExceptionAction<VisibilityLabelsResponse> action = 356 new PrivilegedExceptionAction<VisibilityLabelsResponse>() { 357 @Override 358 public VisibilityLabelsResponse run() throws Exception { 359 String[] labels = { SECRET, TOPSECRET, CONFIDENTIAL, PUBLIC, PRIVATE, UNICODE_VIS_TAG }; 360 try (Connection conn = ConnectionFactory.createConnection(conf)) { 361 VisibilityClient.addLabels(conn, labels); 362 } catch (Throwable t) { 363 throw new IOException(t); 364 } 365 return null; 366 } 367 }; 368 SUPERUSER.runAs(action); 369 } 370 371 public static void setAuths(final Configuration conf) throws Exception { 372 PrivilegedExceptionAction<VisibilityLabelsResponse> action = 373 new PrivilegedExceptionAction<VisibilityLabelsResponse>() { 374 @Override 375 public VisibilityLabelsResponse run() throws Exception { 376 try (Connection conn = ConnectionFactory.createConnection(conf)) { 377 return VisibilityClient.setAuths(conn, 378 new String[] { SECRET, CONFIDENTIAL, PRIVATE, TOPSECRET, UNICODE_VIS_TAG }, "user1"); 379 } catch (Throwable e) { 380 throw new Exception(e); 381 } 382 } 383 }; 384 VisibilityLabelsResponse response = SUPERUSER.runAs(action); 385 } 386 387 static Table writeData(TableName tableName, String... labelExps) throws Exception { 388 Table table = TEST_UTIL.getConnection().getTable(TABLE_NAME); 389 int i = 1; 390 List<Put> puts = new ArrayList<>(labelExps.length); 391 for (String labelExp : labelExps) { 392 Put put = new Put(Bytes.toBytes("row" + i)); 393 put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value); 394 put.setCellVisibility(new CellVisibility(labelExp)); 395 put.setAttribute(NON_VISIBILITY, Bytes.toBytes(TEMP)); 396 puts.add(put); 397 i++; 398 } 399 table.put(puts); 400 return table; 401 } 402 403 // A simple BaseRegionbserver impl that allows to add a non-visibility tag from the 404 // attributes of the Put mutation. The existing cells in the put mutation is overwritten 405 // with a new cell that has the visibility tags and the non visibility tag 406 public static class SimpleCP implements RegionCoprocessor, RegionObserver { 407 @Override 408 public Optional<RegionObserver> getRegionObserver() { 409 return Optional.of(this); 410 } 411 412 @Override 413 public void prePut(ObserverContext<RegionCoprocessorEnvironment> e, Put m, WALEdit edit, 414 Durability durability) throws IOException { 415 byte[] attribute = m.getAttribute(NON_VISIBILITY); 416 byte[] cf = null; 417 List<Cell> updatedCells = new ArrayList<>(); 418 if (attribute != null) { 419 for (List<? extends Cell> edits : m.getFamilyCellMap().values()) { 420 for (Cell cell : edits) { 421 KeyValue kv = KeyValueUtil.ensureKeyValue(cell); 422 if (cf == null) { 423 cf = CellUtil.cloneFamily(kv); 424 } 425 Tag tag = new ArrayBackedTag((byte) NON_VIS_TAG_TYPE, attribute); 426 List<Tag> tagList = new ArrayList<>(PrivateCellUtil.getTags(cell).size() + 1); 427 tagList.add(tag); 428 tagList.addAll(PrivateCellUtil.getTags(cell)); 429 Cell newcell = PrivateCellUtil.createCell(kv, tagList); 430 ((List<Cell>) updatedCells).add(newcell); 431 } 432 } 433 m.getFamilyCellMap().remove(cf); 434 // Update the family map 435 m.getFamilyCellMap().put(cf, updatedCells); 436 } 437 } 438 } 439 440 public static class TestCoprocessorForTagsAtSink implements RegionCoprocessor, RegionObserver { 441 public static List<Tag> tags = null; 442 443 @Override 444 public Optional<RegionObserver> getRegionObserver() { 445 return Optional.of(this); 446 } 447 448 @Override 449 public void postGetOp(ObserverContext<RegionCoprocessorEnvironment> e, Get get, 450 List<Cell> results) throws IOException { 451 if (results.size() > 0) { 452 // Check tag presence in the 1st cell in 1st Result 453 if (!results.isEmpty()) { 454 Cell cell = results.get(0); 455 tags = PrivateCellUtil.getTags(cell); 456 } 457 } 458 } 459 } 460 461 /** 462 * An extn of VisibilityReplicationEndpoint to verify the tags that are replicated 463 */ 464 public static class VisibilityReplicationEndPointForTest extends VisibilityReplicationEndpoint { 465 static AtomicInteger replicateCount = new AtomicInteger(); 466 static volatile List<Entry> lastEntries = null; 467 468 public VisibilityReplicationEndPointForTest(ReplicationEndpoint endpoint, 469 VisibilityLabelService visibilityLabelsService) { 470 super(endpoint, visibilityLabelsService); 471 } 472 473 @Override 474 public boolean replicate(ReplicateContext replicateContext) { 475 boolean ret = super.replicate(replicateContext); 476 lastEntries = replicateContext.getEntries(); 477 replicateCount.incrementAndGet(); 478 return ret; 479 } 480 } 481}