001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.tool; 019 020import static org.junit.jupiter.api.Assertions.assertEquals; 021import static org.junit.jupiter.api.Assertions.assertFalse; 022import static org.junit.jupiter.api.Assertions.assertThrows; 023import static org.junit.jupiter.api.Assertions.assertTrue; 024 025import java.io.IOException; 026import java.security.PrivilegedExceptionAction; 027import java.util.Arrays; 028import java.util.Set; 029import java.util.TreeSet; 030import java.util.UUID; 031import org.apache.hadoop.fs.FileSystem; 032import org.apache.hadoop.fs.Path; 033import org.apache.hadoop.fs.permission.FsAction; 034import org.apache.hadoop.fs.permission.FsPermission; 035import org.apache.hadoop.hbase.HBaseTestingUtil; 036import org.apache.hadoop.hbase.KeyValue; 037import org.apache.hadoop.hbase.TableName; 038import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 039import org.apache.hadoop.hbase.client.Result; 040import org.apache.hadoop.hbase.client.ResultScanner; 041import org.apache.hadoop.hbase.client.Scan; 042import org.apache.hadoop.hbase.client.Table; 043import org.apache.hadoop.hbase.client.TableDescriptor; 044import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 045import org.apache.hadoop.hbase.io.hfile.CacheConfig; 046import org.apache.hadoop.hbase.io.hfile.HFile; 047import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; 048import org.apache.hadoop.hbase.testclassification.LargeTests; 049import org.apache.hadoop.hbase.testclassification.MiscTests; 050import org.apache.hadoop.hbase.util.Bytes; 051import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 052import org.apache.hadoop.security.AccessControlException; 053import org.apache.hadoop.security.UserGroupInformation; 054import org.junit.jupiter.api.AfterAll; 055import org.junit.jupiter.api.BeforeAll; 056import org.junit.jupiter.api.Tag; 057import org.junit.jupiter.api.Test; 058 059/** 060 * End-to-end bulk load test on a MiniDFSCluster with HDFS permissions enabled and three distinct 061 * identities: the HBase service user (which runs the mini cluster and must not be the HDFS 062 * superuser), the bulk load client (which owns the input HFiles and the target table) and an 063 * unrelated attacker user. 064 * <p> 065 * A single HFile spanning the table split point is loaded, so BulkLoadHFilesTool has to split it 066 * into {@code .tmp/<uuid>.bottom} and {@code .tmp/<uuid>.top} inside the client's own directory. 067 * The test verifies that the load succeeds without any world-writable permission on the split 068 * artifacts (they are handed to the service user through the staging directory of 069 * {@code SecureBulkLoadManager}, which is protected by a random directory name), and that a third 070 * user cannot tamper with the split directory. 071 */ 072@Tag(MiscTests.TAG) 073@Tag(LargeTests.TAG) 074public class TestBulkLoadHFilesWithDfsPermissions { 075 076 private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); 077 078 private static final TableName TABLE = TableName.valueOf("bulkload_multiuser"); 079 private static final byte[] FAMILY = Bytes.toBytes("cf"); 080 private static final byte[] QUALIFIER = Bytes.toBytes("q"); 081 private static final byte[] SPLIT_KEY = Bytes.toBytes("m"); 082 private static final String[] ROWS = { "a1", "b1", "y1", "z1" }; 083 084 // The MiniDFSCluster superuser is the OS user running the test (the one that started the 085 // NameNode). The identities below are simple-auth identities enforced by HDFS permissions. 086 private static UserGroupInformation originalLogin; 087 private static UserGroupInformation serviceUgi; 088 private static UserGroupInformation clientUgi; 089 private static UserGroupInformation attackerUgi; 090 private static FileSystem superuserFs; 091 092 @BeforeAll 093 public static void setUpBeforeClass() throws Exception { 094 UTIL.getConfiguration().setBoolean("dfs.permissions", true); 095 UTIL.getConfiguration().setBoolean("dfs.permissions.enabled", true); 096 // Start the DFS cluster first, under the test-runner identity: the test runner becomes the 097 // HDFS superuser of the mini cluster. 098 UTIL.startMiniDFSCluster(1); 099 UTIL.startMiniZKCluster(); 100 superuserFs = UTIL.getDFSCluster().getFileSystem(); 101 102 originalLogin = UserGroupInformation.getLoginUser(); 103 serviceUgi = UserGroupInformation.createUserForTesting("hbase_service_user", 104 new String[] { "hbase_service_group" }); 105 clientUgi = UserGroupInformation.createUserForTesting("bulkload_client_user", 106 new String[] { "bulkload_client_group" }); 107 attackerUgi = UserGroupInformation.createUserForTesting("bulkload_attacker_user", 108 new String[] { "bulkload_attacker_group" }); 109 110 // Hand the cluster data directories to the service user before it starts using them. 111 Path dataDir = UTIL.getDataTestDirOnTestFS(); 112 superuserFs.mkdirs(dataDir); 113 superuserFs.setOwner(dataDir, serviceUgi.getUserName(), serviceUgi.getUserName()); 114 superuserFs.setOwner(dataDir.getParent(), serviceUgi.getUserName(), serviceUgi.getUserName()); 115 116 // Start HBase under the service identity: every FileSystem the servers create is bound to 117 // "hbase_service_user", which is not the HDFS superuser. 118 UserGroupInformation.setLoginUser(serviceUgi); 119 UTIL.startMiniHBaseCluster(); 120 121 // Two regions: ("", "m") and ["m", ""). A single HFile with rows on both sides of "m" has to 122 // be split by BulkLoadHFilesTool before loading. 123 TableDescriptor desc = TableDescriptorBuilder.newBuilder(TABLE) 124 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).build(); 125 UTIL.getAdmin().createTable(desc, new byte[][] { SPLIT_KEY }); 126 } 127 128 @AfterAll 129 public static void tearDownAfterClass() throws Exception { 130 UserGroupInformation.setLoginUser(originalLogin); 131 UTIL.shutdownMiniCluster(); 132 } 133 134 @Test 135 public void testBulkLoadWithSplitUnderHdfsPermissions() throws Exception { 136 // Precondition: the HBase service user is not the HDFS superuser (it cannot write outside 137 // of the directories it owns). 138 assertThrows(AccessControlException.class, () -> runAs(serviceUgi, () -> FileSystem 139 .get(UTIL.getConfiguration()).create(new Path("/superuser-check-" + UUID.randomUUID())))); 140 141 // The client prepares the input HFile in its own directory, spanning the split point. 142 Path inputDir = new Path("/bulkload-input-" + UUID.randomUUID()); 143 superuserFs.mkdirs(inputDir); 144 superuserFs.setOwner(inputDir, clientUgi.getUserName(), clientUgi.getUserName()); 145 runAs(clientUgi, () -> { 146 FileSystem clientFs = FileSystem.get(UTIL.getConfiguration()); 147 Path familyDir = new Path(inputDir, Bytes.toString(FAMILY)); 148 clientFs.mkdirs(familyDir); 149 writeHFile(clientFs, new Path(familyDir, "spanning.hfile")); 150 return null; 151 }); 152 153 // Bulk load as the client. The HFile does not fit into a single region, so it is split into 154 // .tmp/<uuid>.bottom and .tmp/<uuid>.top in the client's directory first, then the split 155 // outputs are moved through the staging directory of SecureBulkLoadManager into the regions. 156 runAs(clientUgi, () -> { 157 BulkLoadHFiles.create(UTIL.getConfiguration()).bulkLoad(TABLE, inputDir); 158 return null; 159 }); 160 161 FileSystem clientFs = runAs(clientUgi, () -> FileSystem.get(UTIL.getConfiguration())); 162 Path tmpDir = new Path(new Path(inputDir, Bytes.toString(FAMILY)), BulkLoadHFilesTool.TMP_DIR); 163 assertTrue(clientFs.exists(tmpDir), "split did not happen, " + tmpDir + " is missing"); 164 assertEquals(0, clientFs.listStatus(tmpDir).length, "split outputs were not consumed"); 165 166 // The split directory must not carry loose permissions: neither the group nor others may 167 // write to it. (BulkLoadHFilesTool used to chmod it to 777.) 168 FsPermission tmpPerm = clientFs.getFileStatus(tmpDir).getPermission(); 169 assertFalse(tmpPerm.getOtherAction().implies(FsAction.WRITE), 170 "split tmp dir must not be world-writable: " + tmpPerm); 171 assertFalse(tmpPerm.getGroupAction().implies(FsAction.WRITE), 172 "split tmp dir must not be group-writable: " + tmpPerm); 173 174 // An unrelated user must not be able to tamper with the split directory, not even after the 175 // load has completed. 176 assertThrows(AccessControlException.class, () -> runAs(attackerUgi, () -> { 177 FileSystem attackerFs = FileSystem.get(UTIL.getConfiguration()); 178 attackerFs.create(new Path(tmpDir, "evil")); 179 return null; 180 })); 181 182 // All rows must have been loaded into the correct regions with their values preserved. 183 Set<String> rows = new TreeSet<>(); 184 try (Table table = UTIL.getConnection().getTable(TABLE); 185 ResultScanner scanner = table.getScanner(new Scan())) { 186 for (Result result : scanner) { 187 String row = Bytes.toString(result.getRow()); 188 rows.add(row); 189 assertEquals(row, Bytes.toString(result.getValue(FAMILY, QUALIFIER))); 190 } 191 } 192 assertEquals(new TreeSet<>(Arrays.asList(ROWS)), rows); 193 } 194 195 private static <T> T runAs(UserGroupInformation ugi, PrivilegedExceptionAction<T> action) 196 throws Exception { 197 return ugi.doAs(action); 198 } 199 200 private static void writeHFile(FileSystem fs, Path path) throws IOException { 201 HFile.Writer writer = HFile.getWriterFactory(UTIL.getConfiguration(), CacheConfig.DISABLED) 202 .withPath(fs, path).withFileContext(new HFileContextBuilder().build()).create(); 203 try { 204 long ts = EnvironmentEdgeManager.currentTime(); 205 for (String row : ROWS) { 206 writer.append(new KeyValue(Bytes.toBytes(row), FAMILY, QUALIFIER, ts, Bytes.toBytes(row))); 207 } 208 } finally { 209 writer.close(); 210 } 211 } 212}