001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.security.token; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertNotNull; 023import static org.junit.Assert.assertNull; 024import static org.junit.Assert.assertTrue; 025 026import java.util.concurrent.CountDownLatch; 027import org.apache.hadoop.conf.Configuration; 028import org.apache.hadoop.hbase.Abortable; 029import org.apache.hadoop.hbase.HBaseClassTestRule; 030import org.apache.hadoop.hbase.HBaseConfiguration; 031import org.apache.hadoop.hbase.HBaseTestingUtil; 032import org.apache.hadoop.hbase.Waiter; 033import org.apache.hadoop.hbase.testclassification.MediumTests; 034import org.apache.hadoop.hbase.testclassification.SecurityTests; 035import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 036import org.apache.hadoop.hbase.zookeeper.ZKWatcher; 037import org.junit.AfterClass; 038import org.junit.BeforeClass; 039import org.junit.ClassRule; 040import org.junit.Test; 041import org.junit.experimental.categories.Category; 042import org.slf4j.Logger; 043import org.slf4j.LoggerFactory; 044 045/** 046 * Test the synchronization of token authentication master keys through ZKSecretWatcher 047 */ 048@Category({ SecurityTests.class, MediumTests.class }) 049public class TestZKSecretWatcher { 050 051 @ClassRule 052 public static final HBaseClassTestRule CLASS_RULE = 053 HBaseClassTestRule.forClass(TestZKSecretWatcher.class); 054 055 private static final Logger LOG = LoggerFactory.getLogger(TestZKSecretWatcher.class); 056 private static HBaseTestingUtil TEST_UTIL; 057 private static AuthenticationTokenSecretManager KEY_MASTER; 058 private static AuthenticationTokenSecretManagerForTest KEY_SLAVE; 059 private static AuthenticationTokenSecretManager KEY_SLAVE2; 060 private static AuthenticationTokenSecretManager KEY_SLAVE3; 061 062 private static class MockAbortable implements Abortable { 063 private boolean abort; 064 065 @Override 066 public void abort(String reason, Throwable e) { 067 LOG.info("Aborting: " + reason, e); 068 abort = true; 069 } 070 071 @Override 072 public boolean isAborted() { 073 return abort; 074 } 075 } 076 077 // We subclass AuthenticationTokenSecretManager so that testKeyUpdate can receive 078 // notification on the removal of keyId 079 private static class AuthenticationTokenSecretManagerForTest 080 extends AuthenticationTokenSecretManager { 081 private CountDownLatch latch = new CountDownLatch(1); 082 083 public AuthenticationTokenSecretManagerForTest(Configuration conf, ZKWatcher zk, 084 String serverName, long keyUpdateInterval, long tokenMaxLifetime) { 085 super(conf, zk, serverName, keyUpdateInterval, tokenMaxLifetime); 086 } 087 088 @Override 089 synchronized boolean removeKey(Integer keyId) { 090 boolean b = super.removeKey(keyId); 091 if (b) { 092 latch.countDown(); 093 } 094 return b; 095 } 096 097 CountDownLatch getLatch() { 098 return latch; 099 } 100 } 101 102 @BeforeClass 103 public static void setupBeforeClass() throws Exception { 104 TEST_UTIL = new HBaseTestingUtil(); 105 TEST_UTIL.startMiniZKCluster(); 106 Configuration conf = TEST_UTIL.getConfiguration(); 107 108 ZKWatcher zk = newZK(conf, "server1", new MockAbortable()); 109 AuthenticationTokenSecretManagerForTest[] tmp = new AuthenticationTokenSecretManagerForTest[2]; 110 tmp[0] = 111 new AuthenticationTokenSecretManagerForTest(conf, zk, "server1", 60 * 60 * 1000, 60 * 1000); 112 tmp[0].start(); 113 114 zk = newZK(conf, "server2", new MockAbortable()); 115 tmp[1] = 116 new AuthenticationTokenSecretManagerForTest(conf, zk, "server2", 60 * 60 * 1000, 60 * 1000); 117 tmp[1].start(); 118 119 while (KEY_MASTER == null) { 120 for (int i = 0; i < 2; i++) { 121 if (tmp[i].isMaster()) { 122 KEY_MASTER = tmp[i]; 123 KEY_SLAVE = tmp[(i + 1) % 2]; 124 break; 125 } 126 } 127 Thread.sleep(500); 128 } 129 LOG.info("Master is " + KEY_MASTER.getName() + ", slave is " + KEY_SLAVE.getName()); 130 } 131 132 @AfterClass 133 public static void tearDownAfterClass() throws Exception { 134 TEST_UTIL.shutdownMiniZKCluster(); 135 } 136 137 @Test 138 public void testKeyUpdate() throws Exception { 139 // sanity check 140 assertTrue(KEY_MASTER.isMaster()); 141 assertFalse(KEY_SLAVE.isMaster()); 142 int maxKeyId = 0; 143 144 KEY_MASTER.rollCurrentKey(); 145 AuthenticationKey key1 = KEY_MASTER.getCurrentKey(); 146 assertNotNull(key1); 147 LOG.debug("Master current key (key1) {}", key1); 148 149 // wait for slave to update 150 Thread.sleep(1000); 151 AuthenticationKey slaveCurrent = KEY_SLAVE.getCurrentKey(); 152 assertNotNull(slaveCurrent); 153 assertEquals(key1, slaveCurrent); 154 LOG.debug("Slave current key (key1) {}", slaveCurrent); 155 156 // generate two more keys then expire the original 157 KEY_MASTER.rollCurrentKey(); 158 AuthenticationKey key2 = KEY_MASTER.getCurrentKey(); 159 LOG.debug("Master new current key (key2) {}", key2); 160 KEY_MASTER.rollCurrentKey(); 161 AuthenticationKey key3 = KEY_MASTER.getCurrentKey(); 162 LOG.debug("Master new current key (key3) {}", key3); 163 164 // force expire the original key 165 key1.setExpiration(EnvironmentEdgeManager.currentTime() - 100000); 166 KEY_MASTER.removeExpiredKeys(); 167 // verify removed from master 168 assertNull(KEY_MASTER.getKey(key1.getKeyId())); 169 170 // Wait for slave to catch up. When remove hits KEY_SLAVE, we'll clear 171 // the latch and will progress beyond the await. 172 KEY_SLAVE.getLatch().await(); 173 // make sure the slave has both new keys 174 AuthenticationKey slave2 = KEY_SLAVE.getKey(key2.getKeyId()); 175 assertNotNull(slave2); 176 assertEquals(key2, slave2); 177 AuthenticationKey slave3 = KEY_SLAVE.getKey(key3.getKeyId()); 178 assertNotNull(slave3); 179 assertEquals(key3, slave3); 180 slaveCurrent = KEY_SLAVE.getCurrentKey(); 181 assertEquals(key3, slaveCurrent); 182 LOG.debug("Slave current key (key3) {}", slaveCurrent); 183 184 // verify that the expired key has been removed 185 Waiter.waitFor(TEST_UTIL.getConfiguration(), 30000, () -> { 186 AuthenticationKey k = KEY_SLAVE.getKey(key1.getKeyId()); 187 LOG.info("AuthKey1={}", k); 188 return k == null; 189 }); 190 assertNull("key1=" + KEY_SLAVE.getKey(key1.getKeyId()), KEY_SLAVE.getKey(key1.getKeyId())); 191 192 // bring up a new slave 193 Configuration conf = TEST_UTIL.getConfiguration(); 194 ZKWatcher zk = newZK(conf, "server3", new MockAbortable()); 195 KEY_SLAVE2 = 196 new AuthenticationTokenSecretManager(conf, zk, "server3", 60 * 60 * 1000, 60 * 1000); 197 KEY_SLAVE2.start(); 198 199 Thread.sleep(1000); 200 // verify the new slave has current keys (and not expired) 201 slave2 = KEY_SLAVE2.getKey(key2.getKeyId()); 202 assertNotNull(slave2); 203 assertEquals(key2, slave2); 204 slave3 = KEY_SLAVE2.getKey(key3.getKeyId()); 205 assertNotNull(slave3); 206 assertEquals(key3, slave3); 207 slaveCurrent = KEY_SLAVE2.getCurrentKey(); 208 assertEquals(key3, slaveCurrent); 209 assertNull(KEY_SLAVE2.getKey(key1.getKeyId())); 210 211 // test leader failover 212 KEY_MASTER.stop(); 213 214 // wait for master to stop 215 Thread.sleep(1000); 216 assertFalse(KEY_MASTER.isMaster()); 217 218 // check for a new master 219 AuthenticationTokenSecretManager[] mgrs = 220 new AuthenticationTokenSecretManager[] { KEY_SLAVE, KEY_SLAVE2 }; 221 AuthenticationTokenSecretManager newMaster = null; 222 int tries = 0; 223 while (newMaster == null && tries++ < 5) { 224 for (AuthenticationTokenSecretManager mgr : mgrs) { 225 if (mgr.isMaster()) { 226 newMaster = mgr; 227 break; 228 } 229 } 230 if (newMaster == null) { 231 Thread.sleep(500); 232 } 233 } 234 assertNotNull(newMaster); 235 236 AuthenticationKey current = newMaster.getCurrentKey(); 237 // new master will immediately roll the current key, so it's current may be greater 238 assertTrue(current.getKeyId() >= slaveCurrent.getKeyId()); 239 LOG.debug("New master, current key: " + current.getKeyId()); 240 241 // roll the current key again on new master and verify the key ID increments 242 newMaster.rollCurrentKey(); 243 AuthenticationKey newCurrent = newMaster.getCurrentKey(); 244 LOG.debug("New master, rolled new current key: " + newCurrent.getKeyId()); 245 assertTrue(newCurrent.getKeyId() > current.getKeyId()); 246 247 // add another slave 248 ZKWatcher zk3 = newZK(conf, "server4", new MockAbortable()); 249 KEY_SLAVE3 = 250 new AuthenticationTokenSecretManager(conf, zk3, "server4", 60 * 60 * 1000, 60 * 1000); 251 KEY_SLAVE3.start(); 252 Thread.sleep(5000); 253 254 // check master failover again 255 newMaster.stop(); 256 257 // wait for master to stop 258 Thread.sleep(5000); 259 assertFalse(newMaster.isMaster()); 260 261 // check for a new master 262 mgrs = new AuthenticationTokenSecretManager[] { KEY_SLAVE, KEY_SLAVE2, KEY_SLAVE3 }; 263 newMaster = null; 264 tries = 0; 265 while (newMaster == null && tries++ < 5) { 266 for (AuthenticationTokenSecretManager mgr : mgrs) { 267 if (mgr.isMaster()) { 268 newMaster = mgr; 269 break; 270 } 271 } 272 if (newMaster == null) { 273 Thread.sleep(500); 274 } 275 } 276 assertNotNull(newMaster); 277 278 AuthenticationKey current2 = newMaster.getCurrentKey(); 279 // new master will immediately roll the current key, so it's current may be greater 280 assertTrue(current2.getKeyId() >= newCurrent.getKeyId()); 281 LOG.debug("New master 2, current key: " + current2.getKeyId()); 282 283 // roll the current key again on new master and verify the key ID increments 284 newMaster.rollCurrentKey(); 285 AuthenticationKey newCurrent2 = newMaster.getCurrentKey(); 286 LOG.debug("New master 2, rolled new current key: " + newCurrent2.getKeyId()); 287 assertTrue(newCurrent2.getKeyId() > current2.getKeyId()); 288 } 289 290 private static ZKWatcher newZK(Configuration conf, String name, Abortable abort) 291 throws Exception { 292 Configuration copy = HBaseConfiguration.create(conf); 293 ZKWatcher zk = new ZKWatcher(copy, name, abort); 294 return zk; 295 } 296}