001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.security.token;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertNull;
024import static org.junit.Assert.assertTrue;
025
026import java.util.concurrent.CountDownLatch;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.Abortable;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseConfiguration;
031import org.apache.hadoop.hbase.HBaseTestingUtility;
032import org.apache.hadoop.hbase.Waiter;
033import org.apache.hadoop.hbase.testclassification.MediumTests;
034import org.apache.hadoop.hbase.testclassification.SecurityTests;
035import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
036import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
037import org.junit.AfterClass;
038import org.junit.BeforeClass;
039import org.junit.ClassRule;
040import org.junit.Test;
041import org.junit.experimental.categories.Category;
042import org.slf4j.Logger;
043import org.slf4j.LoggerFactory;
044
045/**
046 * Test the synchronization of token authentication master keys through
047 * ZKSecretWatcher
048 */
049@Category({SecurityTests.class, MediumTests.class})
050public class TestZKSecretWatcher {
051
052  @ClassRule
053  public static final HBaseClassTestRule CLASS_RULE =
054      HBaseClassTestRule.forClass(TestZKSecretWatcher.class);
055
056  private static final Logger LOG = LoggerFactory.getLogger(TestZKSecretWatcher.class);
057  private static HBaseTestingUtility TEST_UTIL;
058  private static AuthenticationTokenSecretManager KEY_MASTER;
059  private static AuthenticationTokenSecretManagerForTest KEY_SLAVE;
060  private static AuthenticationTokenSecretManager KEY_SLAVE2;
061  private static AuthenticationTokenSecretManager KEY_SLAVE3;
062
063  private static class MockAbortable implements Abortable {
064    private boolean abort;
065    @Override
066    public void abort(String reason, Throwable e) {
067      LOG.info("Aborting: "+reason, e);
068      abort = true;
069    }
070
071    @Override
072    public boolean isAborted() {
073      return abort;
074    }
075  }
076
077  // We subclass AuthenticationTokenSecretManager so that testKeyUpdate can receive
078  // notification on the removal of keyId
079  private static class AuthenticationTokenSecretManagerForTest
080  extends AuthenticationTokenSecretManager {
081    private CountDownLatch latch = new CountDownLatch(1);
082
083    public AuthenticationTokenSecretManagerForTest(Configuration conf,
084                                                   ZKWatcher zk, String serverName,
085                                                   long keyUpdateInterval, long tokenMaxLifetime) {
086      super(conf, zk, serverName, keyUpdateInterval, tokenMaxLifetime);
087    }
088
089    @Override
090    synchronized boolean removeKey(Integer keyId) {
091      boolean b = super.removeKey(keyId);
092      if (b) {
093        latch.countDown();
094      }
095      return b;
096    }
097
098    CountDownLatch getLatch() {
099      return latch;
100    }
101  }
102
103  @BeforeClass
104  public static void setupBeforeClass() throws Exception {
105    TEST_UTIL = new HBaseTestingUtility();
106    TEST_UTIL.startMiniZKCluster();
107    Configuration conf = TEST_UTIL.getConfiguration();
108
109    ZKWatcher zk = newZK(conf, "server1", new MockAbortable());
110    AuthenticationTokenSecretManagerForTest[] tmp = new AuthenticationTokenSecretManagerForTest[2];
111    tmp[0] = new AuthenticationTokenSecretManagerForTest(
112        conf, zk, "server1", 60*60*1000, 60*1000);
113    tmp[0].start();
114
115    zk = newZK(conf, "server2", new MockAbortable());
116    tmp[1] = new AuthenticationTokenSecretManagerForTest(
117        conf, zk, "server2", 60*60*1000, 60*1000);
118    tmp[1].start();
119
120    while (KEY_MASTER == null) {
121      for (int i=0; i<2; i++) {
122        if (tmp[i].isMaster()) {
123          KEY_MASTER = tmp[i];
124          KEY_SLAVE = tmp[ (i+1) % 2 ];
125          break;
126        }
127      }
128      Thread.sleep(500);
129    }
130    LOG.info("Master is "+KEY_MASTER.getName()+
131        ", slave is "+KEY_SLAVE.getName());
132  }
133
134  @AfterClass
135  public static void tearDownAfterClass() throws Exception {
136    TEST_UTIL.shutdownMiniZKCluster();
137  }
138
139  @Test
140  public void testKeyUpdate() throws Exception {
141    // sanity check
142    assertTrue(KEY_MASTER.isMaster());
143    assertFalse(KEY_SLAVE.isMaster());
144    int maxKeyId = 0;
145
146    KEY_MASTER.rollCurrentKey();
147    AuthenticationKey key1 = KEY_MASTER.getCurrentKey();
148    assertNotNull(key1);
149    LOG.debug("Master current key (key1) {}", key1);
150
151    // wait for slave to update
152    Thread.sleep(1000);
153    AuthenticationKey slaveCurrent = KEY_SLAVE.getCurrentKey();
154    assertNotNull(slaveCurrent);
155    assertEquals(key1, slaveCurrent);
156    LOG.debug("Slave current key (key1) {}", slaveCurrent);
157
158    // generate two more keys then expire the original
159    KEY_MASTER.rollCurrentKey();
160    AuthenticationKey key2 = KEY_MASTER.getCurrentKey();
161    LOG.debug("Master new current key (key2) {}", key2);
162    KEY_MASTER.rollCurrentKey();
163    AuthenticationKey key3 = KEY_MASTER.getCurrentKey();
164    LOG.debug("Master new current key (key3) {}", key3);
165
166    // force expire the original key
167    key1.setExpiration(EnvironmentEdgeManager.currentTime() - 100000);
168    KEY_MASTER.removeExpiredKeys();
169    // verify removed from master
170    assertNull(KEY_MASTER.getKey(key1.getKeyId()));
171
172    // Wait for slave to catch up. When remove hits KEY_SLAVE, we'll clear
173    // the latch and will progress beyond the await.
174    KEY_SLAVE.getLatch().await();
175    // make sure the slave has both new keys
176    AuthenticationKey slave2 = KEY_SLAVE.getKey(key2.getKeyId());
177    assertNotNull(slave2);
178    assertEquals(key2, slave2);
179    AuthenticationKey slave3 = KEY_SLAVE.getKey(key3.getKeyId());
180    assertNotNull(slave3);
181    assertEquals(key3, slave3);
182    slaveCurrent = KEY_SLAVE.getCurrentKey();
183    assertEquals(key3, slaveCurrent);
184    LOG.debug("Slave current key (key3) {}", slaveCurrent);
185
186    // verify that the expired key has been removed
187    Waiter.waitFor(TEST_UTIL.getConfiguration(), 30000,
188      () -> {
189      AuthenticationKey k = KEY_SLAVE.getKey(key1.getKeyId());
190      LOG.info("AuthKey1={}", k);
191      return k == null;});
192    assertNull("key1=" + KEY_SLAVE.getKey(key1.getKeyId()),
193      KEY_SLAVE.getKey(key1.getKeyId()));
194
195    // bring up a new slave
196    Configuration conf = TEST_UTIL.getConfiguration();
197    ZKWatcher zk = newZK(conf, "server3", new MockAbortable());
198    KEY_SLAVE2 = new AuthenticationTokenSecretManager(
199        conf, zk, "server3", 60*60*1000, 60*1000);
200    KEY_SLAVE2.start();
201
202    Thread.sleep(1000);
203    // verify the new slave has current keys (and not expired)
204    slave2 = KEY_SLAVE2.getKey(key2.getKeyId());
205    assertNotNull(slave2);
206    assertEquals(key2, slave2);
207    slave3 = KEY_SLAVE2.getKey(key3.getKeyId());
208    assertNotNull(slave3);
209    assertEquals(key3, slave3);
210    slaveCurrent = KEY_SLAVE2.getCurrentKey();
211    assertEquals(key3, slaveCurrent);
212    assertNull(KEY_SLAVE2.getKey(key1.getKeyId()));
213
214    // test leader failover
215    KEY_MASTER.stop();
216
217    // wait for master to stop
218    Thread.sleep(1000);
219    assertFalse(KEY_MASTER.isMaster());
220
221    // check for a new master
222    AuthenticationTokenSecretManager[] mgrs =
223        new AuthenticationTokenSecretManager[]{ KEY_SLAVE, KEY_SLAVE2 };
224    AuthenticationTokenSecretManager newMaster = null;
225    int tries = 0;
226    while (newMaster == null && tries++ < 5) {
227      for (AuthenticationTokenSecretManager mgr : mgrs) {
228        if (mgr.isMaster()) {
229          newMaster = mgr;
230          break;
231        }
232      }
233      if (newMaster == null) {
234        Thread.sleep(500);
235      }
236    }
237    assertNotNull(newMaster);
238
239    AuthenticationKey current = newMaster.getCurrentKey();
240    // new master will immediately roll the current key, so it's current may be greater
241    assertTrue(current.getKeyId() >= slaveCurrent.getKeyId());
242    LOG.debug("New master, current key: "+current.getKeyId());
243
244    // roll the current key again on new master and verify the key ID increments
245    newMaster.rollCurrentKey();
246    AuthenticationKey newCurrent = newMaster.getCurrentKey();
247    LOG.debug("New master, rolled new current key: "+newCurrent.getKeyId());
248    assertTrue(newCurrent.getKeyId() > current.getKeyId());
249
250    // add another slave
251    ZKWatcher zk3 = newZK(conf, "server4", new MockAbortable());
252    KEY_SLAVE3 = new AuthenticationTokenSecretManager(
253        conf, zk3, "server4", 60*60*1000, 60*1000);
254    KEY_SLAVE3.start();
255    Thread.sleep(5000);
256
257    // check master failover again
258    newMaster.stop();
259
260    // wait for master to stop
261    Thread.sleep(5000);
262    assertFalse(newMaster.isMaster());
263
264    // check for a new master
265    mgrs = new AuthenticationTokenSecretManager[]{ KEY_SLAVE, KEY_SLAVE2, KEY_SLAVE3 };
266    newMaster = null;
267    tries = 0;
268    while (newMaster == null && tries++ < 5) {
269      for (AuthenticationTokenSecretManager mgr : mgrs) {
270        if (mgr.isMaster()) {
271          newMaster = mgr;
272          break;
273        }
274      }
275      if (newMaster == null) {
276        Thread.sleep(500);
277      }
278    }
279    assertNotNull(newMaster);
280
281    AuthenticationKey current2 = newMaster.getCurrentKey();
282    // new master will immediately roll the current key, so it's current may be greater
283    assertTrue(current2.getKeyId() >= newCurrent.getKeyId());
284    LOG.debug("New master 2, current key: "+current2.getKeyId());
285
286    // roll the current key again on new master and verify the key ID increments
287    newMaster.rollCurrentKey();
288    AuthenticationKey newCurrent2 = newMaster.getCurrentKey();
289    LOG.debug("New master 2, rolled new current key: "+newCurrent2.getKeyId());
290    assertTrue(newCurrent2.getKeyId() > current2.getKeyId());
291  }
292
293  private static ZKWatcher newZK(Configuration conf, String name,
294                                 Abortable abort) throws Exception {
295    Configuration copy = HBaseConfiguration.create(conf);
296    ZKWatcher zk = new ZKWatcher(copy, name, abort);
297    return zk;
298  }
299}