001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.security.token;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertNull;
024import static org.junit.Assert.assertTrue;
025
026import java.util.concurrent.CountDownLatch;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.Abortable;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseConfiguration;
031import org.apache.hadoop.hbase.HBaseTestingUtility;
032import org.apache.hadoop.hbase.testclassification.LargeTests;
033import org.apache.hadoop.hbase.testclassification.SecurityTests;
034import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
035import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
036import org.junit.AfterClass;
037import org.junit.BeforeClass;
038import org.junit.ClassRule;
039import org.junit.Test;
040import org.junit.experimental.categories.Category;
041import org.slf4j.Logger;
042import org.slf4j.LoggerFactory;
043
044/**
045 * Test the synchronization of token authentication master keys through
046 * ZKSecretWatcher
047 */
048@Category({SecurityTests.class, LargeTests.class})
049public class TestZKSecretWatcher {
050
051  @ClassRule
052  public static final HBaseClassTestRule CLASS_RULE =
053      HBaseClassTestRule.forClass(TestZKSecretWatcher.class);
054
055  private static final Logger LOG = LoggerFactory.getLogger(TestZKSecretWatcher.class);
056  private static HBaseTestingUtility TEST_UTIL;
057  private static AuthenticationTokenSecretManager KEY_MASTER;
058  private static AuthenticationTokenSecretManagerForTest KEY_SLAVE;
059  private static AuthenticationTokenSecretManager KEY_SLAVE2;
060  private static AuthenticationTokenSecretManager KEY_SLAVE3;
061
062  private static class MockAbortable implements Abortable {
063    private boolean abort;
064    @Override
065    public void abort(String reason, Throwable e) {
066      LOG.info("Aborting: "+reason, e);
067      abort = true;
068    }
069
070    @Override
071    public boolean isAborted() {
072      return abort;
073    }
074  }
075
076  // We subclass AuthenticationTokenSecretManager so that testKeyUpdate can receive
077  // notification on the removal of keyId
078  private static class AuthenticationTokenSecretManagerForTest
079  extends AuthenticationTokenSecretManager {
080    private CountDownLatch latch = new CountDownLatch(1);
081
082    public AuthenticationTokenSecretManagerForTest(Configuration conf,
083                                                   ZKWatcher zk, String serverName,
084                                                   long keyUpdateInterval, long tokenMaxLifetime) {
085      super(conf, zk, serverName, keyUpdateInterval, tokenMaxLifetime);
086    }
087
088    @Override
089    synchronized boolean removeKey(Integer keyId) {
090      boolean b = super.removeKey(keyId);
091      if (b) {
092        latch.countDown();
093      }
094      return b;
095    }
096
097    CountDownLatch getLatch() {
098      return latch;
099    }
100  }
101
102  @BeforeClass
103  public static void setupBeforeClass() throws Exception {
104    TEST_UTIL = new HBaseTestingUtility();
105    TEST_UTIL.startMiniZKCluster();
106    Configuration conf = TEST_UTIL.getConfiguration();
107
108    ZKWatcher zk = newZK(conf, "server1", new MockAbortable());
109    AuthenticationTokenSecretManagerForTest[] tmp = new AuthenticationTokenSecretManagerForTest[2];
110    tmp[0] = new AuthenticationTokenSecretManagerForTest(
111        conf, zk, "server1", 60*60*1000, 60*1000);
112    tmp[0].start();
113
114    zk = newZK(conf, "server2", new MockAbortable());
115    tmp[1] = new AuthenticationTokenSecretManagerForTest(
116        conf, zk, "server2", 60*60*1000, 60*1000);
117    tmp[1].start();
118
119    while (KEY_MASTER == null) {
120      for (int i=0; i<2; i++) {
121        if (tmp[i].isMaster()) {
122          KEY_MASTER = tmp[i];
123          KEY_SLAVE = tmp[ (i+1) % 2 ];
124          break;
125        }
126      }
127      Thread.sleep(500);
128    }
129    LOG.info("Master is "+KEY_MASTER.getName()+
130        ", slave is "+KEY_SLAVE.getName());
131  }
132
133  @AfterClass
134  public static void tearDownAfterClass() throws Exception {
135    TEST_UTIL.shutdownMiniZKCluster();
136  }
137
138  @Test
139  public void testKeyUpdate() throws Exception {
140    // sanity check
141    assertTrue(KEY_MASTER.isMaster());
142    assertFalse(KEY_SLAVE.isMaster());
143    int maxKeyId = 0;
144
145    KEY_MASTER.rollCurrentKey();
146    AuthenticationKey key1 = KEY_MASTER.getCurrentKey();
147    assertNotNull(key1);
148    LOG.debug("Master current key: "+key1.getKeyId());
149
150    // wait for slave to update
151    Thread.sleep(1000);
152    AuthenticationKey slaveCurrent = KEY_SLAVE.getCurrentKey();
153    assertNotNull(slaveCurrent);
154    assertEquals(key1, slaveCurrent);
155    LOG.debug("Slave current key: "+slaveCurrent.getKeyId());
156
157    // generate two more keys then expire the original
158    KEY_MASTER.rollCurrentKey();
159    AuthenticationKey key2 = KEY_MASTER.getCurrentKey();
160    LOG.debug("Master new current key: "+key2.getKeyId());
161    KEY_MASTER.rollCurrentKey();
162    AuthenticationKey key3 = KEY_MASTER.getCurrentKey();
163    LOG.debug("Master new current key: "+key3.getKeyId());
164
165    // force expire the original key
166    key1.setExpiration(EnvironmentEdgeManager.currentTime() - 1000);
167    KEY_MASTER.removeExpiredKeys();
168    // verify removed from master
169    assertNull(KEY_MASTER.getKey(key1.getKeyId()));
170
171    // wait for slave to catch up
172    KEY_SLAVE.getLatch().await();
173    // make sure the slave has both new keys
174    AuthenticationKey slave2 = KEY_SLAVE.getKey(key2.getKeyId());
175    assertNotNull(slave2);
176    assertEquals(key2, slave2);
177    AuthenticationKey slave3 = KEY_SLAVE.getKey(key3.getKeyId());
178    assertNotNull(slave3);
179    assertEquals(key3, slave3);
180    slaveCurrent = KEY_SLAVE.getCurrentKey();
181    assertEquals(key3, slaveCurrent);
182    LOG.debug("Slave current key: "+slaveCurrent.getKeyId());
183
184    // verify that the expired key has been removed
185    assertNull(KEY_SLAVE.getKey(key1.getKeyId()));
186
187    // bring up a new slave
188    Configuration conf = TEST_UTIL.getConfiguration();
189    ZKWatcher zk = newZK(conf, "server3", new MockAbortable());
190    KEY_SLAVE2 = new AuthenticationTokenSecretManager(
191        conf, zk, "server3", 60*60*1000, 60*1000);
192    KEY_SLAVE2.start();
193
194    Thread.sleep(1000);
195    // verify the new slave has current keys (and not expired)
196    slave2 = KEY_SLAVE2.getKey(key2.getKeyId());
197    assertNotNull(slave2);
198    assertEquals(key2, slave2);
199    slave3 = KEY_SLAVE2.getKey(key3.getKeyId());
200    assertNotNull(slave3);
201    assertEquals(key3, slave3);
202    slaveCurrent = KEY_SLAVE2.getCurrentKey();
203    assertEquals(key3, slaveCurrent);
204    assertNull(KEY_SLAVE2.getKey(key1.getKeyId()));
205
206    // test leader failover
207    KEY_MASTER.stop();
208
209    // wait for master to stop
210    Thread.sleep(1000);
211    assertFalse(KEY_MASTER.isMaster());
212
213    // check for a new master
214    AuthenticationTokenSecretManager[] mgrs =
215        new AuthenticationTokenSecretManager[]{ KEY_SLAVE, KEY_SLAVE2 };
216    AuthenticationTokenSecretManager newMaster = null;
217    int tries = 0;
218    while (newMaster == null && tries++ < 5) {
219      for (AuthenticationTokenSecretManager mgr : mgrs) {
220        if (mgr.isMaster()) {
221          newMaster = mgr;
222          break;
223        }
224      }
225      if (newMaster == null) {
226        Thread.sleep(500);
227      }
228    }
229    assertNotNull(newMaster);
230
231    AuthenticationKey current = newMaster.getCurrentKey();
232    // new master will immediately roll the current key, so it's current may be greater
233    assertTrue(current.getKeyId() >= slaveCurrent.getKeyId());
234    LOG.debug("New master, current key: "+current.getKeyId());
235
236    // roll the current key again on new master and verify the key ID increments
237    newMaster.rollCurrentKey();
238    AuthenticationKey newCurrent = newMaster.getCurrentKey();
239    LOG.debug("New master, rolled new current key: "+newCurrent.getKeyId());
240    assertTrue(newCurrent.getKeyId() > current.getKeyId());
241
242    // add another slave
243    ZKWatcher zk3 = newZK(conf, "server4", new MockAbortable());
244    KEY_SLAVE3 = new AuthenticationTokenSecretManager(
245        conf, zk3, "server4", 60*60*1000, 60*1000);
246    KEY_SLAVE3.start();
247    Thread.sleep(5000);
248
249    // check master failover again
250    newMaster.stop();
251
252    // wait for master to stop
253    Thread.sleep(5000);
254    assertFalse(newMaster.isMaster());
255
256    // check for a new master
257    mgrs = new AuthenticationTokenSecretManager[]{ KEY_SLAVE, KEY_SLAVE2, KEY_SLAVE3 };
258    newMaster = null;
259    tries = 0;
260    while (newMaster == null && tries++ < 5) {
261      for (AuthenticationTokenSecretManager mgr : mgrs) {
262        if (mgr.isMaster()) {
263          newMaster = mgr;
264          break;
265        }
266      }
267      if (newMaster == null) {
268        Thread.sleep(500);
269      }
270    }
271    assertNotNull(newMaster);
272
273    AuthenticationKey current2 = newMaster.getCurrentKey();
274    // new master will immediately roll the current key, so it's current may be greater
275    assertTrue(current2.getKeyId() >= newCurrent.getKeyId());
276    LOG.debug("New master 2, current key: "+current2.getKeyId());
277
278    // roll the current key again on new master and verify the key ID increments
279    newMaster.rollCurrentKey();
280    AuthenticationKey newCurrent2 = newMaster.getCurrentKey();
281    LOG.debug("New master 2, rolled new current key: "+newCurrent2.getKeyId());
282    assertTrue(newCurrent2.getKeyId() > current2.getKeyId());
283  }
284
285  private static ZKWatcher newZK(Configuration conf, String name,
286                                 Abortable abort) throws Exception {
287    Configuration copy = HBaseConfiguration.create(conf);
288    ZKWatcher zk = new ZKWatcher(copy, name, abort);
289    return zk;
290  }
291}