001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.security.token;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertNull;
024import static org.junit.Assert.assertTrue;
025
026import java.util.concurrent.CountDownLatch;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.Abortable;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseConfiguration;
031import org.apache.hadoop.hbase.HBaseTestingUtility;
032import org.apache.hadoop.hbase.Waiter;
033import org.apache.hadoop.hbase.testclassification.MediumTests;
034import org.apache.hadoop.hbase.testclassification.SecurityTests;
035import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
036import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
037import org.junit.AfterClass;
038import org.junit.BeforeClass;
039import org.junit.ClassRule;
040import org.junit.Test;
041import org.junit.experimental.categories.Category;
042import org.slf4j.Logger;
043import org.slf4j.LoggerFactory;
044
045/**
046 * Test the synchronization of token authentication master keys through ZKSecretWatcher
047 */
048@Category({ SecurityTests.class, MediumTests.class })
049public class TestZKSecretWatcher {
050
051  @ClassRule
052  public static final HBaseClassTestRule CLASS_RULE =
053    HBaseClassTestRule.forClass(TestZKSecretWatcher.class);
054
055  private static final Logger LOG = LoggerFactory.getLogger(TestZKSecretWatcher.class);
056  private static HBaseTestingUtility TEST_UTIL;
057  private static AuthenticationTokenSecretManager KEY_MASTER;
058  private static AuthenticationTokenSecretManagerForTest KEY_SLAVE;
059  private static AuthenticationTokenSecretManager KEY_SLAVE2;
060  private static AuthenticationTokenSecretManager KEY_SLAVE3;
061
062  private static class MockAbortable implements Abortable {
063    private boolean abort;
064
065    @Override
066    public void abort(String reason, Throwable e) {
067      LOG.info("Aborting: " + reason, e);
068      abort = true;
069    }
070
071    @Override
072    public boolean isAborted() {
073      return abort;
074    }
075  }
076
077  // We subclass AuthenticationTokenSecretManager so that testKeyUpdate can receive
078  // notification on the removal of keyId
079  private static class AuthenticationTokenSecretManagerForTest
080    extends AuthenticationTokenSecretManager {
081    private CountDownLatch latch = new CountDownLatch(1);
082
083    public AuthenticationTokenSecretManagerForTest(Configuration conf, ZKWatcher zk,
084      String serverName, long keyUpdateInterval, long tokenMaxLifetime) {
085      super(conf, zk, serverName, keyUpdateInterval, tokenMaxLifetime);
086    }
087
088    @Override
089    synchronized boolean removeKey(Integer keyId) {
090      boolean b = super.removeKey(keyId);
091      if (b) {
092        latch.countDown();
093      }
094      return b;
095    }
096
097    CountDownLatch getLatch() {
098      return latch;
099    }
100  }
101
102  @BeforeClass
103  public static void setupBeforeClass() throws Exception {
104    TEST_UTIL = new HBaseTestingUtility();
105    TEST_UTIL.startMiniZKCluster();
106    Configuration conf = TEST_UTIL.getConfiguration();
107
108    ZKWatcher zk = newZK(conf, "server1", new MockAbortable());
109    AuthenticationTokenSecretManagerForTest[] tmp = new AuthenticationTokenSecretManagerForTest[2];
110    tmp[0] =
111      new AuthenticationTokenSecretManagerForTest(conf, zk, "server1", 60 * 60 * 1000, 60 * 1000);
112    tmp[0].start();
113
114    zk = newZK(conf, "server2", new MockAbortable());
115    tmp[1] =
116      new AuthenticationTokenSecretManagerForTest(conf, zk, "server2", 60 * 60 * 1000, 60 * 1000);
117    tmp[1].start();
118
119    while (KEY_MASTER == null) {
120      for (int i = 0; i < 2; i++) {
121        if (tmp[i].isMaster()) {
122          KEY_MASTER = tmp[i];
123          KEY_SLAVE = tmp[(i + 1) % 2];
124          break;
125        }
126      }
127      Thread.sleep(500);
128    }
129    LOG.info("Master is " + KEY_MASTER.getName() + ", slave is " + KEY_SLAVE.getName());
130  }
131
132  @AfterClass
133  public static void tearDownAfterClass() throws Exception {
134    TEST_UTIL.shutdownMiniZKCluster();
135  }
136
137  @Test
138  public void testKeyUpdate() throws Exception {
139    // sanity check
140    assertTrue(KEY_MASTER.isMaster());
141    assertFalse(KEY_SLAVE.isMaster());
142    int maxKeyId = 0;
143
144    KEY_MASTER.rollCurrentKey();
145    AuthenticationKey key1 = KEY_MASTER.getCurrentKey();
146    assertNotNull(key1);
147    LOG.debug("Master current key (key1) {}", key1);
148
149    // wait for slave to update
150    Thread.sleep(1000);
151    AuthenticationKey slaveCurrent = KEY_SLAVE.getCurrentKey();
152    assertNotNull(slaveCurrent);
153    assertEquals(key1, slaveCurrent);
154    LOG.debug("Slave current key (key1) {}", slaveCurrent);
155
156    // generate two more keys then expire the original
157    KEY_MASTER.rollCurrentKey();
158    AuthenticationKey key2 = KEY_MASTER.getCurrentKey();
159    LOG.debug("Master new current key (key2) {}", key2);
160    KEY_MASTER.rollCurrentKey();
161    AuthenticationKey key3 = KEY_MASTER.getCurrentKey();
162    LOG.debug("Master new current key (key3) {}", key3);
163
164    // force expire the original key
165    key1.setExpiration(EnvironmentEdgeManager.currentTime() - 100000);
166    KEY_MASTER.removeExpiredKeys();
167    // verify removed from master
168    assertNull(KEY_MASTER.getKey(key1.getKeyId()));
169
170    // Wait for slave to catch up. When remove hits KEY_SLAVE, we'll clear
171    // the latch and will progress beyond the await.
172    KEY_SLAVE.getLatch().await();
173    // make sure the slave has both new keys
174    AuthenticationKey slave2 = KEY_SLAVE.getKey(key2.getKeyId());
175    assertNotNull(slave2);
176    assertEquals(key2, slave2);
177    AuthenticationKey slave3 = KEY_SLAVE.getKey(key3.getKeyId());
178    assertNotNull(slave3);
179    assertEquals(key3, slave3);
180    slaveCurrent = KEY_SLAVE.getCurrentKey();
181    assertEquals(key3, slaveCurrent);
182    LOG.debug("Slave current key (key3) {}", slaveCurrent);
183
184    // verify that the expired key has been removed
185    Waiter.waitFor(TEST_UTIL.getConfiguration(), 30000, () -> {
186      AuthenticationKey k = KEY_SLAVE.getKey(key1.getKeyId());
187      LOG.info("AuthKey1={}", k);
188      return k == null;
189    });
190    assertNull("key1=" + KEY_SLAVE.getKey(key1.getKeyId()), KEY_SLAVE.getKey(key1.getKeyId()));
191
192    // bring up a new slave
193    Configuration conf = TEST_UTIL.getConfiguration();
194    ZKWatcher zk = newZK(conf, "server3", new MockAbortable());
195    KEY_SLAVE2 =
196      new AuthenticationTokenSecretManager(conf, zk, "server3", 60 * 60 * 1000, 60 * 1000);
197    KEY_SLAVE2.start();
198
199    Thread.sleep(1000);
200    // verify the new slave has current keys (and not expired)
201    slave2 = KEY_SLAVE2.getKey(key2.getKeyId());
202    assertNotNull(slave2);
203    assertEquals(key2, slave2);
204    slave3 = KEY_SLAVE2.getKey(key3.getKeyId());
205    assertNotNull(slave3);
206    assertEquals(key3, slave3);
207    slaveCurrent = KEY_SLAVE2.getCurrentKey();
208    assertEquals(key3, slaveCurrent);
209    assertNull(KEY_SLAVE2.getKey(key1.getKeyId()));
210
211    // test leader failover
212    KEY_MASTER.stop();
213
214    // wait for master to stop
215    Thread.sleep(1000);
216    assertFalse(KEY_MASTER.isMaster());
217
218    // check for a new master
219    AuthenticationTokenSecretManager[] mgrs =
220      new AuthenticationTokenSecretManager[] { KEY_SLAVE, KEY_SLAVE2 };
221    AuthenticationTokenSecretManager newMaster = null;
222    int tries = 0;
223    while (newMaster == null && tries++ < 5) {
224      for (AuthenticationTokenSecretManager mgr : mgrs) {
225        if (mgr.isMaster()) {
226          newMaster = mgr;
227          break;
228        }
229      }
230      if (newMaster == null) {
231        Thread.sleep(500);
232      }
233    }
234    assertNotNull(newMaster);
235
236    AuthenticationKey current = newMaster.getCurrentKey();
237    // new master will immediately roll the current key, so it's current may be greater
238    assertTrue(current.getKeyId() >= slaveCurrent.getKeyId());
239    LOG.debug("New master, current key: " + current.getKeyId());
240
241    // roll the current key again on new master and verify the key ID increments
242    newMaster.rollCurrentKey();
243    AuthenticationKey newCurrent = newMaster.getCurrentKey();
244    LOG.debug("New master, rolled new current key: " + newCurrent.getKeyId());
245    assertTrue(newCurrent.getKeyId() > current.getKeyId());
246
247    // add another slave
248    ZKWatcher zk3 = newZK(conf, "server4", new MockAbortable());
249    KEY_SLAVE3 =
250      new AuthenticationTokenSecretManager(conf, zk3, "server4", 60 * 60 * 1000, 60 * 1000);
251    KEY_SLAVE3.start();
252    Thread.sleep(5000);
253
254    // check master failover again
255    newMaster.stop();
256
257    // wait for master to stop
258    Thread.sleep(5000);
259    assertFalse(newMaster.isMaster());
260
261    // check for a new master
262    mgrs = new AuthenticationTokenSecretManager[] { KEY_SLAVE, KEY_SLAVE2, KEY_SLAVE3 };
263    newMaster = null;
264    tries = 0;
265    while (newMaster == null && tries++ < 5) {
266      for (AuthenticationTokenSecretManager mgr : mgrs) {
267        if (mgr.isMaster()) {
268          newMaster = mgr;
269          break;
270        }
271      }
272      if (newMaster == null) {
273        Thread.sleep(500);
274      }
275    }
276    assertNotNull(newMaster);
277
278    AuthenticationKey current2 = newMaster.getCurrentKey();
279    // new master will immediately roll the current key, so it's current may be greater
280    assertTrue(current2.getKeyId() >= newCurrent.getKeyId());
281    LOG.debug("New master 2, current key: " + current2.getKeyId());
282
283    // roll the current key again on new master and verify the key ID increments
284    newMaster.rollCurrentKey();
285    AuthenticationKey newCurrent2 = newMaster.getCurrentKey();
286    LOG.debug("New master 2, rolled new current key: " + newCurrent2.getKeyId());
287    assertTrue(newCurrent2.getKeyId() > current2.getKeyId());
288  }
289
290  private static ZKWatcher newZK(Configuration conf, String name, Abortable abort)
291    throws Exception {
292    Configuration copy = HBaseConfiguration.create(conf);
293    ZKWatcher zk = new ZKWatcher(copy, name, abort);
294    return zk;
295  }
296}