001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.security.token;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022import static org.junit.jupiter.api.Assertions.assertNotNull;
023import static org.junit.jupiter.api.Assertions.assertNull;
024import static org.junit.jupiter.api.Assertions.assertTrue;
025
026import java.util.concurrent.CountDownLatch;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.Abortable;
029import org.apache.hadoop.hbase.HBaseConfiguration;
030import org.apache.hadoop.hbase.HBaseTestingUtil;
031import org.apache.hadoop.hbase.Waiter;
032import org.apache.hadoop.hbase.testclassification.SecurityTests;
033import org.apache.hadoop.hbase.testclassification.SmallTests;
034import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
035import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
036import org.junit.jupiter.api.AfterAll;
037import org.junit.jupiter.api.BeforeAll;
038import org.junit.jupiter.api.Tag;
039import org.junit.jupiter.api.Test;
040import org.slf4j.Logger;
041import org.slf4j.LoggerFactory;
042
043/**
044 * Test the synchronization of token authentication master keys through ZKSecretWatcher
045 */
046@Tag(SecurityTests.TAG)
047@Tag(SmallTests.TAG)
048public class TestZKSecretWatcher {
049
050  private static final Logger LOG = LoggerFactory.getLogger(TestZKSecretWatcher.class);
051  private static HBaseTestingUtil TEST_UTIL;
052  private static AuthenticationTokenSecretManager KEY_MASTER;
053  private static AuthenticationTokenSecretManagerForTest KEY_SLAVE;
054  private static AuthenticationTokenSecretManager KEY_SLAVE2;
055  private static AuthenticationTokenSecretManager KEY_SLAVE3;
056
057  private static class MockAbortable implements Abortable {
058    private boolean abort;
059
060    @Override
061    public void abort(String reason, Throwable e) {
062      LOG.info("Aborting: " + reason, e);
063      abort = true;
064    }
065
066    @Override
067    public boolean isAborted() {
068      return abort;
069    }
070  }
071
072  // We subclass AuthenticationTokenSecretManager so that testKeyUpdate can receive
073  // notification on the removal of keyId
074  private static class AuthenticationTokenSecretManagerForTest
075    extends AuthenticationTokenSecretManager {
076    private CountDownLatch latch = new CountDownLatch(1);
077
078    public AuthenticationTokenSecretManagerForTest(Configuration conf, ZKWatcher zk,
079      String serverName, long keyUpdateInterval, long tokenMaxLifetime) {
080      super(conf, zk, serverName, keyUpdateInterval, tokenMaxLifetime);
081    }
082
083    @Override
084    synchronized boolean removeKey(Integer keyId) {
085      boolean b = super.removeKey(keyId);
086      if (b) {
087        latch.countDown();
088      }
089      return b;
090    }
091
092    CountDownLatch getLatch() {
093      return latch;
094    }
095  }
096
097  @BeforeAll
098  public static void setupBeforeClass() throws Exception {
099    TEST_UTIL = new HBaseTestingUtil();
100    TEST_UTIL.startMiniZKCluster();
101    Configuration conf = TEST_UTIL.getConfiguration();
102
103    ZKWatcher zk = newZK(conf, "server1", new MockAbortable());
104    AuthenticationTokenSecretManagerForTest[] tmp = new AuthenticationTokenSecretManagerForTest[2];
105    tmp[0] =
106      new AuthenticationTokenSecretManagerForTest(conf, zk, "server1", 60 * 60 * 1000, 60 * 1000);
107    tmp[0].start();
108
109    zk = newZK(conf, "server2", new MockAbortable());
110    tmp[1] =
111      new AuthenticationTokenSecretManagerForTest(conf, zk, "server2", 60 * 60 * 1000, 60 * 1000);
112    tmp[1].start();
113
114    while (KEY_MASTER == null) {
115      for (int i = 0; i < 2; i++) {
116        if (tmp[i].isMaster()) {
117          KEY_MASTER = tmp[i];
118          KEY_SLAVE = tmp[(i + 1) % 2];
119          break;
120        }
121      }
122      Thread.sleep(500);
123    }
124    LOG.info("Master is " + KEY_MASTER.getName() + ", slave is " + KEY_SLAVE.getName());
125  }
126
127  @AfterAll
128  public static void tearDownAfterClass() throws Exception {
129    TEST_UTIL.shutdownMiniZKCluster();
130  }
131
132  @Test
133  public void testKeyUpdate() throws Exception {
134    // sanity check
135    assertTrue(KEY_MASTER.isMaster());
136    assertFalse(KEY_SLAVE.isMaster());
137
138    KEY_MASTER.rollCurrentKey();
139    AuthenticationKey key1 = KEY_MASTER.getCurrentKey();
140    assertNotNull(key1);
141    LOG.debug("Master current key (key1) {}", key1);
142
143    // wait for slave to update
144    Thread.sleep(1000);
145    AuthenticationKey slaveCurrent = KEY_SLAVE.getCurrentKey();
146    assertNotNull(slaveCurrent);
147    assertEquals(key1, slaveCurrent);
148    LOG.debug("Slave current key (key1) {}", slaveCurrent);
149
150    // generate two more keys then expire the original
151    KEY_MASTER.rollCurrentKey();
152    AuthenticationKey key2 = KEY_MASTER.getCurrentKey();
153    LOG.debug("Master new current key (key2) {}", key2);
154    KEY_MASTER.rollCurrentKey();
155    AuthenticationKey key3 = KEY_MASTER.getCurrentKey();
156    LOG.debug("Master new current key (key3) {}", key3);
157
158    // force expire the original key
159    key1.setExpiration(EnvironmentEdgeManager.currentTime() - 100000);
160    KEY_MASTER.removeExpiredKeys();
161    // verify removed from master
162    assertNull(KEY_MASTER.getKey(key1.getKeyId()));
163
164    // Wait for slave to catch up. When remove hits KEY_SLAVE, we'll clear
165    // the latch and will progress beyond the await.
166    KEY_SLAVE.getLatch().await();
167    // make sure the slave has both new keys
168    AuthenticationKey slave2 = KEY_SLAVE.getKey(key2.getKeyId());
169    assertNotNull(slave2);
170    assertEquals(key2, slave2);
171    AuthenticationKey slave3 = KEY_SLAVE.getKey(key3.getKeyId());
172    assertNotNull(slave3);
173    assertEquals(key3, slave3);
174    slaveCurrent = KEY_SLAVE.getCurrentKey();
175    assertEquals(key3, slaveCurrent);
176    LOG.debug("Slave current key (key3) {}", slaveCurrent);
177
178    // verify that the expired key has been removed
179    Waiter.waitFor(TEST_UTIL.getConfiguration(), 30000, () -> {
180      AuthenticationKey k = KEY_SLAVE.getKey(key1.getKeyId());
181      LOG.info("AuthKey1={}", k);
182      return k == null;
183    });
184    assertNull(KEY_SLAVE.getKey(key1.getKeyId()), "key1=" + KEY_SLAVE.getKey(key1.getKeyId()));
185
186    // bring up a new slave
187    Configuration conf = TEST_UTIL.getConfiguration();
188    ZKWatcher zk = newZK(conf, "server3", new MockAbortable());
189    KEY_SLAVE2 =
190      new AuthenticationTokenSecretManager(conf, zk, "server3", 60 * 60 * 1000, 60 * 1000);
191    KEY_SLAVE2.start();
192
193    Thread.sleep(1000);
194    // verify the new slave has current keys (and not expired)
195    slave2 = KEY_SLAVE2.getKey(key2.getKeyId());
196    assertNotNull(slave2);
197    assertEquals(key2, slave2);
198    slave3 = KEY_SLAVE2.getKey(key3.getKeyId());
199    assertNotNull(slave3);
200    assertEquals(key3, slave3);
201    slaveCurrent = KEY_SLAVE2.getCurrentKey();
202    assertEquals(key3, slaveCurrent);
203    assertNull(KEY_SLAVE2.getKey(key1.getKeyId()));
204
205    // test leader failover
206    KEY_MASTER.stop();
207
208    // wait for master to stop
209    Thread.sleep(1000);
210    assertFalse(KEY_MASTER.isMaster());
211
212    // check for a new master
213    AuthenticationTokenSecretManager[] mgrs =
214      new AuthenticationTokenSecretManager[] { KEY_SLAVE, KEY_SLAVE2 };
215    AuthenticationTokenSecretManager newMaster = null;
216    int tries = 0;
217    while (newMaster == null && tries++ < 5) {
218      for (AuthenticationTokenSecretManager mgr : mgrs) {
219        if (mgr.isMaster()) {
220          newMaster = mgr;
221          break;
222        }
223      }
224      if (newMaster == null) {
225        Thread.sleep(500);
226      }
227    }
228    assertNotNull(newMaster);
229
230    AuthenticationKey current = newMaster.getCurrentKey();
231    // new master will immediately roll the current key, so it's current may be greater
232    assertTrue(current.getKeyId() >= slaveCurrent.getKeyId());
233    LOG.debug("New master, current key: " + current.getKeyId());
234
235    // roll the current key again on new master and verify the key ID increments
236    newMaster.rollCurrentKey();
237    AuthenticationKey newCurrent = newMaster.getCurrentKey();
238    LOG.debug("New master, rolled new current key: " + newCurrent.getKeyId());
239    assertTrue(newCurrent.getKeyId() > current.getKeyId());
240
241    // add another slave
242    ZKWatcher zk3 = newZK(conf, "server4", new MockAbortable());
243    KEY_SLAVE3 =
244      new AuthenticationTokenSecretManager(conf, zk3, "server4", 60 * 60 * 1000, 60 * 1000);
245    KEY_SLAVE3.start();
246    Thread.sleep(5000);
247
248    // check master failover again
249    newMaster.stop();
250
251    // wait for master to stop
252    Thread.sleep(5000);
253    assertFalse(newMaster.isMaster());
254
255    // check for a new master
256    mgrs = new AuthenticationTokenSecretManager[] { KEY_SLAVE, KEY_SLAVE2, KEY_SLAVE3 };
257    newMaster = null;
258    tries = 0;
259    while (newMaster == null && tries++ < 5) {
260      for (AuthenticationTokenSecretManager mgr : mgrs) {
261        if (mgr.isMaster()) {
262          newMaster = mgr;
263          break;
264        }
265      }
266      if (newMaster == null) {
267        Thread.sleep(500);
268      }
269    }
270    assertNotNull(newMaster);
271
272    AuthenticationKey current2 = newMaster.getCurrentKey();
273    // new master will immediately roll the current key, so it's current may be greater
274    assertTrue(current2.getKeyId() >= newCurrent.getKeyId());
275    LOG.debug("New master 2, current key: " + current2.getKeyId());
276
277    // roll the current key again on new master and verify the key ID increments
278    newMaster.rollCurrentKey();
279    AuthenticationKey newCurrent2 = newMaster.getCurrentKey();
280    LOG.debug("New master 2, rolled new current key: " + newCurrent2.getKeyId());
281    assertTrue(newCurrent2.getKeyId() > current2.getKeyId());
282  }
283
284  private static ZKWatcher newZK(Configuration conf, String name, Abortable abort)
285    throws Exception {
286    Configuration copy = HBaseConfiguration.create(conf);
287    ZKWatcher zk = new ZKWatcher(copy, name, abort);
288    return zk;
289  }
290}