001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.coprocessor;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023import static org.junit.Assert.fail;
024
025import java.io.IOException;
026import java.util.Optional;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.Abortable;
029import org.apache.hadoop.hbase.CoprocessorEnvironment;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.HBaseTestingUtil;
032import org.apache.hadoop.hbase.HConstants;
033import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.client.Admin;
036import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
037import org.apache.hadoop.hbase.client.RegionInfo;
038import org.apache.hadoop.hbase.client.TableDescriptor;
039import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
040import org.apache.hadoop.hbase.master.HMaster;
041import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
042import org.apache.hadoop.hbase.testclassification.CoprocessorTests;
043import org.apache.hadoop.hbase.testclassification.MediumTests;
044import org.apache.hadoop.hbase.util.Bytes;
045import org.apache.hadoop.hbase.zookeeper.ZKNodeTracker;
046import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
047import org.junit.AfterClass;
048import org.junit.BeforeClass;
049import org.junit.ClassRule;
050import org.junit.Test;
051import org.junit.experimental.categories.Category;
052
053/**
054 * Tests unhandled exceptions thrown by coprocessors running on master.
055 * Expected result is that the master will abort with an informative
056 * error message describing the set of its loaded coprocessors for crash diagnosis.
057 * (HBASE-4014).
058 */
059@Category({CoprocessorTests.class, MediumTests.class})
060public class TestMasterCoprocessorExceptionWithAbort {
061
062  @ClassRule
063  public static final HBaseClassTestRule CLASS_RULE =
064      HBaseClassTestRule.forClass(TestMasterCoprocessorExceptionWithAbort.class);
065
066  public static class MasterTracker extends ZKNodeTracker {
067    public boolean masterZKNodeWasDeleted = false;
068
069    public MasterTracker(ZKWatcher zkw, String masterNode, Abortable abortable) {
070      super(zkw, masterNode, abortable);
071    }
072
073    @Override
074    public synchronized void nodeDeleted(String path) {
075      if (path.equals("/hbase/master")) {
076        masterZKNodeWasDeleted = true;
077      }
078    }
079  }
080
081  public static class CreateTableThread extends Thread {
082    HBaseTestingUtil UTIL;
083    public CreateTableThread(HBaseTestingUtil UTIL) {
084      this.UTIL = UTIL;
085    }
086
087    @Override
088    public void run() {
089      // create a table : master coprocessor will throw an exception and not
090      // catch it.
091      TableDescriptor tableDescriptor =
092        TableDescriptorBuilder.newBuilder(TableName.valueOf(TEST_TABLE))
093          .setColumnFamily(ColumnFamilyDescriptorBuilder.of(TEST_FAMILY)).build();
094      try {
095        Admin admin = UTIL.getAdmin();
096        admin.createTable(tableDescriptor);
097        fail("BuggyMasterObserver failed to throw an exception.");
098      } catch (IOException e) {
099        assertEquals("HBaseAdmin threw an interrupted IOException as expected.",
100            "java.io.InterruptedIOException", e.getClass().getName());
101      }
102   }
103  }
104
105  public static class BuggyMasterObserver implements MasterCoprocessor, MasterObserver {
106    private boolean preCreateTableCalled;
107    private boolean postCreateTableCalled;
108    private boolean startCalled;
109    private boolean postStartMasterCalled;
110
111    @Override
112    public Optional<MasterObserver> getMasterObserver() {
113      return Optional.of(this);
114    }
115
116    @Override
117    public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> env,
118        TableDescriptor desc, RegionInfo[] regions) throws IOException {
119      // cause a NullPointerException and don't catch it: this will cause the
120      // master to abort().
121      Integer i;
122      i = null;
123      i = i++;
124    }
125
126    public boolean wasCreateTableCalled() {
127      return preCreateTableCalled && postCreateTableCalled;
128    }
129
130    @Override
131    public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx)
132        throws IOException {
133      postStartMasterCalled = true;
134    }
135
136    public boolean wasStartMasterCalled() {
137      return postStartMasterCalled;
138    }
139
140    @Override
141    public void start(CoprocessorEnvironment env) throws IOException {
142      startCalled = true;
143    }
144
145    public boolean wasStarted() {
146      return startCalled;
147    }
148  }
149
150  private static HBaseTestingUtil UTIL = new HBaseTestingUtil();
151  private static byte[] TEST_TABLE = Bytes.toBytes("observed_table");
152  private static byte[] TEST_FAMILY = Bytes.toBytes("fam1");
153
154  @BeforeClass
155  public static void setupBeforeClass() throws Exception {
156    Configuration conf = UTIL.getConfiguration();
157    conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
158        BuggyMasterObserver.class.getName());
159    conf.setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, true);
160    conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);  // Fail fast
161    UTIL.startMiniCluster();
162  }
163
164  @AfterClass
165  public static void teardownAfterClass() throws Exception {
166    UTIL.shutdownMiniCluster();
167  }
168
169  @Test
170  public void testExceptionFromCoprocessorWhenCreatingTable()
171      throws IOException {
172    SingleProcessHBaseCluster cluster = UTIL.getHBaseCluster();
173
174    HMaster master = cluster.getMaster();
175    MasterCoprocessorHost host = master.getMasterCoprocessorHost();
176    BuggyMasterObserver cp = host.findCoprocessor(BuggyMasterObserver.class);
177    assertFalse("No table created yet", cp.wasCreateTableCalled());
178
179    // set a watch on the zookeeper /hbase/master node. If the master dies,
180    // the node will be deleted.
181    ZKWatcher zkw = new ZKWatcher(UTIL.getConfiguration(),
182      "unittest", new Abortable() {
183      @Override
184      public void abort(String why, Throwable e) {
185        throw new RuntimeException("Fatal ZK error: " + why, e);
186      }
187      @Override
188      public boolean isAborted() {
189        return false;
190      }
191    });
192
193    MasterTracker masterTracker = new MasterTracker(zkw,"/hbase/master",
194        new Abortable() {
195          @Override
196          public void abort(String why, Throwable e) {
197            throw new RuntimeException("Fatal ZK master tracker error, why=", e);
198          }
199          @Override
200          public boolean isAborted() {
201            return false;
202          }
203        });
204
205    masterTracker.start();
206    zkw.registerListener(masterTracker);
207
208    // Test (part of the) output that should have be printed by master when it aborts:
209    // (namely the part that shows the set of loaded coprocessors).
210    // In this test, there is only a single coprocessor (BuggyMasterObserver).
211    assertTrue(HMaster.getLoadedCoprocessors().
212      contains(TestMasterCoprocessorExceptionWithAbort.BuggyMasterObserver.class.getName()));
213
214    CreateTableThread createTableThread = new CreateTableThread(UTIL);
215
216    // Attempting to create a table (using createTableThread above) triggers an NPE in BuggyMasterObserver.
217    // Master will then abort and the /hbase/master zk node will be deleted.
218    createTableThread.start();
219
220    // Wait up to 30 seconds for master's /hbase/master zk node to go away after master aborts.
221    for (int i = 0; i < 30; i++) {
222      if (masterTracker.masterZKNodeWasDeleted == true) {
223        break;
224      }
225      try {
226        Thread.sleep(1000);
227      } catch (InterruptedException e) {
228        fail("InterruptedException while waiting for master zk node to "
229            + "be deleted.");
230      }
231    }
232
233    assertTrue("Master aborted on coprocessor exception, as expected.",
234        masterTracker.masterZKNodeWasDeleted);
235
236    createTableThread.interrupt();
237    try {
238      createTableThread.join(1000);
239    } catch (InterruptedException e) {
240      assertTrue("Ignoring InterruptedException while waiting for " +
241          " createTableThread.join().", true);
242    }
243  }
244
245}
246