001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.coprocessor;
019
020import static org.junit.Assert.assertFalse;
021import static org.junit.Assert.assertTrue;
022import static org.junit.Assert.fail;
023
024import java.io.IOException;
025import java.util.Optional;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.hbase.Abortable;
028import org.apache.hadoop.hbase.CoprocessorEnvironment;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseTestingUtility;
031import org.apache.hadoop.hbase.HColumnDescriptor;
032import org.apache.hadoop.hbase.HTableDescriptor;
033import org.apache.hadoop.hbase.MiniHBaseCluster;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.client.Admin;
036import org.apache.hadoop.hbase.client.RegionInfo;
037import org.apache.hadoop.hbase.client.TableDescriptor;
038import org.apache.hadoop.hbase.master.HMaster;
039import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
040import org.apache.hadoop.hbase.testclassification.CoprocessorTests;
041import org.apache.hadoop.hbase.testclassification.MediumTests;
042import org.apache.hadoop.hbase.util.Bytes;
043import org.apache.hadoop.hbase.zookeeper.ZKNodeTracker;
044import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
045import org.junit.AfterClass;
046import org.junit.BeforeClass;
047import org.junit.ClassRule;
048import org.junit.Test;
049import org.junit.experimental.categories.Category;
050
051/**
052 * Tests unhandled exceptions thrown by coprocessors running on master.
053 * Expected result is that the master will remove the buggy coprocessor from
054 * its set of coprocessors and throw a org.apache.hadoop.hbase.exceptions.DoNotRetryIOException
055 * back to the client.
056 * (HBASE-4014).
057 */
058@Category({CoprocessorTests.class, MediumTests.class})
059public class TestMasterCoprocessorExceptionWithRemove {
060
061  @ClassRule
062  public static final HBaseClassTestRule CLASS_RULE =
063      HBaseClassTestRule.forClass(TestMasterCoprocessorExceptionWithRemove.class);
064
065  public static class MasterTracker extends ZKNodeTracker {
066    public boolean masterZKNodeWasDeleted = false;
067
068    public MasterTracker(ZKWatcher zkw, String masterNode, Abortable abortable) {
069      super(zkw, masterNode, abortable);
070    }
071
072    @Override
073    public synchronized void nodeDeleted(String path) {
074      if (path.equals("/hbase/master")) {
075        masterZKNodeWasDeleted = true;
076      }
077    }
078  }
079
080  public static class BuggyMasterObserver implements MasterCoprocessor, MasterObserver {
081    private boolean preCreateTableCalled;
082    private boolean postCreateTableCalled;
083    private boolean startCalled;
084    private boolean postStartMasterCalled;
085
086    @Override
087    public Optional<MasterObserver> getMasterObserver() {
088      return Optional.of(this);
089    }
090
091    @SuppressWarnings("null")
092    @Override
093    public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> env,
094        TableDescriptor desc, RegionInfo[] regions) throws IOException {
095      // Cause a NullPointerException and don't catch it: this should cause the
096      // master to throw an o.apache.hadoop.hbase.DoNotRetryIOException to the
097      // client.
098      Integer i;
099      i = null;
100      i = i++;
101    }
102
103    public boolean wasCreateTableCalled() {
104      return preCreateTableCalled && postCreateTableCalled;
105    }
106
107    @Override
108    public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx)
109        throws IOException {
110      postStartMasterCalled = true;
111    }
112
113    public boolean wasStartMasterCalled() {
114      return postStartMasterCalled;
115    }
116
117    @Override
118    public void start(CoprocessorEnvironment env) throws IOException {
119      startCalled = true;
120    }
121
122    public boolean wasStarted() {
123      return startCalled;
124    }
125  }
126
127  private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
128
129  private static byte[] TEST_TABLE1 = Bytes.toBytes("observed_table1");
130  private static byte[] TEST_FAMILY1 = Bytes.toBytes("fam1");
131
132  private static byte[] TEST_TABLE2 = Bytes.toBytes("table2");
133  private static byte[] TEST_FAMILY2 = Bytes.toBytes("fam2");
134
135  @BeforeClass
136  public static void setupBeforeClass() throws Exception {
137    Configuration conf = UTIL.getConfiguration();
138    conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
139        BuggyMasterObserver.class.getName());
140    UTIL.getConfiguration().setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, false);
141    UTIL.startMiniCluster();
142  }
143
144  @AfterClass
145  public static void teardownAfterClass() throws Exception {
146    UTIL.shutdownMiniCluster();
147  }
148
149  @Test
150  public void testExceptionFromCoprocessorWhenCreatingTable()
151      throws IOException {
152    MiniHBaseCluster cluster = UTIL.getHBaseCluster();
153
154    HMaster master = cluster.getMaster();
155    MasterCoprocessorHost host = master.getMasterCoprocessorHost();
156    BuggyMasterObserver cp = host.findCoprocessor(BuggyMasterObserver.class);
157    assertFalse("No table created yet", cp.wasCreateTableCalled());
158
159    // Set a watch on the zookeeper /hbase/master node. If the master dies,
160    // the node will be deleted.
161    // Master should *NOT* die:
162    // we are testing that the default setting of hbase.coprocessor.abortonerror
163    // =false
164    // is respected.
165    ZKWatcher zkw = new ZKWatcher(UTIL.getConfiguration(),
166      "unittest", new Abortable() {
167      @Override
168      public void abort(String why, Throwable e) {
169        throw new RuntimeException("Fatal ZK error: " + why, e);
170      }
171      @Override
172      public boolean isAborted() {
173        return false;
174      }
175    });
176
177    MasterTracker masterTracker = new MasterTracker(zkw,"/hbase/master",
178        new Abortable() {
179          @Override
180          public void abort(String why, Throwable e) {
181            throw new RuntimeException("Fatal ZooKeeper tracker error, why=", e);
182          }
183          @Override
184          public boolean isAborted() {
185            return false;
186          }
187        });
188
189    masterTracker.start();
190    zkw.registerListener(masterTracker);
191
192    // Test (part of the) output that should have be printed by master when it aborts:
193    // (namely the part that shows the set of loaded coprocessors).
194    // In this test, there is only a single coprocessor (BuggyMasterObserver).
195    String coprocessorName =
196        BuggyMasterObserver.class.getName();
197    assertTrue(HMaster.getLoadedCoprocessors().contains(coprocessorName));
198
199    HTableDescriptor htd1 = new HTableDescriptor(TableName.valueOf(TEST_TABLE1));
200    htd1.addFamily(new HColumnDescriptor(TEST_FAMILY1));
201
202    boolean threwDNRE = false;
203    try {
204      Admin admin = UTIL.getAdmin();
205      admin.createTable(htd1);
206    } catch (IOException e) {
207      if (e.getClass().getName().equals("org.apache.hadoop.hbase.DoNotRetryIOException")) {
208        threwDNRE = true;
209      }
210    } finally {
211      assertTrue(threwDNRE);
212    }
213
214    // wait for a few seconds to make sure that the Master hasn't aborted.
215    try {
216      Thread.sleep(3000);
217    } catch (InterruptedException e) {
218      fail("InterruptedException while sleeping.");
219    }
220
221    assertFalse("Master survived coprocessor NPE, as expected.",
222        masterTracker.masterZKNodeWasDeleted);
223
224    String loadedCoprocessors = HMaster.getLoadedCoprocessors();
225    assertTrue(loadedCoprocessors.contains(coprocessorName));
226
227    // Verify that BuggyMasterObserver has been removed due to its misbehavior
228    // by creating another table: should not have a problem this time.
229    HTableDescriptor htd2 = new HTableDescriptor(TableName.valueOf(TEST_TABLE2));
230    htd2.addFamily(new HColumnDescriptor(TEST_FAMILY2));
231    Admin admin = UTIL.getAdmin();
232    try {
233      admin.createTable(htd2);
234    } catch (IOException e) {
235      fail("Failed to create table after buggy coprocessor removal: " + e);
236    }
237  }
238
239}