001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.coprocessor;
019
020import static org.junit.Assert.assertFalse;
021import static org.junit.Assert.assertTrue;
022import static org.junit.Assert.fail;
023
024import java.io.IOException;
025import java.util.Optional;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.hbase.Abortable;
028import org.apache.hadoop.hbase.CoprocessorEnvironment;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseTestingUtil;
031import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.client.Admin;
034import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
035import org.apache.hadoop.hbase.client.RegionInfo;
036import org.apache.hadoop.hbase.client.TableDescriptor;
037import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
038import org.apache.hadoop.hbase.master.HMaster;
039import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
040import org.apache.hadoop.hbase.testclassification.CoprocessorTests;
041import org.apache.hadoop.hbase.testclassification.MediumTests;
042import org.apache.hadoop.hbase.util.Bytes;
043import org.apache.hadoop.hbase.zookeeper.ZKNodeTracker;
044import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
045import org.junit.AfterClass;
046import org.junit.BeforeClass;
047import org.junit.ClassRule;
048import org.junit.Test;
049import org.junit.experimental.categories.Category;
050
051/**
052 * Tests unhandled exceptions thrown by coprocessors running on master. Expected result is that the
053 * master will remove the buggy coprocessor from its set of coprocessors and throw a
054 * org.apache.hadoop.hbase.exceptions.DoNotRetryIOException back to the client. (HBASE-4014).
055 */
056@Category({ CoprocessorTests.class, MediumTests.class })
057public class TestMasterCoprocessorExceptionWithRemove {
058
059  @ClassRule
060  public static final HBaseClassTestRule CLASS_RULE =
061    HBaseClassTestRule.forClass(TestMasterCoprocessorExceptionWithRemove.class);
062
063  public static class MasterTracker extends ZKNodeTracker {
064    public boolean masterZKNodeWasDeleted = false;
065
066    public MasterTracker(ZKWatcher zkw, String masterNode, Abortable abortable) {
067      super(zkw, masterNode, abortable);
068    }
069
070    @Override
071    public synchronized void nodeDeleted(String path) {
072      if (path.equals("/hbase/master")) {
073        masterZKNodeWasDeleted = true;
074      }
075    }
076  }
077
078  public static class BuggyMasterObserver implements MasterCoprocessor, MasterObserver {
079    private boolean preCreateTableCalled;
080    private boolean postCreateTableCalled;
081    private boolean startCalled;
082    private boolean postStartMasterCalled;
083
084    @Override
085    public Optional<MasterObserver> getMasterObserver() {
086      return Optional.of(this);
087    }
088
089    @SuppressWarnings("null")
090    @Override
091    public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> env,
092      TableDescriptor desc, RegionInfo[] regions) throws IOException {
093      // Cause a NullPointerException and don't catch it: this should cause the
094      // master to throw an o.apache.hadoop.hbase.DoNotRetryIOException to the
095      // client.
096      Integer i;
097      i = null;
098      i = i++;
099    }
100
101    public boolean wasCreateTableCalled() {
102      return preCreateTableCalled && postCreateTableCalled;
103    }
104
105    @Override
106    public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx)
107      throws IOException {
108      postStartMasterCalled = true;
109    }
110
111    public boolean wasStartMasterCalled() {
112      return postStartMasterCalled;
113    }
114
115    @Override
116    public void start(CoprocessorEnvironment env) throws IOException {
117      startCalled = true;
118    }
119
120    public boolean wasStarted() {
121      return startCalled;
122    }
123  }
124
125  private static HBaseTestingUtil UTIL = new HBaseTestingUtil();
126
127  private static byte[] TEST_TABLE1 = Bytes.toBytes("observed_table1");
128  private static byte[] TEST_FAMILY1 = Bytes.toBytes("fam1");
129
130  private static byte[] TEST_TABLE2 = Bytes.toBytes("table2");
131  private static byte[] TEST_FAMILY2 = Bytes.toBytes("fam2");
132
133  @BeforeClass
134  public static void setupBeforeClass() throws Exception {
135    Configuration conf = UTIL.getConfiguration();
136    conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, BuggyMasterObserver.class.getName());
137    UTIL.getConfiguration().setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, false);
138    UTIL.startMiniCluster();
139  }
140
141  @AfterClass
142  public static void teardownAfterClass() throws Exception {
143    UTIL.shutdownMiniCluster();
144  }
145
146  @Test
147  public void testExceptionFromCoprocessorWhenCreatingTable() throws IOException {
148    SingleProcessHBaseCluster cluster = UTIL.getHBaseCluster();
149
150    HMaster master = cluster.getMaster();
151    MasterCoprocessorHost host = master.getMasterCoprocessorHost();
152    BuggyMasterObserver cp = host.findCoprocessor(BuggyMasterObserver.class);
153    assertFalse("No table created yet", cp.wasCreateTableCalled());
154
155    // Set a watch on the zookeeper /hbase/master node. If the master dies,
156    // the node will be deleted.
157    // Master should *NOT* die:
158    // we are testing that the default setting of hbase.coprocessor.abortonerror
159    // =false
160    // is respected.
161    ZKWatcher zkw = new ZKWatcher(UTIL.getConfiguration(), "unittest", new Abortable() {
162      @Override
163      public void abort(String why, Throwable e) {
164        throw new RuntimeException("Fatal ZK error: " + why, e);
165      }
166
167      @Override
168      public boolean isAborted() {
169        return false;
170      }
171    });
172
173    MasterTracker masterTracker = new MasterTracker(zkw, "/hbase/master", new Abortable() {
174      @Override
175      public void abort(String why, Throwable e) {
176        throw new RuntimeException("Fatal ZooKeeper tracker error, why=", e);
177      }
178
179      @Override
180      public boolean isAborted() {
181        return false;
182      }
183    });
184
185    masterTracker.start();
186    zkw.registerListener(masterTracker);
187
188    // Test (part of the) output that should have be printed by master when it aborts:
189    // (namely the part that shows the set of loaded coprocessors).
190    // In this test, there is only a single coprocessor (BuggyMasterObserver).
191    String coprocessorName = BuggyMasterObserver.class.getName();
192    assertTrue(HMaster.getLoadedCoprocessors().contains(coprocessorName));
193
194    TableDescriptor tableDescriptor1 =
195      TableDescriptorBuilder.newBuilder(TableName.valueOf(TEST_TABLE1))
196        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(TEST_FAMILY1)).build();
197
198    boolean threwDNRE = false;
199    try {
200      Admin admin = UTIL.getAdmin();
201      admin.createTable(tableDescriptor1);
202    } catch (IOException e) {
203      if (e.getClass().getName().equals("org.apache.hadoop.hbase.DoNotRetryIOException")) {
204        threwDNRE = true;
205      }
206    } finally {
207      assertTrue(threwDNRE);
208    }
209
210    // wait for a few seconds to make sure that the Master hasn't aborted.
211    try {
212      Thread.sleep(3000);
213    } catch (InterruptedException e) {
214      fail("InterruptedException while sleeping.");
215    }
216
217    assertFalse("Master survived coprocessor NPE, as expected.",
218      masterTracker.masterZKNodeWasDeleted);
219
220    String loadedCoprocessors = HMaster.getLoadedCoprocessors();
221    assertTrue(loadedCoprocessors.contains(coprocessorName));
222
223    // Verify that BuggyMasterObserver has been removed due to its misbehavior
224    // by creating another table: should not have a problem this time.
225    TableDescriptor tableDescriptor2 =
226      TableDescriptorBuilder.newBuilder(TableName.valueOf(TEST_TABLE2))
227        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(TEST_FAMILY2)).build();
228    Admin admin = UTIL.getAdmin();
229    try {
230      admin.createTable(tableDescriptor2);
231    } catch (IOException e) {
232      fail("Failed to create table after buggy coprocessor removal: " + e);
233    }
234  }
235
236}