001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.coprocessor;
019
020import static org.junit.jupiter.api.Assertions.assertFalse;
021import static org.junit.jupiter.api.Assertions.assertTrue;
022import static org.junit.jupiter.api.Assertions.fail;
023
024import java.io.IOException;
025import java.util.Optional;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.hbase.Abortable;
028import org.apache.hadoop.hbase.CoprocessorEnvironment;
029import org.apache.hadoop.hbase.HBaseTestingUtil;
030import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.client.Admin;
033import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
034import org.apache.hadoop.hbase.client.RegionInfo;
035import org.apache.hadoop.hbase.client.TableDescriptor;
036import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
037import org.apache.hadoop.hbase.master.HMaster;
038import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
039import org.apache.hadoop.hbase.testclassification.CoprocessorTests;
040import org.apache.hadoop.hbase.testclassification.MediumTests;
041import org.apache.hadoop.hbase.util.Bytes;
042import org.apache.hadoop.hbase.zookeeper.ZKNodeTracker;
043import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
044import org.junit.jupiter.api.AfterAll;
045import org.junit.jupiter.api.BeforeAll;
046import org.junit.jupiter.api.Tag;
047import org.junit.jupiter.api.Test;
048
049/**
050 * Tests unhandled exceptions thrown by coprocessors running on master. Expected result is that the
051 * master will remove the buggy coprocessor from its set of coprocessors and throw a
052 * org.apache.hadoop.hbase.exceptions.DoNotRetryIOException back to the client. (HBASE-4014).
053 */
054@Tag(CoprocessorTests.TAG)
055@Tag(MediumTests.TAG)
056public class TestMasterCoprocessorExceptionWithRemove {
057
058  public static class MasterTracker extends ZKNodeTracker {
059    public boolean masterZKNodeWasDeleted = false;
060
061    public MasterTracker(ZKWatcher zkw, String masterNode, Abortable abortable) {
062      super(zkw, masterNode, abortable);
063    }
064
065    @Override
066    public synchronized void nodeDeleted(String path) {
067      if (path.equals("/hbase/master")) {
068        masterZKNodeWasDeleted = true;
069      }
070    }
071  }
072
073  public static class BuggyMasterObserver implements MasterCoprocessor, MasterObserver {
074    private boolean preCreateTableCalled;
075    private boolean postCreateTableCalled;
076    private boolean startCalled;
077    private boolean postStartMasterCalled;
078
079    @Override
080    public Optional<MasterObserver> getMasterObserver() {
081      return Optional.of(this);
082    }
083
084    @SuppressWarnings("null")
085    @Override
086    public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> env,
087      TableDescriptor desc, RegionInfo[] regions) throws IOException {
088      // Cause a NullPointerException and don't catch it: this should cause the
089      // master to throw an o.apache.hadoop.hbase.DoNotRetryIOException to the
090      // client.
091      Integer i;
092      i = null;
093      i = i++;
094    }
095
096    public boolean wasCreateTableCalled() {
097      return preCreateTableCalled && postCreateTableCalled;
098    }
099
100    @Override
101    public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx)
102      throws IOException {
103      postStartMasterCalled = true;
104    }
105
106    public boolean wasStartMasterCalled() {
107      return postStartMasterCalled;
108    }
109
110    @Override
111    public void start(CoprocessorEnvironment env) throws IOException {
112      startCalled = true;
113    }
114
115    public boolean wasStarted() {
116      return startCalled;
117    }
118  }
119
120  private static HBaseTestingUtil UTIL = new HBaseTestingUtil();
121
122  private static byte[] TEST_TABLE1 = Bytes.toBytes("observed_table1");
123  private static byte[] TEST_FAMILY1 = Bytes.toBytes("fam1");
124
125  private static byte[] TEST_TABLE2 = Bytes.toBytes("table2");
126  private static byte[] TEST_FAMILY2 = Bytes.toBytes("fam2");
127
128  @BeforeAll
129  public static void setupBeforeClass() throws Exception {
130    Configuration conf = UTIL.getConfiguration();
131    conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, BuggyMasterObserver.class.getName());
132    UTIL.getConfiguration().setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, false);
133    UTIL.startMiniCluster();
134  }
135
136  @AfterAll
137  public static void teardownAfterClass() throws Exception {
138    UTIL.shutdownMiniCluster();
139  }
140
141  @Test
142  public void testExceptionFromCoprocessorWhenCreatingTable() throws IOException {
143    SingleProcessHBaseCluster cluster = UTIL.getHBaseCluster();
144
145    HMaster master = cluster.getMaster();
146    MasterCoprocessorHost host = master.getMasterCoprocessorHost();
147    BuggyMasterObserver cp = host.findCoprocessor(BuggyMasterObserver.class);
148    assertFalse(cp.wasCreateTableCalled(), "No table created yet");
149
150    // Set a watch on the zookeeper /hbase/master node. If the master dies,
151    // the node will be deleted.
152    // Master should *NOT* die:
153    // we are testing that the default setting of hbase.coprocessor.abortonerror
154    // =false
155    // is respected.
156    ZKWatcher zkw = new ZKWatcher(UTIL.getConfiguration(), "unittest", new Abortable() {
157      @Override
158      public void abort(String why, Throwable e) {
159        throw new RuntimeException("Fatal ZK error: " + why, e);
160      }
161
162      @Override
163      public boolean isAborted() {
164        return false;
165      }
166    });
167
168    MasterTracker masterTracker = new MasterTracker(zkw, "/hbase/master", new Abortable() {
169      @Override
170      public void abort(String why, Throwable e) {
171        throw new RuntimeException("Fatal ZooKeeper tracker error, why=", e);
172      }
173
174      @Override
175      public boolean isAborted() {
176        return false;
177      }
178    });
179
180    masterTracker.start();
181    zkw.registerListener(masterTracker);
182
183    // Test (part of the) output that should have be printed by master when it aborts:
184    // (namely the part that shows the set of loaded coprocessors).
185    // In this test, there is only a single coprocessor (BuggyMasterObserver).
186    String coprocessorName = BuggyMasterObserver.class.getName();
187    assertTrue(HMaster.getLoadedCoprocessors().contains(coprocessorName));
188
189    TableDescriptor tableDescriptor1 =
190      TableDescriptorBuilder.newBuilder(TableName.valueOf(TEST_TABLE1))
191        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(TEST_FAMILY1)).build();
192
193    boolean threwDNRE = false;
194    try {
195      Admin admin = UTIL.getAdmin();
196      admin.createTable(tableDescriptor1);
197    } catch (IOException e) {
198      if (e.getClass().getName().equals("org.apache.hadoop.hbase.DoNotRetryIOException")) {
199        threwDNRE = true;
200      }
201    } finally {
202      assertTrue(threwDNRE);
203    }
204
205    // wait for a few seconds to make sure that the Master hasn't aborted.
206    try {
207      Thread.sleep(3000);
208    } catch (InterruptedException e) {
209      fail("InterruptedException while sleeping.");
210    }
211
212    assertFalse(masterTracker.masterZKNodeWasDeleted,
213      "Master survived coprocessor NPE, as expected.");
214
215    String loadedCoprocessors = HMaster.getLoadedCoprocessors();
216    assertTrue(loadedCoprocessors.contains(coprocessorName));
217
218    // Verify that BuggyMasterObserver has been removed due to its misbehavior
219    // by creating another table: should not have a problem this time.
220    TableDescriptor tableDescriptor2 =
221      TableDescriptorBuilder.newBuilder(TableName.valueOf(TEST_TABLE2))
222        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(TEST_FAMILY2)).build();
223    Admin admin = UTIL.getAdmin();
224    try {
225      admin.createTable(tableDescriptor2);
226    } catch (IOException e) {
227      fail("Failed to create table after buggy coprocessor removal: " + e);
228    }
229  }
230
231}