001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.snapshot;
019
020import static org.junit.Assert.assertFalse;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.Collections;
026import java.util.List;
027import java.util.Objects;
028import java.util.concurrent.atomic.AtomicBoolean;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.HBaseConfiguration;
032import org.apache.hadoop.hbase.ServerName;
033import org.apache.hadoop.hbase.TableName;
034import org.apache.hadoop.hbase.client.RegionInfo;
035import org.apache.hadoop.hbase.client.RegionInfoBuilder;
036import org.apache.hadoop.hbase.master.MasterServices;
037import org.apache.hadoop.hbase.master.ServerManager;
038import org.apache.hadoop.hbase.master.assignment.AssignProcedure;
039import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
040import org.apache.hadoop.hbase.master.assignment.RegionStates;
041import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
042import org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher;
043import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
044import org.apache.hadoop.hbase.testclassification.RegionServerTests;
045import org.apache.hadoop.hbase.testclassification.SmallTests;
046import org.apache.hadoop.hbase.util.Bytes;
047import org.junit.ClassRule;
048import org.junit.Rule;
049import org.junit.Test;
050import org.junit.experimental.categories.Category;
051import org.junit.rules.TestName;
052import org.mockito.Mockito;
053import org.slf4j.Logger;
054import org.slf4j.LoggerFactory;
055
056@Category({RegionServerTests.class, SmallTests.class})
057public class TestAssignProcedure {
058
059  @ClassRule
060  public static final HBaseClassTestRule CLASS_RULE =
061      HBaseClassTestRule.forClass(TestAssignProcedure.class);
062
063  private static final Logger LOG = LoggerFactory.getLogger(TestAssignProcedure.class);
064  @Rule public TestName name = new TestName();
065
066  /**
067   * An override that opens up the updateTransition method inside in AssignProcedure so can call it
068   * below directly in test and mess with targetServer. Used by test
069   * {@link #testTargetServerBeingNulledOnUs()}.
070   */
071  public static class TargetServerBeingNulledOnUsAssignProcedure extends AssignProcedure {
072    public final AtomicBoolean addToRemoteDispatcherWasCalled = new AtomicBoolean(false);
073    public final AtomicBoolean remoteCallFailedWasCalled = new AtomicBoolean(false);
074    private final RegionStates.RegionStateNode rsn;
075
076    public TargetServerBeingNulledOnUsAssignProcedure(RegionInfo regionInfo,
077        RegionStates.RegionStateNode rsn) {
078      super(regionInfo);
079      this.rsn = rsn;
080    }
081
082    /**
083     * Override so can change access from protected to public.
084     */
085    @Override
086    public boolean updateTransition(MasterProcedureEnv env, RegionStates.RegionStateNode regionNode)
087        throws IOException, ProcedureSuspendedException {
088      return super.updateTransition(env, regionNode);
089    }
090
091    @Override
092    protected boolean addToRemoteDispatcher(MasterProcedureEnv env, ServerName targetServer) {
093      // So, mock the ServerCrashProcedure nulling out the targetServer AFTER updateTransition
094      // has been called and BEFORE updateTransition gets to here.
095      // We used to throw a NullPointerException. Now we just say the assign failed so it will
096      // be rescheduled.
097      boolean b = super.addToRemoteDispatcher(env, null);
098      assertFalse(b);
099      // Assert we were actually called.
100      this.addToRemoteDispatcherWasCalled.set(true);
101      return b;
102    }
103
104    @Override
105    public RegionStates.RegionStateNode getRegionState(MasterProcedureEnv env) {
106      // Do this so we don't have to mock a bunch of stuff.
107      return this.rsn;
108    }
109
110    @Override
111    public boolean remoteCallFailed(final MasterProcedureEnv env,
112        final ServerName serverName, final IOException exception) {
113      // Just skip this remoteCallFailed. Its too hard to mock. Assert it is called though.
114      // Happens after the code we are testing has been called.
115      this.remoteCallFailedWasCalled.set(true);
116      return true;
117    }
118  };
119
120  /**
121   * Test that we deal with ServerCrashProcedure zero'ing out the targetServer in the
122   * RegionStateNode in the midst of our doing an assign. The trickery is done above in
123   * TargetServerBeingNulledOnUsAssignProcedure. We skip a bunch of logic to get at the guts
124   * where the problem happens (We also skip-out the failure handling because it'd take a bunch
125   * of mocking to get it to run). Fix is inside in RemoteProcedureDispatch#addOperationToNode.
126   * It now notices empty targetServer and just returns false so we fall into failure processing
127   * and we'll reassign elsewhere instead of NPE'ing. The fake of ServerCrashProcedure nulling out
128   * the targetServer happens inside in updateTransition just after it was called but before it
129   * gets to the near the end when addToRemoteDispatcher is called. See the
130   * TargetServerBeingNulledOnUsAssignProcedure class above. See HBASE-19218.
131   * Before fix, this test would fail w/ a NullPointerException.
132   */
133  @Test
134  public void testTargetServerBeingNulledOnUs() throws ProcedureSuspendedException, IOException {
135    TableName tn = TableName.valueOf(this.name.getMethodName());
136    RegionInfo ri = RegionInfoBuilder.newBuilder(tn).build();
137    // Create an RSN with location/target server. Will be cleared above in addToRemoteDispatcher to
138    // simulate issue in HBASE-19218
139    RegionStates.RegionStateNode rsn = new RegionStates.RegionStateNode(ri);
140    rsn.setRegionLocation(ServerName.valueOf("server.example.org", 0, 0));
141    MasterProcedureEnv env = Mockito.mock(MasterProcedureEnv.class);
142    AssignmentManager am = Mockito.mock(AssignmentManager.class);
143    ServerManager sm = Mockito.mock(ServerManager.class);
144    Mockito.when(sm.isServerOnline(Mockito.any())).thenReturn(true);
145    MasterServices ms = Mockito.mock(MasterServices.class);
146    Mockito.when(ms.getServerManager()).thenReturn(sm);
147    Configuration configuration = HBaseConfiguration.create();
148    Mockito.when(ms.getConfiguration()).thenReturn(configuration);
149    Mockito.when(env.getAssignmentManager()).thenReturn(am);
150    Mockito.when(env.getMasterServices()).thenReturn(ms);
151    RSProcedureDispatcher rsd = new RSProcedureDispatcher(ms);
152    Mockito.when(env.getRemoteDispatcher()).thenReturn(rsd);
153
154    TargetServerBeingNulledOnUsAssignProcedure assignProcedure =
155        new TargetServerBeingNulledOnUsAssignProcedure(ri, rsn);
156    assignProcedure.updateTransition(env, rsn);
157    assertTrue(assignProcedure.remoteCallFailedWasCalled.get());
158    assertTrue(assignProcedure.addToRemoteDispatcherWasCalled.get());
159  }
160
161  @Test
162  public void testSimpleComparator() {
163    List<AssignProcedure> procedures = new ArrayList<AssignProcedure>();
164    RegionInfo user1 = RegionInfoBuilder.newBuilder(TableName.valueOf("user_space1")).build();
165    procedures.add(new AssignProcedure(user1));
166    RegionInfo user2 = RegionInfoBuilder.newBuilder(TableName.valueOf("user_space2")).build();
167    procedures.add(new AssignProcedure(RegionInfoBuilder.FIRST_META_REGIONINFO));
168    procedures.add(new AssignProcedure(user2));
169    RegionInfo system = RegionInfoBuilder.newBuilder(TableName.NAMESPACE_TABLE_NAME).build();
170    procedures.add(new AssignProcedure(system));
171    procedures.sort(AssignProcedure.COMPARATOR);
172    assertTrue(procedures.get(0).isMeta());
173    assertTrue(procedures.get(1).getRegionInfo().getTable().equals(TableName.NAMESPACE_TABLE_NAME));
174  }
175
176  @Test
177  public void testComparatorWithMetas() {
178    List<AssignProcedure> procedures = new ArrayList<AssignProcedure>();
179    RegionInfo user3 = RegionInfoBuilder.newBuilder(TableName.valueOf("user3")).build();
180    procedures.add(new AssignProcedure(user3));
181    RegionInfo system = RegionInfoBuilder.newBuilder(TableName.NAMESPACE_TABLE_NAME).build();
182    procedures.add(new AssignProcedure(system));
183    RegionInfo user1 = RegionInfoBuilder.newBuilder(TableName.valueOf("user_space1")).build();
184    RegionInfo user2 = RegionInfoBuilder.newBuilder(TableName.valueOf("user_space2")).build();
185    procedures.add(new AssignProcedure(user1));
186    RegionInfo meta2 = RegionInfoBuilder.newBuilder(TableName.META_TABLE_NAME).
187        setStartKey(Bytes.toBytes("002")).build();
188    procedures.add(new AssignProcedure(meta2));
189    procedures.add(new AssignProcedure(user2));
190    RegionInfo meta1 = RegionInfoBuilder.newBuilder(TableName.META_TABLE_NAME).
191        setStartKey(Bytes.toBytes("001")).build();
192    procedures.add(new AssignProcedure(meta1));
193    procedures.add(new AssignProcedure(RegionInfoBuilder.FIRST_META_REGIONINFO));
194    RegionInfo meta0 = RegionInfoBuilder.newBuilder(TableName.META_TABLE_NAME).
195        setStartKey(Bytes.toBytes("000")).build();
196    procedures.add(new AssignProcedure(meta0));
197    for (int i = 0; i < 10; i++) {
198      Collections.shuffle(procedures);
199      procedures.sort(AssignProcedure.COMPARATOR);
200      try {
201        assertTrue(procedures.get(0).getRegionInfo().equals(RegionInfoBuilder.FIRST_META_REGIONINFO));
202        assertTrue(procedures.get(1).getRegionInfo().equals(meta0));
203        assertTrue(procedures.get(2).getRegionInfo().equals(meta1));
204        assertTrue(procedures.get(3).getRegionInfo().equals(meta2));
205        assertTrue(procedures.get(4).getRegionInfo().getTable().equals(TableName.NAMESPACE_TABLE_NAME));
206        assertTrue(procedures.get(5).getRegionInfo().equals(user1));
207        assertTrue(procedures.get(6).getRegionInfo().equals(user2));
208        assertTrue(procedures.get(7).getRegionInfo().equals(user3));
209      } catch (Throwable t) {
210        for (AssignProcedure proc : procedures) {
211          LOG.debug(Objects.toString(proc));
212        }
213        throw t;
214      }
215    }
216  }
217}