001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNull;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import java.net.ConnectException;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.client.ClusterConnection;
029import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
030import org.apache.hadoop.hbase.ipc.HBaseRpcController;
031import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
032import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
033import org.apache.hadoop.hbase.master.RegionState;
034import org.apache.hadoop.hbase.testclassification.MediumTests;
035import org.apache.hadoop.hbase.testclassification.MiscTests;
036import org.apache.hadoop.hbase.util.Threads;
037import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
038import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
039import org.apache.zookeeper.KeeperException;
040import org.junit.After;
041import org.junit.AfterClass;
042import org.junit.Before;
043import org.junit.BeforeClass;
044import org.junit.ClassRule;
045import org.junit.Test;
046import org.junit.experimental.categories.Category;
047import org.mockito.Mockito;
048import org.slf4j.Logger;
049import org.slf4j.LoggerFactory;
050
051import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
052import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
053
054import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
055import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest;
056import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
057import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.GetRequest;
058import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.GetResponse;
059
060/**
061 * Test {@link org.apache.hadoop.hbase.zookeeper.MetaTableLocator}
062 */
063@Category({MiscTests.class, MediumTests.class})
064public class TestMetaTableLocator {
065
066  @ClassRule
067  public static final HBaseClassTestRule CLASS_RULE =
068      HBaseClassTestRule.forClass(TestMetaTableLocator.class);
069
070  private static final Logger LOG = LoggerFactory.getLogger(TestMetaTableLocator.class);
071  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
072  private static final ServerName SN =
073      ServerName.valueOf("example.org", 1234, System.currentTimeMillis());
074  private ZKWatcher watcher;
075  private Abortable abortable;
076
077  @BeforeClass public static void beforeClass() throws Exception {
078    // Set this down so tests run quicker
079    UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
080    UTIL.startMiniZKCluster();
081  }
082
083  @AfterClass public static void afterClass() throws IOException {
084    UTIL.getZkCluster().shutdown();
085  }
086
087  @Before public void before() throws IOException {
088    this.abortable = new Abortable() {
089      @Override
090      public void abort(String why, Throwable e) {
091        LOG.info(why, e);
092      }
093
094      @Override
095      public boolean isAborted()  {
096        return false;
097      }
098    };
099    this.watcher = new ZKWatcher(UTIL.getConfiguration(),
100      this.getClass().getSimpleName(), this.abortable, true);
101  }
102
103  @After public void after() {
104    try {
105      // Clean out meta location or later tests will be confused... they presume
106      // start fresh in zk.
107      new MetaTableLocator().deleteMetaLocation(this.watcher);
108    } catch (KeeperException e) {
109      LOG.warn("Unable to delete hbase:meta location", e);
110    }
111
112    this.watcher.close();
113  }
114
115  /**
116   * Test normal operations
117   */
118  @Test public void testMetaLookup()
119          throws IOException, InterruptedException, ServiceException, KeeperException {
120    final ClientProtos.ClientService.BlockingInterface client =
121            Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
122
123    Mockito.when(client.get((RpcController)Mockito.any(), (GetRequest)Mockito.any())).
124            thenReturn(GetResponse.newBuilder().build());
125
126    final MetaTableLocator mtl = new MetaTableLocator();
127    assertNull(mtl.getMetaRegionLocation(this.watcher));
128    for (RegionState.State state : RegionState.State.values()) {
129      if (state.equals(RegionState.State.OPEN))
130        continue;
131      MetaTableLocator.setMetaLocation(this.watcher, SN, state);
132      assertNull(mtl.getMetaRegionLocation(this.watcher));
133      assertEquals(state, MetaTableLocator.getMetaRegionState(this.watcher).getState());
134    }
135    MetaTableLocator.setMetaLocation(this.watcher, SN, RegionState.State.OPEN);
136    assertEquals(SN, mtl.getMetaRegionLocation(this.watcher));
137    assertEquals(RegionState.State.OPEN,
138      MetaTableLocator.getMetaRegionState(this.watcher).getState());
139
140    mtl.deleteMetaLocation(this.watcher);
141    assertNull(MetaTableLocator.getMetaRegionState(this.watcher).getServerName());
142    assertEquals(RegionState.State.OFFLINE,
143        MetaTableLocator.getMetaRegionState(this.watcher).getState());
144    assertNull(mtl.getMetaRegionLocation(this.watcher));
145  }
146
147
148  /**
149   * Test interruptable while blocking wait on meta.
150   * @throws IOException
151   * @throws ServiceException
152   * @throws InterruptedException
153   */
154  @Test public void testInterruptWaitOnMeta()
155  throws IOException, InterruptedException, ServiceException {
156    final ClientProtos.ClientService.BlockingInterface client =
157      Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
158
159    Mockito.when(client.get((RpcController)Mockito.any(), (GetRequest)Mockito.any())).
160    thenReturn(GetResponse.newBuilder().build());
161
162    final MetaTableLocator mtl = new MetaTableLocator();
163    ServerName meta = new MetaTableLocator().getMetaRegionLocation(this.watcher);
164    assertNull(meta);
165    Thread t = new Thread() {
166      @Override
167      public void run() {
168        try {
169          mtl.waitMetaRegionLocation(watcher);
170        } catch (InterruptedException e) {
171          throw new RuntimeException("Interrupted", e);
172        }
173      }
174    };
175    t.start();
176    while (!t.isAlive())
177      Threads.sleep(1);
178    Threads.sleep(1);
179    assertTrue(t.isAlive());
180    mtl.stop();
181    // Join the thread... should exit shortly.
182    t.join();
183  }
184
185  private void testVerifyMetaRegionLocationWithException(Exception ex)
186  throws IOException, InterruptedException, KeeperException, ServiceException {
187    // Mock an ClientProtocol.
188    final ClientProtos.ClientService.BlockingInterface implementation =
189      Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
190
191    ClusterConnection connection = mockConnection(null, implementation);
192
193    // If a 'get' is called on mocked interface, throw connection refused.
194    Mockito.when(implementation.get((RpcController) Mockito.any(), (GetRequest) Mockito.any())).
195      thenThrow(new ServiceException(ex));
196
197    long timeout = UTIL.getConfiguration().
198            getLong("hbase.catalog.verification.timeout", 1000);
199    MetaTableLocator.setMetaLocation(this.watcher, SN, RegionState.State.OPENING);
200    assertFalse(new MetaTableLocator().verifyMetaRegionLocation(
201      connection, watcher, timeout));
202
203    MetaTableLocator.setMetaLocation(this.watcher, SN, RegionState.State.OPEN);
204    assertFalse(new MetaTableLocator().verifyMetaRegionLocation(
205            connection, watcher, timeout));
206  }
207
208  /**
209   * Test we survive a connection refused {@link ConnectException}
210   * @throws IOException
211   * @throws InterruptedException
212   * @throws KeeperException
213   * @throws ServiceException
214   */
215  @Test
216  public void testGetMetaServerConnectionFails()
217  throws IOException, InterruptedException, KeeperException, ServiceException {
218    testVerifyMetaRegionLocationWithException(new ConnectException("Connection refused"));
219  }
220
221  /**
222   * Test that verifyMetaRegionLocation properly handles getting a
223   * ServerNotRunningException. See HBASE-4470.
224   * Note this doesn't check the exact exception thrown in the
225   * HBASE-4470 as there it is thrown from getHConnection() and
226   * here it is thrown from get() -- but those are both called
227   * from the same function anyway, and this way is less invasive than
228   * throwing from getHConnection would be.
229   *
230   * @throws IOException
231   * @throws InterruptedException
232   * @throws KeeperException
233   * @throws ServiceException
234   */
235  @Test
236  public void testVerifyMetaRegionServerNotRunning()
237  throws IOException, InterruptedException, KeeperException, ServiceException {
238    testVerifyMetaRegionLocationWithException(new ServerNotRunningYetException("mock"));
239  }
240
241  /**
242   * Test get of meta region fails properly if nothing to connect to.
243   * @throws IOException
244   * @throws InterruptedException
245   * @throws KeeperException
246   * @throws ServiceException
247   */
248  @Test
249  public void testVerifyMetaRegionLocationFails()
250  throws IOException, InterruptedException, KeeperException, ServiceException {
251    ClusterConnection connection = Mockito.mock(ClusterConnection.class);
252    ServiceException connectException =
253      new ServiceException(new ConnectException("Connection refused"));
254    final AdminProtos.AdminService.BlockingInterface implementation =
255      Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
256    Mockito.when(implementation.getRegionInfo((RpcController)Mockito.any(),
257      (GetRegionInfoRequest)Mockito.any())).thenThrow(connectException);
258    Mockito.when(connection.getAdmin(Mockito.any())).
259      thenReturn(implementation);
260        RpcControllerFactory controllerFactory = Mockito.mock(RpcControllerFactory.class);
261        Mockito.when(controllerFactory.newController()).thenReturn(
262          Mockito.mock(HBaseRpcController.class));
263        Mockito.when(connection.getRpcControllerFactory()).thenReturn(controllerFactory);
264
265    ServerName sn = ServerName.valueOf("example.com", 1234, System.currentTimeMillis());
266    MetaTableLocator.setMetaLocation(this.watcher,
267            sn,
268            RegionState.State.OPENING);
269    assertFalse(new MetaTableLocator().verifyMetaRegionLocation(connection, watcher, 100));
270    MetaTableLocator.setMetaLocation(this.watcher, sn, RegionState.State.OPEN);
271    assertFalse(new MetaTableLocator().verifyMetaRegionLocation(connection, watcher, 100));
272  }
273
274  @Test (expected = NotAllMetaRegionsOnlineException.class)
275  public void testTimeoutWaitForMeta()
276  throws IOException, InterruptedException {
277    new MetaTableLocator().waitMetaRegionLocation(watcher, 100);
278  }
279
280  /**
281   * Test waiting on meat w/ no timeout specified.
282   * @throws IOException
283   * @throws InterruptedException
284   * @throws KeeperException
285   */
286  @Test public void testNoTimeoutWaitForMeta()
287  throws IOException, InterruptedException, KeeperException {
288    final MetaTableLocator mtl = new MetaTableLocator();
289    ServerName hsa = mtl.getMetaRegionLocation(watcher);
290    assertNull(hsa);
291
292    // Now test waiting on meta location getting set.
293    Thread t = new WaitOnMetaThread();
294    startWaitAliveThenWaitItLives(t, 1);
295    // Set a meta location.
296    MetaTableLocator.setMetaLocation(this.watcher, SN, RegionState.State.OPEN);
297    hsa = SN;
298    // Join the thread... should exit shortly.
299    t.join();
300    // Now meta is available.
301    assertTrue(mtl.getMetaRegionLocation(watcher).equals(hsa));
302  }
303
304  /**
305   * @param admin An {@link AdminProtos.AdminService.BlockingInterface} instance; you'll likely
306   * want to pass a mocked HRS; can be null.
307   * @param client A mocked ClientProtocol instance, can be null
308   * @return Mock up a connection that returns a {@link Configuration} when
309   * {@link org.apache.hadoop.hbase.client.ClusterConnection#getConfiguration()} is called, a 'location' when
310   * {@link org.apache.hadoop.hbase.client.RegionLocator#getRegionLocation(byte[], boolean)} is called,
311   * and that returns the passed {@link AdminProtos.AdminService.BlockingInterface} instance when
312   * {@link org.apache.hadoop.hbase.client.ClusterConnection#getAdmin(ServerName)} is called, returns the passed
313   * {@link ClientProtos.ClientService.BlockingInterface} instance when
314   * {@link org.apache.hadoop.hbase.client.ClusterConnection#getClient(ServerName)} is called.
315   * @throws IOException
316   */
317  private ClusterConnection mockConnection(final AdminProtos.AdminService.BlockingInterface admin,
318      final ClientProtos.ClientService.BlockingInterface client)
319  throws IOException {
320    ClusterConnection connection =
321      HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
322    Mockito.doNothing().when(connection).close();
323    // Make it so we return any old location when asked.
324    final HRegionLocation anyLocation = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN);
325    Mockito.when(connection.getRegionLocation((TableName) Mockito.any(),
326        (byte[]) Mockito.any(), Mockito.anyBoolean())).
327      thenReturn(anyLocation);
328    Mockito.when(connection.locateRegion((TableName) Mockito.any(),
329        (byte[]) Mockito.any())).
330      thenReturn(anyLocation);
331    if (admin != null) {
332      // If a call to getHRegionConnection, return this implementation.
333      Mockito.when(connection.getAdmin(Mockito.any())).
334        thenReturn(admin);
335    }
336    if (client != null) {
337      // If a call to getClient, return this implementation.
338      Mockito.when(connection.getClient(Mockito.any())).
339        thenReturn(client);
340    }
341    return connection;
342  }
343
344  private void startWaitAliveThenWaitItLives(final Thread t, final int ms) {
345    t.start();
346    while(!t.isAlive()) {
347      // Wait
348    }
349    // Wait one second.
350    Threads.sleep(ms);
351    assertTrue("Assert " + t.getName() + " still waiting", t.isAlive());
352  }
353
354  /**
355   * Wait on META.
356   */
357  class WaitOnMetaThread extends Thread {
358
359    WaitOnMetaThread() {
360      super("WaitOnMeta");
361    }
362
363    @Override
364    public void run() {
365      try {
366        doWaiting();
367      } catch (InterruptedException e) {
368        throw new RuntimeException("Failed wait", e);
369      }
370      LOG.info("Exiting " + getName());
371    }
372
373    void doWaiting() throws InterruptedException {
374      try {
375        while (new MetaTableLocator().waitMetaRegionLocation(watcher, 10000) == null);
376      } catch (NotAllMetaRegionsOnlineException e) {
377        //Ignore
378      }
379    }
380  }
381}