001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.replication; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertFalse; 022import static org.junit.Assert.assertTrue; 023import static org.mockito.ArgumentMatchers.any; 024import static org.mockito.ArgumentMatchers.anyBoolean; 025import static org.mockito.ArgumentMatchers.anyString; 026import static org.mockito.Mockito.doAnswer; 027import static org.mockito.Mockito.spy; 028 029import java.io.IOException; 030import org.apache.hadoop.conf.Configuration; 031import org.apache.hadoop.hbase.HBaseClassTestRule; 032import org.apache.hadoop.hbase.HBaseTestingUtility; 033import org.apache.hadoop.hbase.HConstants; 034import org.apache.hadoop.hbase.client.Admin; 035import org.apache.hadoop.hbase.master.HMaster; 036import org.apache.hadoop.hbase.master.replication.ReplicationPeerManager; 037import org.apache.hadoop.hbase.testclassification.MediumTests; 038import org.apache.hadoop.hbase.testclassification.ReplicationTests; 039import org.apache.zookeeper.KeeperException; 040import org.junit.After; 041import org.junit.AfterClass; 042import org.junit.BeforeClass; 043import org.junit.ClassRule; 044import org.junit.Test; 045import org.junit.experimental.categories.Category; 046import org.mockito.invocation.InvocationOnMock; 047 048/** 049 * All the modification method will fail once in the test and should finally succeed. 050 */ 051@Category({ ReplicationTests.class, MediumTests.class }) 052public class TestReplicationProcedureRetry { 053 054 @ClassRule 055 public static final HBaseClassTestRule CLASS_RULE = 056 HBaseClassTestRule.forClass(TestReplicationProcedureRetry.class); 057 058 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); 059 060 @BeforeClass 061 public static void setUp() throws Exception { 062 UTIL.getConfiguration().setClass(HConstants.MASTER_IMPL, MockHMaster.class, HMaster.class); 063 UTIL.startMiniCluster(3); 064 } 065 066 @AfterClass 067 public static void tearDown() throws Exception { 068 UTIL.shutdownMiniCluster(); 069 } 070 071 @After 072 public void tearDownAfterTest() throws IOException { 073 for (ReplicationPeerDescription desc : UTIL.getAdmin().listReplicationPeers()) { 074 UTIL.getAdmin().removeReplicationPeer(desc.getPeerId()); 075 } 076 } 077 078 private void doTest() throws IOException { 079 Admin admin = UTIL.getAdmin(); 080 String peerId = "1"; 081 ReplicationPeerConfig peerConfig = ReplicationPeerConfig.newBuilder() 082 .setClusterKey("localhost:" + UTIL.getZkCluster().getClientPort() + ":/hbase2").build(); 083 admin.addReplicationPeer(peerId, peerConfig, true); 084 085 assertEquals(peerConfig.getClusterKey(), 086 admin.getReplicationPeerConfig(peerId).getClusterKey()); 087 ReplicationPeerConfig newPeerConfig = 088 ReplicationPeerConfig.newBuilder(peerConfig).setBandwidth(123456).build(); 089 admin.updateReplicationPeerConfig(peerId, newPeerConfig); 090 assertEquals(newPeerConfig.getBandwidth(), 091 admin.getReplicationPeerConfig(peerId).getBandwidth()); 092 093 admin.disableReplicationPeer(peerId); 094 assertFalse(admin.listReplicationPeers().get(0).isEnabled()); 095 096 admin.enableReplicationPeer(peerId); 097 assertTrue(admin.listReplicationPeers().get(0).isEnabled()); 098 099 admin.removeReplicationPeer(peerId); 100 assertTrue(admin.listReplicationPeers().isEmpty()); 101 102 // make sure that we have run into the mocked method 103 MockHMaster master = (MockHMaster) UTIL.getHBaseCluster().getMaster(); 104 assertTrue(master.addPeerCalled); 105 assertTrue(master.removePeerCalled); 106 assertTrue(master.updatePeerConfigCalled); 107 assertTrue(master.enablePeerCalled); 108 assertTrue(master.disablePeerCalled); 109 } 110 111 @Test 112 public void testErrorBeforeUpdate() throws IOException, ReplicationException { 113 ((MockHMaster) UTIL.getHBaseCluster().getMaster()).reset(true); 114 doTest(); 115 } 116 117 @Test 118 public void testErrorAfterUpdate() throws IOException, ReplicationException { 119 ((MockHMaster) UTIL.getHBaseCluster().getMaster()).reset(false); 120 doTest(); 121 } 122 123 public static final class MockHMaster extends HMaster { 124 125 volatile boolean addPeerCalled; 126 127 volatile boolean removePeerCalled; 128 129 volatile boolean updatePeerConfigCalled; 130 131 volatile boolean enablePeerCalled; 132 133 volatile boolean disablePeerCalled; 134 135 private ReplicationPeerManager manager; 136 137 public MockHMaster(Configuration conf) throws IOException, KeeperException { 138 super(conf); 139 } 140 141 private Object invokeWithError(InvocationOnMock invocation, boolean errorBeforeUpdate) 142 throws Throwable { 143 if (errorBeforeUpdate) { 144 throw new ReplicationException("mock error before update"); 145 } 146 invocation.callRealMethod(); 147 throw new ReplicationException("mock error after update"); 148 } 149 150 public void reset(boolean errorBeforeUpdate) throws ReplicationException { 151 addPeerCalled = false; 152 removePeerCalled = false; 153 updatePeerConfigCalled = false; 154 enablePeerCalled = false; 155 disablePeerCalled = false; 156 ReplicationPeerManager m = super.getReplicationPeerManager(); 157 manager = spy(m); 158 doAnswer(invocation -> { 159 if (!addPeerCalled) { 160 addPeerCalled = true; 161 return invokeWithError(invocation, errorBeforeUpdate); 162 } else { 163 return invocation.callRealMethod(); 164 } 165 }).when(manager).addPeer(anyString(), any(ReplicationPeerConfig.class), anyBoolean()); 166 doAnswer(invocation -> { 167 if (!removePeerCalled) { 168 removePeerCalled = true; 169 return invokeWithError(invocation, errorBeforeUpdate); 170 } else { 171 return invocation.callRealMethod(); 172 } 173 }).when(manager).removePeer(anyString()); 174 doAnswer(invocation -> { 175 if (!updatePeerConfigCalled) { 176 updatePeerConfigCalled = true; 177 return invokeWithError(invocation, errorBeforeUpdate); 178 } else { 179 return invocation.callRealMethod(); 180 } 181 }).when(manager).updatePeerConfig(anyString(), any(ReplicationPeerConfig.class)); 182 doAnswer(invocation -> { 183 if (!enablePeerCalled) { 184 enablePeerCalled = true; 185 return invokeWithError(invocation, errorBeforeUpdate); 186 } else { 187 return invocation.callRealMethod(); 188 } 189 }).when(manager).enablePeer(anyString()); 190 doAnswer(invocation -> { 191 if (!disablePeerCalled) { 192 disablePeerCalled = true; 193 return invokeWithError(invocation, errorBeforeUpdate); 194 } else { 195 return invocation.callRealMethod(); 196 } 197 }).when(manager).disablePeer(anyString()); 198 } 199 200 @Override 201 public ReplicationPeerManager getReplicationPeerManager() { 202 return manager; 203 } 204 } 205}