001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase; 019 020import static org.junit.Assert.assertTrue; 021 022import java.io.IOException; 023import java.util.HashSet; 024import java.util.Hashtable; 025import java.util.Iterator; 026import java.util.List; 027import java.util.Map; 028import java.util.Random; 029import java.util.Set; 030import javax.management.MBeanAttributeInfo; 031import javax.management.MBeanInfo; 032import javax.management.MBeanServerConnection; 033import javax.management.ObjectInstance; 034import javax.management.ObjectName; 035import javax.management.remote.JMXConnector; 036import javax.management.remote.JMXConnectorFactory; 037import org.apache.hadoop.conf.Configuration; 038import org.apache.hadoop.hbase.client.RegionInfo; 039import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; 040import org.apache.hadoop.hbase.master.balancer.BalancerTestBase; 041import org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer; 042import org.apache.hadoop.hbase.testclassification.MediumTests; 043import org.apache.hadoop.hbase.testclassification.MiscTests; 044import org.apache.hadoop.hbase.util.Threads; 045import org.apache.hadoop.net.DNSToSwitchMapping; 046import org.junit.AfterClass; 047import org.junit.BeforeClass; 048import org.junit.ClassRule; 049import org.junit.FixMethodOrder; 050import org.junit.Ignore; 051import org.junit.Test; 052import org.junit.experimental.categories.Category; 053import org.junit.runners.MethodSorters; 054import org.slf4j.Logger; 055import org.slf4j.LoggerFactory; 056 057@Category({ MiscTests.class, MediumTests.class }) 058@FixMethodOrder(MethodSorters.NAME_ASCENDING) 059@Ignore 060public class TestStochasticBalancerJmxMetrics extends BalancerTestBase { 061 062 @ClassRule 063 public static final HBaseClassTestRule CLASS_RULE = 064 HBaseClassTestRule.forClass(TestStochasticBalancerJmxMetrics.class); 065 066 private static final Logger LOG = LoggerFactory.getLogger(TestStochasticBalancerJmxMetrics.class); 067 private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); 068 private static int connectorPort = 61120; 069 private static StochasticLoadBalancer loadBalancer; 070 /** 071 * a simple cluster for testing JMX. 072 */ 073 private static int[] mockCluster_ensemble = new int[] { 0, 1, 2, 3 }; 074 private static int[] mockCluster_pertable_1 = new int[] { 0, 1, 2 }; 075 private static int[] mockCluster_pertable_2 = new int[] { 3, 1, 1 }; 076 private static int[] mockCluster_pertable_namespace = new int[] { 1, 3, 1 }; 077 078 private static final String TABLE_NAME_1 = "Table1"; 079 private static final String TABLE_NAME_2 = "Table2"; 080 private static final String TABLE_NAME_NAMESPACE = "hbase:namespace"; 081 082 private static Configuration conf = null; 083 084 /** 085 * Setup the environment for the test. 086 */ 087 @BeforeClass 088 public static void setupBeforeClass() throws Exception { 089 090 conf = UTIL.getConfiguration(); 091 092 conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class); 093 conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 0.75f); 094 conf.setFloat("hbase.regions.slop", 0.0f); 095 conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, JMXListener.class.getName()); 096 Random rand = new Random(); 097 for (int i = 0; i < 10; i++) { 098 do { 099 int sign = i % 2 == 0 ? 1 : -1; 100 connectorPort += sign * rand.nextInt(100); 101 } while (!HBaseTestingUtility.available(connectorPort)); 102 try { 103 conf.setInt("regionserver.rmi.registry.port", connectorPort); 104 105 UTIL.startMiniCluster(); 106 break; 107 } catch (Exception e) { 108 LOG.debug("Encountered exception when starting cluster. Trying port " + connectorPort, e); 109 try { 110 // this is to avoid "IllegalStateException: A mini-cluster is already running" 111 UTIL.shutdownMiniCluster(); 112 } catch (Exception ex) { 113 LOG.debug("Encountered exception shutting down cluster", ex); 114 } 115 } 116 } 117 } 118 119 @AfterClass 120 public static void tearDownAfterClass() throws Exception { 121 UTIL.shutdownMiniCluster(); 122 } 123 124 /** 125 * In Ensemble mode, there should be only one ensemble table 126 */ 127 @Test 128 public void testJmxMetrics_EnsembleMode() throws Exception { 129 loadBalancer = new StochasticLoadBalancer(); 130 131 conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, false); 132 loadBalancer.setConf(conf); 133 134 TableName tableName = HConstants.ENSEMBLE_TABLE_NAME; 135 Map<ServerName, List<RegionInfo>> clusterState = mockClusterServers(mockCluster_ensemble); 136 loadBalancer.balanceTable(tableName, clusterState); 137 138 String[] tableNames = new String[] { tableName.getNameAsString() }; 139 String[] functionNames = loadBalancer.getCostFunctionNames(); 140 Set<String> jmxMetrics = readJmxMetricsWithRetry(); 141 Set<String> expectedMetrics = getExpectedJmxMetrics(tableNames, functionNames); 142 143 // printMetrics(jmxMetrics, "existing metrics in ensemble mode"); 144 // printMetrics(expectedMetrics, "expected metrics in ensemble mode"); 145 146 // assert that every expected is in the JMX 147 for (String expected : expectedMetrics) { 148 assertTrue("Metric " + expected + " can not be found in JMX in ensemble mode.", 149 jmxMetrics.contains(expected)); 150 } 151 } 152 153 /** 154 * In per-table mode, each table has a set of metrics 155 */ 156 @Test 157 public void testJmxMetrics_PerTableMode() throws Exception { 158 loadBalancer = new StochasticLoadBalancer(); 159 160 conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, true); 161 loadBalancer.setConf(conf); 162 163 // NOTE the size is normally set in setClusterMetrics, for test purpose, we set it manually 164 // Tables: hbase:namespace, table1, table2 165 // Functions: costFunctions, overall 166 String[] functionNames = loadBalancer.getCostFunctionNames(); 167 loadBalancer.updateMetricsSize(3 * (functionNames.length + 1)); 168 169 // table 1 170 TableName tableName = TableName.valueOf(TABLE_NAME_1); 171 Map<ServerName, List<RegionInfo>> clusterState = mockClusterServers(mockCluster_pertable_1); 172 loadBalancer.balanceTable(tableName, clusterState); 173 174 // table 2 175 tableName = TableName.valueOf(TABLE_NAME_2); 176 clusterState = mockClusterServers(mockCluster_pertable_2); 177 loadBalancer.balanceTable(tableName, clusterState); 178 179 // table hbase:namespace 180 tableName = TableName.valueOf(TABLE_NAME_NAMESPACE); 181 clusterState = mockClusterServers(mockCluster_pertable_namespace); 182 loadBalancer.balanceTable(tableName, clusterState); 183 184 String[] tableNames = new String[] { TABLE_NAME_1, TABLE_NAME_2, TABLE_NAME_NAMESPACE }; 185 Set<String> jmxMetrics = readJmxMetricsWithRetry(); 186 Set<String> expectedMetrics = getExpectedJmxMetrics(tableNames, functionNames); 187 188 // printMetrics(jmxMetrics, "existing metrics in per-table mode"); 189 // printMetrics(expectedMetrics, "expected metrics in per-table mode"); 190 191 // assert that every expected is in the JMX 192 for (String expected : expectedMetrics) { 193 assertTrue("Metric " + expected + " can not be found in JMX in per-table mode.", 194 jmxMetrics.contains(expected)); 195 } 196 } 197 198 private Set<String> readJmxMetricsWithRetry() throws IOException { 199 final int count = 0; 200 for (int i = 0; i < 10; i++) { 201 Set<String> metrics = readJmxMetrics(); 202 if (metrics != null) return metrics; 203 LOG.warn("Failed to get jmxmetrics... sleeping, retrying; " + i + " of " + count + " times"); 204 Threads.sleep(1000); 205 } 206 return null; 207 } 208 209 /** 210 * Read the attributes from Hadoop->HBase->Master->Balancer in JMX 211 * @throws IOException 212 */ 213 private Set<String> readJmxMetrics() throws IOException { 214 JMXConnector connector = null; 215 ObjectName target = null; 216 MBeanServerConnection mb = null; 217 try { 218 connector = 219 JMXConnectorFactory.connect(JMXListener.buildJMXServiceURL(connectorPort, connectorPort)); 220 mb = connector.getMBeanServerConnection(); 221 222 Hashtable<String, String> pairs = new Hashtable<>(); 223 pairs.put("service", "HBase"); 224 pairs.put("name", "Master"); 225 pairs.put("sub", "Balancer"); 226 target = new ObjectName("Hadoop", pairs); 227 MBeanInfo beanInfo = mb.getMBeanInfo(target); 228 229 Set<String> existingAttrs = new HashSet<>(); 230 for (MBeanAttributeInfo attrInfo : beanInfo.getAttributes()) { 231 existingAttrs.add(attrInfo.getName()); 232 } 233 return existingAttrs; 234 } catch (Exception e) { 235 LOG.warn("Failed to get bean!!! " + target, e); 236 if (mb != null) { 237 Set<ObjectInstance> instances = mb.queryMBeans(null, null); 238 Iterator<ObjectInstance> iterator = instances.iterator(); 239 System.out.println("MBean Found:"); 240 while (iterator.hasNext()) { 241 ObjectInstance instance = iterator.next(); 242 System.out.println("Class Name: " + instance.getClassName()); 243 System.out.println("Object Name: " + instance.getObjectName()); 244 } 245 } 246 } finally { 247 if (connector != null) { 248 try { 249 connector.close(); 250 } catch (Exception e) { 251 e.printStackTrace(); 252 } 253 } 254 } 255 return null; 256 } 257 258 /** 259 * Given the tables and functions, return metrics names that should exist in JMX 260 */ 261 private Set<String> getExpectedJmxMetrics(String[] tableNames, String[] functionNames) { 262 Set<String> ret = new HashSet<>(); 263 264 for (String tableName : tableNames) { 265 ret.add(StochasticLoadBalancer.composeAttributeName(tableName, "Overall")); 266 for (String functionName : functionNames) { 267 String metricsName = StochasticLoadBalancer.composeAttributeName(tableName, functionName); 268 ret.add(metricsName); 269 } 270 } 271 272 return ret; 273 } 274 275 private static void printMetrics(Set<String> metrics, String info) { 276 if (null != info) LOG.info("++++ ------ " + info + " ------"); 277 278 LOG.info("++++ metrics count = " + metrics.size()); 279 for (String str : metrics) { 280 LOG.info(" ++++ " + str); 281 } 282 } 283}