001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.balancer;
019
020import static org.junit.Assert.assertTrue;
021
022import java.io.IOException;
023import java.util.HashSet;
024import java.util.Hashtable;
025import java.util.Iterator;
026import java.util.List;
027import java.util.Map;
028import java.util.Random;
029import java.util.Set;
030import java.util.concurrent.ThreadLocalRandom;
031import javax.management.MBeanAttributeInfo;
032import javax.management.MBeanInfo;
033import javax.management.MBeanServerConnection;
034import javax.management.ObjectInstance;
035import javax.management.ObjectName;
036import javax.management.remote.JMXConnector;
037import javax.management.remote.JMXConnectorFactory;
038import org.apache.hadoop.conf.Configuration;
039import org.apache.hadoop.hbase.HBaseClassTestRule;
040import org.apache.hadoop.hbase.HBaseTestingUtility;
041import org.apache.hadoop.hbase.HConstants;
042import org.apache.hadoop.hbase.JMXListener;
043import org.apache.hadoop.hbase.ServerName;
044import org.apache.hadoop.hbase.TableName;
045import org.apache.hadoop.hbase.client.RegionInfo;
046import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
047import org.apache.hadoop.hbase.testclassification.MediumTests;
048import org.apache.hadoop.hbase.testclassification.MiscTests;
049import org.apache.hadoop.hbase.util.Threads;
050import org.apache.hadoop.net.DNSToSwitchMapping;
051import org.junit.AfterClass;
052import org.junit.BeforeClass;
053import org.junit.ClassRule;
054import org.junit.FixMethodOrder;
055import org.junit.Ignore;
056import org.junit.Test;
057import org.junit.experimental.categories.Category;
058import org.junit.runners.MethodSorters;
059import org.slf4j.Logger;
060import org.slf4j.LoggerFactory;
061
062@Category({ MiscTests.class, MediumTests.class })
063@FixMethodOrder(MethodSorters.NAME_ASCENDING)
064@Ignore
065public class TestStochasticBalancerJmxMetrics extends BalancerTestBase {
066
067  @ClassRule
068  public static final HBaseClassTestRule CLASS_RULE =
069    HBaseClassTestRule.forClass(TestStochasticBalancerJmxMetrics.class);
070
071  private static final Logger LOG = LoggerFactory.getLogger(TestStochasticBalancerJmxMetrics.class);
072  private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
073  private static int connectorPort = 61120;
074  private static StochasticLoadBalancer loadBalancer;
075  /**
076   * a simple cluster for testing JMX.
077   */
078  private static int[] mockCluster_ensemble = new int[] { 0, 1, 2, 3 };
079  private static int[] mockCluster_pertable_1 = new int[] { 0, 1, 2 };
080  private static int[] mockCluster_pertable_2 = new int[] { 3, 1, 1 };
081  private static int[] mockCluster_pertable_namespace = new int[] { 1, 3, 1 };
082
083  private static final String TABLE_NAME_1 = "Table1";
084  private static final String TABLE_NAME_2 = "Table2";
085  private static final String TABLE_NAME_NAMESPACE = "hbase:namespace";
086
087  private static Configuration conf = null;
088
089  /**
090   * Setup the environment for the test.
091   */
092  @BeforeClass
093  public static void setupBeforeClass() throws Exception {
094
095    conf = UTIL.getConfiguration();
096
097    conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class);
098    conf.setFloat("hbase.regions.slop", 0.0f);
099    conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, JMXListener.class.getName());
100    Random rand = ThreadLocalRandom.current();
101    for (int i = 0; i < 10; i++) {
102      do {
103        int sign = i % 2 == 0 ? 1 : -1;
104        connectorPort += sign * rand.nextInt(100);
105      } while (!HBaseTestingUtility.available(connectorPort));
106      try {
107        conf.setInt("regionserver.rmi.registry.port", connectorPort);
108
109        UTIL.startMiniCluster();
110        break;
111      } catch (Exception e) {
112        LOG.debug("Encountered exception when starting cluster. Trying port " + connectorPort, e);
113        try {
114          // this is to avoid "IllegalStateException: A mini-cluster is already running"
115          UTIL.shutdownMiniCluster();
116        } catch (Exception ex) {
117          LOG.debug("Encountered exception shutting down cluster", ex);
118        }
119      }
120    }
121  }
122
123  @AfterClass
124  public static void tearDownAfterClass() throws Exception {
125    UTIL.shutdownMiniCluster();
126  }
127
128  /**
129   * In Ensemble mode, there should be only one ensemble table
130   */
131  @Test
132  public void testJmxMetrics_EnsembleMode() throws Exception {
133    loadBalancer = new StochasticLoadBalancer();
134
135    conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, false);
136    loadBalancer.initialize();
137
138    TableName tableName = HConstants.ENSEMBLE_TABLE_NAME;
139    Map<ServerName, List<RegionInfo>> clusterState = mockClusterServers(mockCluster_ensemble);
140    loadBalancer.balanceTable(tableName, clusterState);
141
142    String[] tableNames = new String[] { tableName.getNameAsString() };
143    String[] functionNames = loadBalancer.getCostFunctionNames();
144    Set<String> jmxMetrics = readJmxMetricsWithRetry();
145    Set<String> expectedMetrics = getExpectedJmxMetrics(tableNames, functionNames);
146
147    // printMetrics(jmxMetrics, "existing metrics in ensemble mode");
148    // printMetrics(expectedMetrics, "expected metrics in ensemble mode");
149
150    // assert that every expected is in the JMX
151    for (String expected : expectedMetrics) {
152      assertTrue("Metric " + expected + " can not be found in JMX in ensemble mode.",
153        jmxMetrics.contains(expected));
154    }
155  }
156
157  /**
158   * In per-table mode, each table has a set of metrics
159   */
160  @Test
161  public void testJmxMetrics_PerTableMode() throws Exception {
162    loadBalancer = new StochasticLoadBalancer();
163
164    conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, true);
165    loadBalancer.initialize();
166
167    // NOTE the size is normally set in setClusterMetrics, for test purpose, we set it manually
168    // Tables: hbase:namespace, table1, table2
169    // Functions: costFunctions, overall
170    String[] functionNames = loadBalancer.getCostFunctionNames();
171    loadBalancer.updateMetricsSize(3 * (functionNames.length + 1));
172
173    // table 1
174    TableName tableName = TableName.valueOf(TABLE_NAME_1);
175    Map<ServerName, List<RegionInfo>> clusterState = mockClusterServers(mockCluster_pertable_1);
176    loadBalancer.balanceTable(tableName, clusterState);
177
178    // table 2
179    tableName = TableName.valueOf(TABLE_NAME_2);
180    clusterState = mockClusterServers(mockCluster_pertable_2);
181    loadBalancer.balanceTable(tableName, clusterState);
182
183    // table hbase:namespace
184    tableName = TableName.valueOf(TABLE_NAME_NAMESPACE);
185    clusterState = mockClusterServers(mockCluster_pertable_namespace);
186    loadBalancer.balanceTable(tableName, clusterState);
187
188    String[] tableNames = new String[] { TABLE_NAME_1, TABLE_NAME_2, TABLE_NAME_NAMESPACE };
189    Set<String> jmxMetrics = readJmxMetricsWithRetry();
190    Set<String> expectedMetrics = getExpectedJmxMetrics(tableNames, functionNames);
191
192    // printMetrics(jmxMetrics, "existing metrics in per-table mode");
193    // printMetrics(expectedMetrics, "expected metrics in per-table mode");
194
195    // assert that every expected is in the JMX
196    for (String expected : expectedMetrics) {
197      assertTrue("Metric " + expected + " can not be found in JMX in per-table mode.",
198        jmxMetrics.contains(expected));
199    }
200  }
201
202  private Set<String> readJmxMetricsWithRetry() throws IOException {
203    final int count = 0;
204    for (int i = 0; i < 10; i++) {
205      Set<String> metrics = readJmxMetrics();
206      if (metrics != null) return metrics;
207      LOG.warn("Failed to get jmxmetrics... sleeping, retrying; " + i + " of " + count + " times");
208      Threads.sleep(1000);
209    }
210    return null;
211  }
212
213  /**
214   * Read the attributes from Hadoop->HBase->Master->Balancer in JMX n
215   */
216  private Set<String> readJmxMetrics() throws IOException {
217    JMXConnector connector = null;
218    ObjectName target = null;
219    MBeanServerConnection mb = null;
220    try {
221      connector =
222        JMXConnectorFactory.connect(JMXListener.buildJMXServiceURL(connectorPort, connectorPort));
223      mb = connector.getMBeanServerConnection();
224
225      Hashtable<String, String> pairs = new Hashtable<>();
226      pairs.put("service", "HBase");
227      pairs.put("name", "Master");
228      pairs.put("sub", "Balancer");
229      target = new ObjectName("Hadoop", pairs);
230      MBeanInfo beanInfo = mb.getMBeanInfo(target);
231
232      Set<String> existingAttrs = new HashSet<>();
233      for (MBeanAttributeInfo attrInfo : beanInfo.getAttributes()) {
234        existingAttrs.add(attrInfo.getName());
235      }
236      return existingAttrs;
237    } catch (Exception e) {
238      LOG.warn("Failed to get bean!!! " + target, e);
239      if (mb != null) {
240        Set<ObjectInstance> instances = mb.queryMBeans(null, null);
241        Iterator<ObjectInstance> iterator = instances.iterator();
242        System.out.println("MBean Found:");
243        while (iterator.hasNext()) {
244          ObjectInstance instance = iterator.next();
245          System.out.println("Class Name: " + instance.getClassName());
246          System.out.println("Object Name: " + instance.getObjectName());
247        }
248      }
249    } finally {
250      if (connector != null) {
251        try {
252          connector.close();
253        } catch (Exception e) {
254          e.printStackTrace();
255        }
256      }
257    }
258    return null;
259  }
260
261  /**
262   * Given the tables and functions, return metrics names that should exist in JMX
263   */
264  private Set<String> getExpectedJmxMetrics(String[] tableNames, String[] functionNames) {
265    Set<String> ret = new HashSet<>();
266
267    for (String tableName : tableNames) {
268      ret.add(StochasticLoadBalancer.composeAttributeName(tableName, "Overall"));
269      for (String functionName : functionNames) {
270        String metricsName = StochasticLoadBalancer.composeAttributeName(tableName, functionName);
271        ret.add(metricsName);
272      }
273    }
274
275    return ret;
276  }
277}