001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase;
019
020import static org.junit.Assert.assertTrue;
021
022import java.io.IOException;
023import java.util.HashSet;
024import java.util.Hashtable;
025import java.util.Iterator;
026import java.util.List;
027import java.util.Map;
028import java.util.Random;
029import java.util.Set;
030import javax.management.MBeanAttributeInfo;
031import javax.management.MBeanInfo;
032import javax.management.MBeanServerConnection;
033import javax.management.ObjectInstance;
034import javax.management.ObjectName;
035import javax.management.remote.JMXConnector;
036import javax.management.remote.JMXConnectorFactory;
037import org.apache.hadoop.conf.Configuration;
038import org.apache.hadoop.hbase.client.RegionInfo;
039import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
040import org.apache.hadoop.hbase.master.balancer.BalancerTestBase;
041import org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer;
042import org.apache.hadoop.hbase.testclassification.MediumTests;
043import org.apache.hadoop.hbase.testclassification.MiscTests;
044import org.apache.hadoop.hbase.util.Threads;
045import org.apache.hadoop.net.DNSToSwitchMapping;
046import org.junit.AfterClass;
047import org.junit.BeforeClass;
048import org.junit.ClassRule;
049import org.junit.FixMethodOrder;
050import org.junit.Ignore;
051import org.junit.Test;
052import org.junit.experimental.categories.Category;
053import org.junit.runners.MethodSorters;
054import org.slf4j.Logger;
055import org.slf4j.LoggerFactory;
056
057@Category({ MiscTests.class, MediumTests.class })
058@FixMethodOrder(MethodSorters.NAME_ASCENDING)
059@Ignore
060public class TestStochasticBalancerJmxMetrics extends BalancerTestBase {
061
062  @ClassRule
063  public static final HBaseClassTestRule CLASS_RULE =
064      HBaseClassTestRule.forClass(TestStochasticBalancerJmxMetrics.class);
065
066  private static final Logger LOG = LoggerFactory.getLogger(TestStochasticBalancerJmxMetrics.class);
067  private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
068  private static int connectorPort = 61120;
069  private static StochasticLoadBalancer loadBalancer;
070  /**
071   * a simple cluster for testing JMX.
072   */
073  private static int[] mockCluster_ensemble = new int[] { 0, 1, 2, 3 };
074  private static int[] mockCluster_pertable_1 = new int[] { 0, 1, 2 };
075  private static int[] mockCluster_pertable_2 = new int[] { 3, 1, 1 };
076  private static int[] mockCluster_pertable_namespace = new int[] { 1, 3, 1 };
077
078  private static final String TABLE_NAME_1 = "Table1";
079  private static final String TABLE_NAME_2 = "Table2";
080  private static final String TABLE_NAME_NAMESPACE = "hbase:namespace";
081
082  private static Configuration conf = null;
083
084  /**
085   * Setup the environment for the test.
086   */
087  @BeforeClass
088  public static void setupBeforeClass() throws Exception {
089
090    conf = UTIL.getConfiguration();
091
092    conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class);
093    conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 0.75f);
094    conf.setFloat("hbase.regions.slop", 0.0f);
095    conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, JMXListener.class.getName());
096    Random rand = new Random();
097    for (int i = 0; i < 10; i++) {
098      do {
099        int sign = i % 2 == 0 ? 1 : -1;
100        connectorPort += sign * rand.nextInt(100);
101      } while (!HBaseTestingUtility.available(connectorPort));
102      try {
103        conf.setInt("regionserver.rmi.registry.port", connectorPort);
104
105        UTIL.startMiniCluster();
106        break;
107      } catch (Exception e) {
108        LOG.debug("Encountered exception when starting cluster. Trying port " + connectorPort, e);
109        try {
110          // this is to avoid "IllegalStateException: A mini-cluster is already running"
111          UTIL.shutdownMiniCluster();
112        } catch (Exception ex) {
113          LOG.debug("Encountered exception shutting down cluster", ex);
114        }
115      }
116    }
117  }
118
119  @AfterClass
120  public static void tearDownAfterClass() throws Exception {
121    UTIL.shutdownMiniCluster();
122  }
123
124  /**
125   * In Ensemble mode, there should be only one ensemble table
126   */
127  @Test
128  public void testJmxMetrics_EnsembleMode() throws Exception {
129    loadBalancer = new StochasticLoadBalancer();
130
131    conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, false);
132    loadBalancer.setConf(conf);
133
134    TableName tableName = HConstants.ENSEMBLE_TABLE_NAME;
135    Map<ServerName, List<RegionInfo>> clusterState = mockClusterServers(mockCluster_ensemble);
136    loadBalancer.balanceCluster(tableName, clusterState);
137
138    String[] tableNames = new String[] { tableName.getNameAsString() };
139    String[] functionNames = loadBalancer.getCostFunctionNames();
140    Set<String> jmxMetrics = readJmxMetricsWithRetry();
141    Set<String> expectedMetrics = getExpectedJmxMetrics(tableNames, functionNames);
142
143    // printMetrics(jmxMetrics, "existing metrics in ensemble mode");
144    // printMetrics(expectedMetrics, "expected metrics in ensemble mode");
145
146    // assert that every expected is in the JMX
147    for (String expected : expectedMetrics) {
148      assertTrue("Metric " + expected + " can not be found in JMX in ensemble mode.",
149        jmxMetrics.contains(expected));
150    }
151  }
152
153  /**
154   * In per-table mode, each table has a set of metrics
155   */
156  @Test
157  public void testJmxMetrics_PerTableMode() throws Exception {
158    loadBalancer = new StochasticLoadBalancer();
159
160    conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, true);
161    loadBalancer.setConf(conf);
162
163    // NOTE the size is normally set in setClusterMetrics, for test purpose, we set it manually
164    // Tables: hbase:namespace, table1, table2
165    // Functions: costFunctions, overall
166    String[] functionNames = loadBalancer.getCostFunctionNames();
167    loadBalancer.updateMetricsSize(3 * (functionNames.length + 1));
168
169    // table 1
170    TableName tableName = TableName.valueOf(TABLE_NAME_1);
171    Map<ServerName, List<RegionInfo>> clusterState = mockClusterServers(mockCluster_pertable_1);
172    loadBalancer.balanceCluster(tableName, clusterState);
173
174    // table 2
175    tableName = TableName.valueOf(TABLE_NAME_2);
176    clusterState = mockClusterServers(mockCluster_pertable_2);
177    loadBalancer.balanceCluster(tableName, clusterState);
178
179    // table hbase:namespace
180    tableName = TableName.valueOf(TABLE_NAME_NAMESPACE);
181    clusterState = mockClusterServers(mockCluster_pertable_namespace);
182    loadBalancer.balanceCluster(tableName, clusterState);
183
184    String[] tableNames = new String[] { TABLE_NAME_1, TABLE_NAME_2, TABLE_NAME_NAMESPACE };
185    Set<String> jmxMetrics = readJmxMetricsWithRetry();
186    Set<String> expectedMetrics = getExpectedJmxMetrics(tableNames, functionNames);
187
188    // printMetrics(jmxMetrics, "existing metrics in per-table mode");
189    // printMetrics(expectedMetrics, "expected metrics in per-table mode");
190
191    // assert that every expected is in the JMX
192    for (String expected : expectedMetrics) {
193      assertTrue("Metric " + expected + " can not be found in JMX in per-table mode.",
194        jmxMetrics.contains(expected));
195    }
196  }
197
198  private Set<String> readJmxMetricsWithRetry() throws IOException {
199    final int count = 0;
200    for (int i = 0; i < 10; i++) {
201      Set<String> metrics = readJmxMetrics();
202      if (metrics != null) return metrics;
203      LOG.warn("Failed to get jmxmetrics... sleeping, retrying; " + i + " of " + count + " times");
204      Threads.sleep(1000);
205    }
206    return null;
207  }
208
209  /**
210   * Read the attributes from Hadoop->HBase->Master->Balancer in JMX
211   * @throws IOException
212   */
213  private Set<String> readJmxMetrics() throws IOException {
214    JMXConnector connector = null;
215    ObjectName target = null;
216    MBeanServerConnection mb = null;
217    try {
218      connector =
219          JMXConnectorFactory.connect(JMXListener.buildJMXServiceURL(connectorPort, connectorPort));
220      mb = connector.getMBeanServerConnection();
221
222      Hashtable<String, String> pairs = new Hashtable<>();
223      pairs.put("service", "HBase");
224      pairs.put("name", "Master");
225      pairs.put("sub", "Balancer");
226      target = new ObjectName("Hadoop", pairs);
227      MBeanInfo beanInfo = mb.getMBeanInfo(target);
228
229      Set<String> existingAttrs = new HashSet<>();
230      for (MBeanAttributeInfo attrInfo : beanInfo.getAttributes()) {
231        existingAttrs.add(attrInfo.getName());
232      }
233      return existingAttrs;
234    } catch (Exception e) {
235      LOG.warn("Failed to get bean!!! " + target, e);
236      if (mb != null) {
237        Set<ObjectInstance> instances = mb.queryMBeans(null, null);
238        Iterator<ObjectInstance> iterator = instances.iterator();
239        System.out.println("MBean Found:");
240        while (iterator.hasNext()) {
241          ObjectInstance instance = iterator.next();
242          System.out.println("Class Name: " + instance.getClassName());
243          System.out.println("Object Name: " + instance.getObjectName());
244        }
245      }
246    } finally {
247      if (connector != null) {
248        try {
249          connector.close();
250        } catch (Exception e) {
251          e.printStackTrace();
252        }
253      }
254    }
255    return null;
256  }
257
258  /**
259   * Given the tables and functions, return metrics names that should exist in JMX
260   */
261  private Set<String> getExpectedJmxMetrics(String[] tableNames, String[] functionNames) {
262    Set<String> ret = new HashSet<>();
263
264    for (String tableName : tableNames) {
265      ret.add(StochasticLoadBalancer.composeAttributeName(tableName, "Overall"));
266      for (String functionName : functionNames) {
267        String metricsName = StochasticLoadBalancer.composeAttributeName(tableName, functionName);
268        ret.add(metricsName);
269      }
270    }
271
272    return ret;
273  }
274
275  private static void printMetrics(Set<String> metrics, String info) {
276    if (null != info) LOG.info("++++ ------ " + info + " ------");
277
278    LOG.info("++++ metrics count = " + metrics.size());
279    for (String str : metrics) {
280      LOG.info(" ++++ " + str);
281    }
282  }
283}