001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.balancer;
019
020import static org.junit.Assert.assertTrue;
021
022import java.io.IOException;
023import java.util.HashSet;
024import java.util.Hashtable;
025import java.util.Iterator;
026import java.util.List;
027import java.util.Map;
028import java.util.Random;
029import java.util.Set;
030import javax.management.MBeanAttributeInfo;
031import javax.management.MBeanInfo;
032import javax.management.MBeanServerConnection;
033import javax.management.ObjectInstance;
034import javax.management.ObjectName;
035import javax.management.remote.JMXConnector;
036import javax.management.remote.JMXConnectorFactory;
037import org.apache.hadoop.conf.Configuration;
038import org.apache.hadoop.hbase.HBaseClassTestRule;
039import org.apache.hadoop.hbase.HBaseTestingUtil;
040import org.apache.hadoop.hbase.HConstants;
041import org.apache.hadoop.hbase.JMXListener;
042import org.apache.hadoop.hbase.ServerName;
043import org.apache.hadoop.hbase.TableName;
044import org.apache.hadoop.hbase.client.RegionInfo;
045import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
046import org.apache.hadoop.hbase.testclassification.MediumTests;
047import org.apache.hadoop.hbase.testclassification.MiscTests;
048import org.apache.hadoop.hbase.util.Threads;
049import org.apache.hadoop.net.DNSToSwitchMapping;
050import org.junit.AfterClass;
051import org.junit.BeforeClass;
052import org.junit.ClassRule;
053import org.junit.FixMethodOrder;
054import org.junit.Ignore;
055import org.junit.Test;
056import org.junit.experimental.categories.Category;
057import org.junit.runners.MethodSorters;
058import org.slf4j.Logger;
059import org.slf4j.LoggerFactory;
060
061@Category({ MiscTests.class, MediumTests.class })
062@FixMethodOrder(MethodSorters.NAME_ASCENDING)
063@Ignore
064public class TestStochasticBalancerJmxMetrics extends BalancerTestBase {
065
066  @ClassRule
067  public static final HBaseClassTestRule CLASS_RULE =
068      HBaseClassTestRule.forClass(TestStochasticBalancerJmxMetrics.class);
069
070  private static final Logger LOG = LoggerFactory.getLogger(TestStochasticBalancerJmxMetrics.class);
071  private static HBaseTestingUtil UTIL = new HBaseTestingUtil();
072  private static int connectorPort = 61120;
073  private static StochasticLoadBalancer loadBalancer;
074  /**
075   * a simple cluster for testing JMX.
076   */
077  private static int[] mockCluster_ensemble = new int[] { 0, 1, 2, 3 };
078  private static int[] mockCluster_pertable_1 = new int[] { 0, 1, 2 };
079  private static int[] mockCluster_pertable_2 = new int[] { 3, 1, 1 };
080  private static int[] mockCluster_pertable_namespace = new int[] { 1, 3, 1 };
081
082  private static final String TABLE_NAME_1 = "Table1";
083  private static final String TABLE_NAME_2 = "Table2";
084  private static final String TABLE_NAME_NAMESPACE = "hbase:namespace";
085
086  private static Configuration conf = null;
087
088  /**
089   * Setup the environment for the test.
090   */
091  @BeforeClass
092  public static void setupBeforeClass() throws Exception {
093
094    conf = UTIL.getConfiguration();
095
096    conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class);
097    conf.setFloat("hbase.regions.slop", 0.0f);
098    conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, JMXListener.class.getName());
099    Random rand = new Random();
100    for (int i = 0; i < 10; i++) {
101      do {
102        int sign = i % 2 == 0 ? 1 : -1;
103        connectorPort += sign * rand.nextInt(100);
104      } while (!HBaseTestingUtil.available(connectorPort));
105      try {
106        conf.setInt("regionserver.rmi.registry.port", connectorPort);
107
108        UTIL.startMiniCluster();
109        break;
110      } catch (Exception e) {
111        LOG.debug("Encountered exception when starting cluster. Trying port " + connectorPort, e);
112        try {
113          // this is to avoid "IllegalStateException: A mini-cluster is already running"
114          UTIL.shutdownMiniCluster();
115        } catch (Exception ex) {
116          LOG.debug("Encountered exception shutting down cluster", ex);
117        }
118      }
119    }
120  }
121
122  @AfterClass
123  public static void tearDownAfterClass() throws Exception {
124    UTIL.shutdownMiniCluster();
125  }
126
127  /**
128   * In Ensemble mode, there should be only one ensemble table
129   */
130  @Test
131  public void testJmxMetrics_EnsembleMode() throws Exception {
132    loadBalancer = new StochasticLoadBalancer();
133
134    conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, false);
135    loadBalancer.setClusterInfoProvider(new DummyClusterInfoProvider(conf));
136    loadBalancer.initialize();
137
138    TableName tableName = HConstants.ENSEMBLE_TABLE_NAME;
139    Map<ServerName, List<RegionInfo>> clusterState = mockClusterServers(mockCluster_ensemble);
140    loadBalancer.balanceTable(tableName, clusterState);
141
142    String[] tableNames = new String[] { tableName.getNameAsString() };
143    String[] functionNames = loadBalancer.getCostFunctionNames();
144    Set<String> jmxMetrics = readJmxMetricsWithRetry();
145    Set<String> expectedMetrics = getExpectedJmxMetrics(tableNames, functionNames);
146
147    // printMetrics(jmxMetrics, "existing metrics in ensemble mode");
148    // printMetrics(expectedMetrics, "expected metrics in ensemble mode");
149
150    // assert that every expected is in the JMX
151    for (String expected : expectedMetrics) {
152      assertTrue("Metric " + expected + " can not be found in JMX in ensemble mode.",
153        jmxMetrics.contains(expected));
154    }
155  }
156
157  /**
158   * In per-table mode, each table has a set of metrics
159   */
160  @Test
161  public void testJmxMetrics_PerTableMode() throws Exception {
162    loadBalancer = new StochasticLoadBalancer();
163
164    conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, true);
165    loadBalancer.setClusterInfoProvider(new DummyClusterInfoProvider(conf));
166    loadBalancer.initialize();
167
168    // NOTE the size is normally set in setClusterMetrics, for test purpose, we set it manually
169    // Tables: hbase:namespace, table1, table2
170    // Functions: costFunctions, overall
171    String[] functionNames = loadBalancer.getCostFunctionNames();
172    loadBalancer.updateMetricsSize(3 * (functionNames.length + 1));
173
174    // table 1
175    TableName tableName = TableName.valueOf(TABLE_NAME_1);
176    Map<ServerName, List<RegionInfo>> clusterState = mockClusterServers(mockCluster_pertable_1);
177    loadBalancer.balanceTable(tableName, clusterState);
178
179    // table 2
180    tableName = TableName.valueOf(TABLE_NAME_2);
181    clusterState = mockClusterServers(mockCluster_pertable_2);
182    loadBalancer.balanceTable(tableName, clusterState);
183
184    // table hbase:namespace
185    tableName = TableName.valueOf(TABLE_NAME_NAMESPACE);
186    clusterState = mockClusterServers(mockCluster_pertable_namespace);
187    loadBalancer.balanceTable(tableName, clusterState);
188
189    String[] tableNames = new String[] { TABLE_NAME_1, TABLE_NAME_2, TABLE_NAME_NAMESPACE };
190    Set<String> jmxMetrics = readJmxMetricsWithRetry();
191    Set<String> expectedMetrics = getExpectedJmxMetrics(tableNames, functionNames);
192
193    // printMetrics(jmxMetrics, "existing metrics in per-table mode");
194    // printMetrics(expectedMetrics, "expected metrics in per-table mode");
195
196    // assert that every expected is in the JMX
197    for (String expected : expectedMetrics) {
198      assertTrue("Metric " + expected + " can not be found in JMX in per-table mode.",
199        jmxMetrics.contains(expected));
200    }
201  }
202
203  private Set<String> readJmxMetricsWithRetry() throws IOException {
204    final int count = 0;
205    for (int i = 0; i < 10; i++) {
206      Set<String> metrics = readJmxMetrics();
207      if (metrics != null) return metrics;
208      LOG.warn("Failed to get jmxmetrics... sleeping, retrying; " + i + " of " + count + " times");
209      Threads.sleep(1000);
210    }
211    return null;
212  }
213
214  /**
215   * Read the attributes from Hadoop->HBase->Master->Balancer in JMX
216   * @throws IOException
217   */
218  private Set<String> readJmxMetrics() throws IOException {
219    JMXConnector connector = null;
220    ObjectName target = null;
221    MBeanServerConnection mb = null;
222    try {
223      connector =
224          JMXConnectorFactory.connect(JMXListener.buildJMXServiceURL(connectorPort, connectorPort));
225      mb = connector.getMBeanServerConnection();
226
227      Hashtable<String, String> pairs = new Hashtable<>();
228      pairs.put("service", "HBase");
229      pairs.put("name", "Master");
230      pairs.put("sub", "Balancer");
231      target = new ObjectName("Hadoop", pairs);
232      MBeanInfo beanInfo = mb.getMBeanInfo(target);
233
234      Set<String> existingAttrs = new HashSet<>();
235      for (MBeanAttributeInfo attrInfo : beanInfo.getAttributes()) {
236        existingAttrs.add(attrInfo.getName());
237      }
238      return existingAttrs;
239    } catch (Exception e) {
240      LOG.warn("Failed to get bean!!! " + target, e);
241      if (mb != null) {
242        Set<ObjectInstance> instances = mb.queryMBeans(null, null);
243        Iterator<ObjectInstance> iterator = instances.iterator();
244        System.out.println("MBean Found:");
245        while (iterator.hasNext()) {
246          ObjectInstance instance = iterator.next();
247          System.out.println("Class Name: " + instance.getClassName());
248          System.out.println("Object Name: " + instance.getObjectName());
249        }
250      }
251    } finally {
252      if (connector != null) {
253        try {
254          connector.close();
255        } catch (Exception e) {
256          e.printStackTrace();
257        }
258      }
259    }
260    return null;
261  }
262
263  /**
264   * Given the tables and functions, return metrics names that should exist in JMX
265   */
266  private Set<String> getExpectedJmxMetrics(String[] tableNames, String[] functionNames) {
267    Set<String> ret = new HashSet<>();
268
269    for (String tableName : tableNames) {
270      ret.add(StochasticLoadBalancer.composeAttributeName(tableName, "Overall"));
271      for (String functionName : functionNames) {
272        String metricsName = StochasticLoadBalancer.composeAttributeName(tableName, functionName);
273        ret.add(metricsName);
274      }
275    }
276
277    return ret;
278  }
279}