001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
003 * agreements. See the NOTICE file distributed with this work for additional information regarding
004 * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
005 * "License"); you may not use this file except in compliance with the License. You may obtain a
006 * copy of the License at
007 * <p>
008 * http://www.apache.org/licenses/LICENSE-2.0
009 * <p>
010 * Unless required by applicable law or agreed to in writing, software distributed under the License
011 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
012 * or implied. See the License for the specific language governing permissions and limitations under
013 * the License.
014 */
015package org.apache.hadoop.hbase.master.balancer;
016
017import java.io.IOException;
018import java.nio.charset.Charset;
019import java.nio.file.FileSystems;
020import java.nio.file.NoSuchFileException;
021import java.util.Arrays;
022import java.util.Collections;
023import java.util.List;
024import org.apache.hadoop.conf.Configuration;
025import org.apache.hadoop.fs.FSDataOutputStream;
026import org.apache.hadoop.fs.Path;
027import org.apache.hadoop.hbase.HBaseClassTestRule;
028import org.apache.hadoop.hbase.HBaseTestingUtility;
029import org.apache.hadoop.hbase.testclassification.MasterTests;
030import org.apache.hadoop.hbase.testclassification.MediumTests;
031import org.apache.hadoop.hdfs.DistributedFileSystem;
032import org.apache.hadoop.hdfs.MiniDFSCluster;
033import org.junit.Assert;
034import org.junit.Before;
035import org.junit.BeforeClass;
036import org.junit.ClassRule;
037import org.junit.Rule;
038import org.junit.Test;
039import org.junit.experimental.categories.Category;
040import org.junit.rules.TestName;
041
042import static junit.framework.TestCase.assertTrue;
043
044@Category({ MasterTests.class, MediumTests.class })
045public class TestStochasticLoadBalancerHeterogeneousCostRules extends BalancerTestBase {
046  @ClassRule
047  public static final HBaseClassTestRule CLASS_RULE =
048      HBaseClassTestRule.forClass(TestStochasticLoadBalancerHeterogeneousCostRules.class);
049  @Rule
050  public TestName name = new TestName();
051
052  static final String DEFAULT_RULES_FILE_NAME = "hbase-balancer.rules";
053  private HeterogeneousRegionCountCostFunction costFunction;
054  private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
055
056  /**
057   * Make a file for rules that is inside a temporary test dir named for the method so it doesn't
058   * clash w/ other rule files.
059   */
060  private String rulesFilename;
061
062  @BeforeClass
063  public static void beforeClass() throws IOException {
064    // Ensure test dir is created
065    HTU.getTestFileSystem().mkdirs(HTU.getDataTestDir());
066  }
067
068  @Before
069  public void before() throws IOException {
070    // New rules file name per test.
071    this.rulesFilename = HTU.getDataTestDir(
072      this.name.getMethodName() + "." + DEFAULT_RULES_FILE_NAME).toString();
073    // Set the created rules filename into the configuration.
074    HTU.getConfiguration().set(
075      HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE,
076      this.rulesFilename);
077  }
078
079  /**
080   * @param file Name of file to write rules into.
081   * @return Full file name of the rules file which is <code>dir</code> + DEFAULT_RULES_FILE_NAME.
082   */
083  static String createRulesFile(String file, final List<String> lines) throws IOException {
084    cleanup(file);
085    java.nio.file.Path path =
086      java.nio.file.Files.createFile(FileSystems.getDefault().getPath(file));
087    return java.nio.file.Files.write(path, lines, Charset.forName("UTF-8")).toString();
088  }
089
090  /**
091   * @param file Name of file to write rules into.
092   * @return Full file name of the rules file which is <code>dir</code> + DEFAULT_RULES_FILE_NAME.
093   */
094  static String createRulesFile(String file) throws IOException {
095    return createRulesFile(file, Collections.emptyList());
096  }
097
098  private static void cleanup(String file) throws IOException {
099    try {
100      java.nio.file.Files.delete(FileSystems.getDefault().getPath(file));
101    } catch (NoSuchFileException nsfe) {
102      System.out.println("FileNotFoundException for " + file);
103    }
104  }
105
106  @Test
107  public void testNoRules() throws IOException {
108    // Override what is in the configuration with the name of a non-existent file!
109    HTU.getConfiguration().set(
110      HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE,
111      "non-existent-file!");
112    this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration());
113    this.costFunction.loadRules();
114    Assert.assertEquals(0, this.costFunction.getNumberOfRulesLoaded());
115  }
116
117  @Test
118  public void testBadFormatInRules() throws IOException {
119    // See {@link #before} above. It sets this.rulesFilename, and
120    // HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE,
121    // in the configuration.
122    this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration());
123    this.costFunction.loadRules();
124    Assert.assertEquals(0, this.costFunction.getNumberOfRulesLoaded());
125
126    createRulesFile(this.rulesFilename, Collections.singletonList("bad rules format"));
127    this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration());
128    this.costFunction.loadRules();
129    Assert.assertEquals(0, this.costFunction.getNumberOfRulesLoaded());
130
131    createRulesFile(this.rulesFilename, Arrays.asList("srv[1-2] 10",
132      "bad_rules format", "a"));
133    this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration());
134    this.costFunction.loadRules();
135    Assert.assertEquals(1, this.costFunction.getNumberOfRulesLoaded());
136  }
137
138  @Test
139  public void testTwoRules() throws IOException {
140    // See {@link #before} above. It sets this.rulesFilename, and
141    // HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE,
142    // in the configuration.
143    // See {@link #before} above. It sets
144    // HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE,
145    // in the configuration.
146    createRulesFile(this.rulesFilename, Arrays.asList("^server1$ 10", "^server2 21"));
147    this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration());
148    this.costFunction.loadRules();
149    Assert.assertEquals(2, this.costFunction.getNumberOfRulesLoaded());
150  }
151
152  @Test
153  public void testBadRegexp() throws IOException {
154    // See {@link #before} above. It sets this.rulesFilename, and
155    // HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE,
156    // in the configuration.
157    // See {@link #before} above. It sets
158    // HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE,
159    // in the configuration.
160    createRulesFile(this.rulesFilename, Collections.singletonList("server[ 1"));
161    this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration());
162    this.costFunction.loadRules();
163    Assert.assertEquals(0, this.costFunction.getNumberOfRulesLoaded());
164  }
165
166  @Test
167  public void testNoOverride() throws IOException {
168    // See {@link #before} above. It sets this.rulesFilename, and
169    // HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE,
170    // in the configuration.
171    createRulesFile(this.rulesFilename, Arrays.asList("^server1$ 10", "^server2 21"));
172    this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration());
173    this.costFunction.loadRules();
174    Assert.assertEquals(2, this.costFunction.getNumberOfRulesLoaded());
175
176    // loading malformed configuration does not overload current
177    cleanup(this.rulesFilename);
178    this.costFunction.loadRules();
179    Assert.assertEquals(2, this.costFunction.getNumberOfRulesLoaded());
180  }
181
182  @Test
183  public void testLoadingFomHDFS() throws Exception {
184    HTU.startMiniDFSCluster(3);
185    try {
186      MiniDFSCluster cluster = HTU.getDFSCluster();
187      DistributedFileSystem fs = cluster.getFileSystem();
188      // Writing file
189      Path path = new Path(fs.getHomeDirectory(), DEFAULT_RULES_FILE_NAME);
190      FSDataOutputStream stream = fs.create(path);
191      stream.write("server1 10".getBytes());
192      stream.flush();
193      stream.close();
194
195      Configuration configuration = HTU.getConfiguration();
196
197      // start costFunction
198      configuration.set(
199        HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE,
200        path.toString());
201      this.costFunction = new HeterogeneousRegionCountCostFunction(configuration);
202      this.costFunction.loadRules();
203      Assert.assertEquals(1, this.costFunction.getNumberOfRulesLoaded());
204    } finally {
205      HTU.shutdownMiniCluster();
206    }
207  }
208}