001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.balancer;
019
020import java.io.IOException;
021import java.nio.charset.Charset;
022import java.nio.file.FileSystems;
023import java.nio.file.NoSuchFileException;
024import java.util.Arrays;
025import java.util.Collections;
026import java.util.List;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.fs.FSDataOutputStream;
029import org.apache.hadoop.fs.Path;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.HBaseTestingUtility;
032import org.apache.hadoop.hbase.testclassification.MasterTests;
033import org.apache.hadoop.hbase.testclassification.MediumTests;
034import org.apache.hadoop.hdfs.DistributedFileSystem;
035import org.apache.hadoop.hdfs.MiniDFSCluster;
036import org.junit.Assert;
037import org.junit.Before;
038import org.junit.BeforeClass;
039import org.junit.ClassRule;
040import org.junit.Rule;
041import org.junit.Test;
042import org.junit.experimental.categories.Category;
043import org.junit.rules.TestName;
044
045@Category({ MasterTests.class, MediumTests.class })
046public class TestStochasticLoadBalancerHeterogeneousCostRules extends BalancerTestBase {
047  @ClassRule
048  public static final HBaseClassTestRule CLASS_RULE =
049    HBaseClassTestRule.forClass(TestStochasticLoadBalancerHeterogeneousCostRules.class);
050  @Rule
051  public TestName name = new TestName();
052
053  static final String DEFAULT_RULES_FILE_NAME = "hbase-balancer.rules";
054  private HeterogeneousRegionCountCostFunction costFunction;
055  private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
056
057  /**
058   * Make a file for rules that is inside a temporary test dir named for the method so it doesn't
059   * clash w/ other rule files.
060   */
061  private String rulesFilename;
062
063  @BeforeClass
064  public static void beforeClass() throws IOException {
065    // Ensure test dir is created
066    HTU.getTestFileSystem().mkdirs(HTU.getDataTestDir());
067  }
068
069  @Before
070  public void before() throws IOException {
071    // New rules file name per test.
072    this.rulesFilename =
073      HTU.getDataTestDir(this.name.getMethodName() + "." + DEFAULT_RULES_FILE_NAME).toString();
074    // Set the created rules filename into the configuration.
075    HTU.getConfiguration().set(
076      HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE,
077      this.rulesFilename);
078  }
079
080  /**
081   * @param file Name of file to write rules into.
082   * @return Full file name of the rules file which is <code>dir</code> + DEFAULT_RULES_FILE_NAME.
083   */
084  static String createRulesFile(String file, final List<String> lines) throws IOException {
085    cleanup(file);
086    java.nio.file.Path path =
087      java.nio.file.Files.createFile(FileSystems.getDefault().getPath(file));
088    return java.nio.file.Files.write(path, lines, Charset.forName("UTF-8")).toString();
089  }
090
091  /**
092   * @param file Name of file to write rules into.
093   * @return Full file name of the rules file which is <code>dir</code> + DEFAULT_RULES_FILE_NAME.
094   */
095  static String createRulesFile(String file) throws IOException {
096    return createRulesFile(file, Collections.emptyList());
097  }
098
099  private static void cleanup(String file) throws IOException {
100    try {
101      java.nio.file.Files.delete(FileSystems.getDefault().getPath(file));
102    } catch (NoSuchFileException nsfe) {
103      System.out.println("FileNotFoundException for " + file);
104    }
105  }
106
107  @Test
108  public void testNoRules() throws IOException {
109    // Override what is in the configuration with the name of a non-existent file!
110    HTU.getConfiguration().set(
111      HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE,
112      "non-existent-file!");
113    this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration());
114    this.costFunction.loadRules();
115    Assert.assertEquals(0, this.costFunction.getNumberOfRulesLoaded());
116  }
117
118  @Test
119  public void testBadFormatInRules() throws IOException {
120    // See {@link #before} above. It sets this.rulesFilename, and
121    // HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE,
122    // in the configuration.
123    this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration());
124    this.costFunction.loadRules();
125    Assert.assertEquals(0, this.costFunction.getNumberOfRulesLoaded());
126
127    createRulesFile(this.rulesFilename, Collections.singletonList("bad rules format"));
128    this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration());
129    this.costFunction.loadRules();
130    Assert.assertEquals(0, this.costFunction.getNumberOfRulesLoaded());
131
132    createRulesFile(this.rulesFilename, Arrays.asList("srv[1-2] 10", "bad_rules format", "a"));
133    this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration());
134    this.costFunction.loadRules();
135    Assert.assertEquals(1, this.costFunction.getNumberOfRulesLoaded());
136  }
137
138  @Test
139  public void testTwoRules() throws IOException {
140    // See {@link #before} above. It sets this.rulesFilename, and
141    // HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE,
142    // in the configuration.
143    // See {@link #before} above. It sets
144    // HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE,
145    // in the configuration.
146    createRulesFile(this.rulesFilename, Arrays.asList("^server1$ 10", "^server2 21"));
147    this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration());
148    this.costFunction.loadRules();
149    Assert.assertEquals(2, this.costFunction.getNumberOfRulesLoaded());
150  }
151
152  @Test
153  public void testBadRegexp() throws IOException {
154    // See {@link #before} above. It sets this.rulesFilename, and
155    // HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE,
156    // in the configuration.
157    // See {@link #before} above. It sets
158    // HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE,
159    // in the configuration.
160    createRulesFile(this.rulesFilename, Collections.singletonList("server[ 1"));
161    this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration());
162    this.costFunction.loadRules();
163    Assert.assertEquals(0, this.costFunction.getNumberOfRulesLoaded());
164  }
165
166  @Test
167  public void testNoOverride() throws IOException {
168    // See {@link #before} above. It sets this.rulesFilename, and
169    // HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE,
170    // in the configuration.
171    createRulesFile(this.rulesFilename, Arrays.asList("^server1$ 10", "^server2 21"));
172    this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration());
173    this.costFunction.loadRules();
174    Assert.assertEquals(2, this.costFunction.getNumberOfRulesLoaded());
175
176    // loading malformed configuration does not overload current
177    cleanup(this.rulesFilename);
178    this.costFunction.loadRules();
179    Assert.assertEquals(2, this.costFunction.getNumberOfRulesLoaded());
180  }
181
182  @Test
183  public void testLoadingFomHDFS() throws Exception {
184    HTU.startMiniDFSCluster(3);
185    try {
186      MiniDFSCluster cluster = HTU.getDFSCluster();
187      DistributedFileSystem fs = cluster.getFileSystem();
188      // Writing file
189      Path path = new Path(fs.getHomeDirectory(), DEFAULT_RULES_FILE_NAME);
190      FSDataOutputStream stream = fs.create(path);
191      stream.write("server1 10".getBytes());
192      stream.flush();
193      stream.close();
194
195      Configuration configuration = HTU.getConfiguration();
196
197      // start costFunction
198      configuration.set(
199        HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE,
200        path.toString());
201      this.costFunction = new HeterogeneousRegionCountCostFunction(configuration);
202      this.costFunction.loadRules();
203      Assert.assertEquals(1, this.costFunction.getNumberOfRulesLoaded());
204    } finally {
205      HTU.shutdownMiniCluster();
206    }
207  }
208}