001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import java.util.concurrent.TimeUnit;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.HBaseTestingUtil;
030import org.apache.hadoop.hbase.HConstants;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.Waiter;
033import org.apache.hadoop.hbase.client.Admin;
034import org.apache.hadoop.hbase.client.Connection;
035import org.apache.hadoop.hbase.client.ConnectionFactory;
036import org.apache.hadoop.hbase.client.RegionInfo;
037import org.apache.hadoop.hbase.client.RegionLocator;
038import org.apache.hadoop.hbase.client.Table;
039import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
040import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
041import org.apache.hadoop.hbase.testclassification.MediumTests;
042import org.apache.hadoop.hbase.util.Bytes;
043import org.junit.AfterClass;
044import org.junit.Before;
045import org.junit.BeforeClass;
046import org.junit.ClassRule;
047import org.junit.Test;
048import org.junit.experimental.categories.Category;
049import org.slf4j.Logger;
050import org.slf4j.LoggerFactory;
051
052/**
053 * Verify that the Online config Changes on the HRegionServer side are actually
054 * happening. We should add tests for important configurations which will be
055 * changed online.
056 */
057
058@Category({MediumTests.class})
059public class TestRegionServerOnlineConfigChange {
060
061  @ClassRule
062  public static final HBaseClassTestRule CLASS_RULE =
063      HBaseClassTestRule.forClass(TestRegionServerOnlineConfigChange.class);
064
065  private static final Logger LOG =
066          LoggerFactory.getLogger(TestRegionServerOnlineConfigChange.class.getName());
067  private static final long WAIT_TIMEOUT = TimeUnit.MINUTES.toMillis(2);
068  private static HBaseTestingUtil hbaseTestingUtility = new HBaseTestingUtil();
069  private static Configuration conf = null;
070
071  private static Table t1 = null;
072  private static HRegionServer rs1 = null;
073  private static byte[] r1name = null;
074  private static Region r1 = null;
075
076  private final static String table1Str = "table1";
077  private final static String columnFamily1Str = "columnFamily1";
078  private final static TableName TABLE1 = TableName.valueOf(table1Str);
079  private final static byte[] COLUMN_FAMILY1 = Bytes.toBytes(columnFamily1Str);
080  private final static long MAX_FILE_SIZE = 20 * 1024 * 1024L;
081
082
083  @BeforeClass
084  public static void setUpBeforeClass() throws Exception {
085    conf = hbaseTestingUtility.getConfiguration();
086    hbaseTestingUtility.startMiniCluster(2);
087    t1 = hbaseTestingUtility.createTable(
088      TableDescriptorBuilder.newBuilder(TABLE1).setMaxFileSize(MAX_FILE_SIZE).build(),
089      new byte[][] { COLUMN_FAMILY1 }, conf);
090  }
091
092  @AfterClass
093  public static void tearDown() throws Exception {
094    hbaseTestingUtility.shutdownMiniCluster();
095  }
096
097  @Before
098  public void setUp() throws Exception {
099    try (RegionLocator locator = hbaseTestingUtility.getConnection().getRegionLocator(TABLE1)) {
100      RegionInfo firstHRI = locator.getAllRegionLocations().get(0).getRegion();
101      r1name = firstHRI.getRegionName();
102      rs1 = hbaseTestingUtility.getHBaseCluster().getRegionServer(
103        hbaseTestingUtility.getHBaseCluster().getServerWith(r1name));
104      r1 = rs1.getRegion(r1name);
105    }
106  }
107
108  /**
109   * Check if the number of compaction threads changes online
110   */
111  @Test
112  public void testNumCompactionThreadsOnlineChange() {
113    assertNotNull(rs1.getCompactSplitThread());
114    int newNumSmallThreads =
115      rs1.getCompactSplitThread().getSmallCompactionThreadNum() + 1;
116    int newNumLargeThreads =
117      rs1.getCompactSplitThread().getLargeCompactionThreadNum() + 1;
118
119    conf.setInt("hbase.regionserver.thread.compaction.small",
120      newNumSmallThreads);
121    conf.setInt("hbase.regionserver.thread.compaction.large",
122      newNumLargeThreads);
123    rs1.getConfigurationManager().notifyAllObservers(conf);
124
125    assertEquals(newNumSmallThreads,
126      rs1.getCompactSplitThread().getSmallCompactionThreadNum());
127    assertEquals(newNumLargeThreads,
128      rs1.getCompactSplitThread().getLargeCompactionThreadNum());
129  }
130
131  /**
132   * Test that the configurations in the CompactionConfiguration class change
133   * properly.
134   *
135   * @throws IOException
136   */
137  @Test
138  public void testCompactionConfigurationOnlineChange() throws IOException {
139    String strPrefix = "hbase.hstore.compaction.";
140    Store s = r1.getStore(COLUMN_FAMILY1);
141    if (!(s instanceof HStore)) {
142      LOG.error("Can't test the compaction configuration of HStore class. "
143          + "Got a different implementation other than HStore");
144      return;
145    }
146    HStore hstore = (HStore)s;
147
148    // Set the new compaction ratio to a different value.
149    double newCompactionRatio =
150            hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatio() + 0.1;
151    conf.setFloat(strPrefix + "ratio", (float)newCompactionRatio);
152
153    // Notify all the observers, which includes the Store object.
154    rs1.getConfigurationManager().notifyAllObservers(conf);
155
156    // Check if the compaction ratio got updated in the Compaction Configuration
157    assertEquals(newCompactionRatio,
158                 hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatio(),
159                 0.00001);
160
161    // Check if the off peak compaction ratio gets updated.
162    double newOffPeakCompactionRatio =
163        hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatioOffPeak() + 0.1;
164    conf.setFloat(strPrefix + "ratio.offpeak",
165            (float)newOffPeakCompactionRatio);
166    rs1.getConfigurationManager().notifyAllObservers(conf);
167    assertEquals(newOffPeakCompactionRatio,
168        hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatioOffPeak(),
169                 0.00001);
170
171    // Check if the throttle point gets updated.
172    long newThrottlePoint =
173        hstore.getStoreEngine().getCompactionPolicy().getConf().getThrottlePoint() + 10;
174    conf.setLong("hbase.regionserver.thread.compaction.throttle",
175                  newThrottlePoint);
176    rs1.getConfigurationManager().notifyAllObservers(conf);
177    assertEquals(newThrottlePoint,
178        hstore.getStoreEngine().getCompactionPolicy().getConf().getThrottlePoint());
179
180    // Check if the minFilesToCompact gets updated.
181    int newMinFilesToCompact =
182            hstore.getStoreEngine().getCompactionPolicy().getConf().getMinFilesToCompact() + 1;
183    conf.setLong(strPrefix + "min", newMinFilesToCompact);
184    rs1.getConfigurationManager().notifyAllObservers(conf);
185    assertEquals(newMinFilesToCompact,
186        hstore.getStoreEngine().getCompactionPolicy().getConf().getMinFilesToCompact());
187
188    // Check if the maxFilesToCompact gets updated.
189    int newMaxFilesToCompact =
190            hstore.getStoreEngine().getCompactionPolicy().getConf().getMaxFilesToCompact() + 1;
191    conf.setLong(strPrefix + "max", newMaxFilesToCompact);
192    rs1.getConfigurationManager().notifyAllObservers(conf);
193    assertEquals(newMaxFilesToCompact,
194        hstore.getStoreEngine().getCompactionPolicy().getConf().getMaxFilesToCompact());
195
196    // Check OffPeak hours is updated in an online fashion.
197    conf.setLong(CompactionConfiguration.HBASE_HSTORE_OFFPEAK_START_HOUR, 6);
198    conf.setLong(CompactionConfiguration.HBASE_HSTORE_OFFPEAK_END_HOUR, 7);
199    rs1.getConfigurationManager().notifyAllObservers(conf);
200    assertFalse(hstore.getOffPeakHours().isOffPeakHour(4));
201
202    // Check if the minCompactSize gets updated.
203    long newMinCompactSize =
204            hstore.getStoreEngine().getCompactionPolicy().getConf().getMinCompactSize() + 1;
205    conf.setLong(strPrefix + "min.size", newMinCompactSize);
206    rs1.getConfigurationManager().notifyAllObservers(conf);
207    assertEquals(newMinCompactSize,
208                 hstore.getStoreEngine().getCompactionPolicy().getConf().getMinCompactSize());
209
210    // Check if the maxCompactSize gets updated.
211    long newMaxCompactSize =
212            hstore.getStoreEngine().getCompactionPolicy().getConf().getMaxCompactSize() - 1;
213    conf.setLong(strPrefix + "max.size", newMaxCompactSize);
214    rs1.getConfigurationManager().notifyAllObservers(conf);
215    assertEquals(newMaxCompactSize,
216                 hstore.getStoreEngine().getCompactionPolicy().getConf().getMaxCompactSize());
217    // Check if the offPeakMaxCompactSize gets updated.
218    long newOffpeakMaxCompactSize =
219            hstore.getStoreEngine().getCompactionPolicy().getConf().getOffPeakMaxCompactSize() - 1;
220    conf.setLong(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY,
221      newOffpeakMaxCompactSize);
222    rs1.getConfigurationManager().notifyAllObservers(conf);
223    assertEquals(newOffpeakMaxCompactSize,
224                 hstore.getStoreEngine().getCompactionPolicy().getConf().getOffPeakMaxCompactSize());
225
226    // Check if majorCompactionPeriod gets updated.
227    long newMajorCompactionPeriod =
228            hstore.getStoreEngine().getCompactionPolicy().getConf().getMajorCompactionPeriod() + 10;
229    conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, newMajorCompactionPeriod);
230    rs1.getConfigurationManager().notifyAllObservers(conf);
231    assertEquals(newMajorCompactionPeriod,
232            hstore.getStoreEngine().getCompactionPolicy().getConf().getMajorCompactionPeriod());
233
234    // Check if majorCompactionJitter gets updated.
235    float newMajorCompactionJitter =
236        hstore.getStoreEngine().getCompactionPolicy().getConf().getMajorCompactionJitter() + 0.02F;
237    conf.setFloat("hbase.hregion.majorcompaction.jitter",
238                  newMajorCompactionJitter);
239    rs1.getConfigurationManager().notifyAllObservers(conf);
240    assertEquals(newMajorCompactionJitter,
241      hstore.getStoreEngine().getCompactionPolicy().getConf().getMajorCompactionJitter(), 0.00001);
242  }
243
244  @Test
245  public void removeClosedRegionFromConfigurationManager() throws Exception {
246    try (Connection connection = ConnectionFactory.createConnection(conf)) {
247      Admin admin = connection.getAdmin();
248      assertTrue("The open region doesn't register as a ConfigurationObserver",
249        rs1.getConfigurationManager().containsObserver(r1));
250      admin.move(r1name);
251      hbaseTestingUtility.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
252        @Override public boolean evaluate() throws Exception {
253          return rs1.getOnlineRegion(r1name) == null;
254        }
255      });
256      assertFalse("The closed region is not removed from ConfigurationManager",
257        rs1.getConfigurationManager().containsObserver(r1));
258      admin.move(r1name, rs1.getServerName());
259      hbaseTestingUtility.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
260        @Override public boolean evaluate() throws Exception {
261          return rs1.getOnlineRegion(r1name) != null;
262        }
263      });
264    }
265  }
266
267  @Test
268  public void testStoreConfigurationOnlineChange() {
269    rs1.getConfigurationManager().notifyAllObservers(conf);
270    long actualMaxFileSize = r1.getStore(COLUMN_FAMILY1).getReadOnlyConfiguration()
271        .getLong(TableDescriptorBuilder.MAX_FILESIZE, -1);
272    assertEquals(MAX_FILE_SIZE, actualMaxFileSize);
273  }
274}