001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.List;
026import java.util.Random;
027import org.apache.hadoop.hbase.HBaseClassTestRule;
028import org.apache.hadoop.hbase.HBaseTestingUtility;
029import org.apache.hadoop.hbase.TableName;
030import org.apache.hadoop.hbase.client.Admin;
031import org.apache.hadoop.hbase.client.CompactionState;
032import org.apache.hadoop.hbase.client.Put;
033import org.apache.hadoop.hbase.client.Table;
034import org.apache.hadoop.hbase.master.HMaster;
035import org.apache.hadoop.hbase.testclassification.LargeTests;
036import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests;
037import org.apache.hadoop.hbase.util.Bytes;
038import org.junit.AfterClass;
039import org.junit.BeforeClass;
040import org.junit.ClassRule;
041import org.junit.Rule;
042import org.junit.Test;
043import org.junit.experimental.categories.Category;
044import org.junit.rules.TestName;
045import org.slf4j.Logger;
046import org.slf4j.LoggerFactory;
047
048/** Unit tests to test retrieving table/region compaction state*/
049@Category({VerySlowRegionServerTests.class, LargeTests.class})
050public class TestCompactionState {
051
052  @ClassRule
053  public static final HBaseClassTestRule CLASS_RULE =
054      HBaseClassTestRule.forClass(TestCompactionState.class);
055
056  private static final Logger LOG = LoggerFactory.getLogger(TestCompactionState.class);
057  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
058  private final static Random random = new Random();
059
060  @Rule
061  public TestName name = new TestName();
062
063  @BeforeClass
064  public static void setUpBeforeClass() throws Exception {
065    TEST_UTIL.startMiniCluster();
066  }
067
068  @AfterClass
069  public static void tearDownAfterClass() throws Exception {
070    TEST_UTIL.shutdownMiniCluster();
071  }
072
073  enum StateSource {
074    ADMIN, MASTER
075  }
076
077  @Test
078  public void testMajorCompactionStateFromAdmin() throws IOException, InterruptedException {
079    compaction(name.getMethodName(), 8, CompactionState.MAJOR, false, StateSource.ADMIN);
080  }
081
082  @Test
083  public void testMinorCompactionStateFromAdmin() throws IOException, InterruptedException {
084    compaction(name.getMethodName(), 15, CompactionState.MINOR, false, StateSource.ADMIN);
085  }
086
087  @Test
088  public void testMajorCompactionOnFamilyStateFromAdmin() throws IOException, InterruptedException {
089    compaction(name.getMethodName(), 8, CompactionState.MAJOR, true, StateSource.ADMIN);
090  }
091
092  @Test
093  public void testMinorCompactionOnFamilyStateFromAdmin() throws IOException, InterruptedException {
094    compaction(name.getMethodName(), 15, CompactionState.MINOR, true, StateSource.ADMIN);
095  }
096
097  @Test
098  public void testMajorCompactionStateFromMaster() throws IOException, InterruptedException {
099    compaction(name.getMethodName(), 8, CompactionState.MAJOR, false, StateSource.MASTER);
100  }
101
102  @Test
103  public void testMinorCompactionStateFromMaster() throws IOException, InterruptedException {
104    compaction(name.getMethodName(), 15, CompactionState.MINOR, false, StateSource.MASTER);
105  }
106
107  @Test
108  public void testMajorCompactionOnFamilyStateFromMaster()
109      throws IOException, InterruptedException {
110    compaction(name.getMethodName(), 8, CompactionState.MAJOR, true, StateSource.MASTER);
111  }
112
113  @Test
114  public void testMinorCompactionOnFamilyStateFromMaster()
115      throws IOException, InterruptedException {
116    compaction(name.getMethodName(), 15, CompactionState.MINOR, true, StateSource.MASTER);
117  }
118
119  @Test
120  public void testInvalidColumnFamily() throws IOException, InterruptedException {
121    final TableName tableName = TableName.valueOf(name.getMethodName());
122    byte [] family = Bytes.toBytes("family");
123    byte [] fakecf = Bytes.toBytes("fakecf");
124    boolean caughtMinorCompact = false;
125    boolean caughtMajorCompact = false;
126    Table ht = null;
127    try {
128      ht = TEST_UTIL.createTable(tableName, family);
129      Admin admin = TEST_UTIL.getAdmin();
130      try {
131        admin.compact(tableName, fakecf);
132      } catch (IOException ioe) {
133        caughtMinorCompact = true;
134      }
135      try {
136        admin.majorCompact(tableName, fakecf);
137      } catch (IOException ioe) {
138        caughtMajorCompact = true;
139      }
140    } finally {
141      if (ht != null) {
142        TEST_UTIL.deleteTable(tableName);
143      }
144      assertTrue(caughtMinorCompact);
145      assertTrue(caughtMajorCompact);
146    }
147  }
148
149  /**
150   * Load data to a table, flush it to disk, trigger compaction,
151   * confirm the compaction state is right and wait till it is done.
152   *
153   * @param tableName
154   * @param flushes
155   * @param expectedState
156   * @param singleFamily otherwise, run compaction on all cfs
157   * @param stateSource get the state by Admin or Master
158   * @throws IOException
159   * @throws InterruptedException
160   */
161  private void compaction(final String tableName, final int flushes,
162      final CompactionState expectedState, boolean singleFamily, StateSource stateSource)
163      throws IOException, InterruptedException {
164    // Create a table with regions
165    TableName table = TableName.valueOf(tableName);
166    byte [] family = Bytes.toBytes("family");
167    byte [][] families =
168      {family, Bytes.add(family, Bytes.toBytes("2")), Bytes.add(family, Bytes.toBytes("3"))};
169    Table ht = null;
170    try {
171      ht = TEST_UTIL.createTable(table, families);
172      loadData(ht, families, 3000, flushes);
173      HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
174      HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
175      List<HRegion> regions = rs.getRegions(table);
176      int countBefore = countStoreFilesInFamilies(regions, families);
177      int countBeforeSingleFamily = countStoreFilesInFamily(regions, family);
178      assertTrue(countBefore > 0); // there should be some data files
179      Admin admin = TEST_UTIL.getAdmin();
180      if (expectedState == CompactionState.MINOR) {
181        if (singleFamily) {
182          admin.compact(table, family);
183        } else {
184          admin.compact(table);
185        }
186      } else {
187        if (singleFamily) {
188          admin.majorCompact(table, family);
189        } else {
190          admin.majorCompact(table);
191        }
192      }
193      long curt = System.currentTimeMillis();
194      long waitTime = 5000;
195      long endt = curt + waitTime;
196      CompactionState state = getCompactionState(stateSource, master, admin, table);
197      while (state == CompactionState.NONE && curt < endt) {
198        Thread.sleep(10);
199        state = getCompactionState(stateSource, master, admin, table);
200        curt = System.currentTimeMillis();
201      }
202      // Now, should have the right compaction state,
203      // otherwise, the compaction should have already been done
204      if (expectedState != state) {
205        for (Region region: regions) {
206          state = CompactionState.valueOf(region.getCompactionState().toString());
207          assertEquals(CompactionState.NONE, state);
208        }
209      } else {
210        // Wait until the compaction is done
211        state = getCompactionState(stateSource, master, admin, table);
212        while (state != CompactionState.NONE && curt < endt) {
213          Thread.sleep(10);
214          state = getCompactionState(stateSource, master, admin, table);
215        }
216        // Now, compaction should be done.
217        assertEquals(CompactionState.NONE, state);
218      }
219      int countAfter = countStoreFilesInFamilies(regions, families);
220      int countAfterSingleFamily = countStoreFilesInFamily(regions, family);
221      assertTrue(countAfter < countBefore);
222      if (!singleFamily) {
223        if (expectedState == CompactionState.MAJOR) assertTrue(families.length == countAfter);
224        else assertTrue(families.length < countAfter);
225      } else {
226        int singleFamDiff = countBeforeSingleFamily - countAfterSingleFamily;
227        // assert only change was to single column family
228        assertTrue(singleFamDiff == (countBefore - countAfter));
229        if (expectedState == CompactionState.MAJOR) {
230          assertTrue(1 == countAfterSingleFamily);
231        } else {
232          assertTrue(1 < countAfterSingleFamily);
233        }
234      }
235    } finally {
236      if (ht != null) {
237        TEST_UTIL.deleteTable(table);
238      }
239    }
240  }
241
242  private static CompactionState getCompactionState(StateSource stateSource, HMaster master,
243      Admin admin, TableName table) throws IOException {
244    CompactionState state = stateSource == StateSource.ADMIN ?
245      admin.getCompactionState(table) :
246      master.getCompactionState(table);
247    return state;
248  }
249
250  private static int countStoreFilesInFamily(
251      List<HRegion> regions, final byte[] family) {
252    return countStoreFilesInFamilies(regions, new byte[][]{family});
253  }
254
255  private static int countStoreFilesInFamilies(List<HRegion> regions, final byte[][] families) {
256    int count = 0;
257    for (HRegion region: regions) {
258      count += region.getStoreFileList(families).size();
259    }
260    return count;
261  }
262
263  private static void loadData(final Table ht, final byte[][] families,
264      final int rows, final int flushes) throws IOException {
265    List<Put> puts = new ArrayList<>(rows);
266    byte[] qualifier = Bytes.toBytes("val");
267    for (int i = 0; i < flushes; i++) {
268      for (int k = 0; k < rows; k++) {
269        byte[] row = Bytes.toBytes(random.nextLong());
270        Put p = new Put(row);
271        for (int j = 0; j < families.length; ++j) {
272          p.addColumn(families[j], qualifier, row);
273        }
274        puts.add(p);
275      }
276      ht.put(puts);
277      TEST_UTIL.flush();
278      puts.clear();
279    }
280  }
281}