001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023
024import java.io.IOException;
025import org.apache.hadoop.fs.FSDataOutputStream;
026import org.apache.hadoop.fs.Path;
027import org.apache.hadoop.hbase.HBaseClassTestRule;
028import org.apache.hadoop.hbase.HBaseTestingUtility;
029import org.apache.hadoop.hbase.ServerName;
030import org.apache.hadoop.hbase.TableName;
031import org.apache.hadoop.hbase.client.Put;
032import org.apache.hadoop.hbase.client.Table;
033import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
034import org.apache.hadoop.hbase.testclassification.MediumTests;
035import org.apache.hadoop.hbase.testclassification.RegionServerTests;
036import org.apache.hadoop.hbase.util.Bytes;
037import org.junit.After;
038import org.junit.Before;
039import org.junit.ClassRule;
040import org.junit.Test;
041import org.junit.experimental.categories.Category;
042
043@Category({ MediumTests.class, RegionServerTests.class })
044public class TestBrokenStoreFileCleaner {
045
046  @ClassRule
047  public static final HBaseClassTestRule CLASS_RULE =
048    HBaseClassTestRule.forClass(TestBrokenStoreFileCleaner.class);
049
050  private final HBaseTestingUtility testUtil = new HBaseTestingUtility();
051  private final static byte[] fam = Bytes.toBytes("cf_1");
052  private final static byte[] qual1 = Bytes.toBytes("qf_1");
053  private final static byte[] val = Bytes.toBytes("val");
054  private final static String junkFileName = "409fad9a751c4e8c86d7f32581bdc156";
055  TableName tableName;
056
057  @Before
058  public void setUp() throws Exception {
059    testUtil.getConfiguration().set(StoreFileTrackerFactory.TRACKER_IMPL,
060      "org.apache.hadoop.hbase.regionserver.storefiletracker.FileBasedStoreFileTracker");
061    testUtil.getConfiguration().set(BrokenStoreFileCleaner.BROKEN_STOREFILE_CLEANER_ENABLED,
062      "true");
063    testUtil.getConfiguration().set(BrokenStoreFileCleaner.BROKEN_STOREFILE_CLEANER_TTL, "0");
064    testUtil.getConfiguration().set(BrokenStoreFileCleaner.BROKEN_STOREFILE_CLEANER_PERIOD,
065      "15000000");
066    testUtil.getConfiguration().set(BrokenStoreFileCleaner.BROKEN_STOREFILE_CLEANER_DELAY, "0");
067    testUtil.startMiniCluster(1);
068  }
069
070  @After
071  public void tearDown() throws Exception {
072    testUtil.deleteTable(tableName);
073    testUtil.shutdownMiniCluster();
074  }
075
076  @Test
077  public void testDeletingJunkFile() throws Exception {
078    tableName = TableName.valueOf(getClass().getSimpleName() + "testDeletingJunkFile");
079    createTableWithData(tableName);
080
081    HRegion region = testUtil.getMiniHBaseCluster().getRegions(tableName).get(0);
082    ServerName sn = testUtil.getMiniHBaseCluster().getServerHoldingRegion(tableName,
083      region.getRegionInfo().getRegionName());
084    HRegionServer rs = testUtil.getMiniHBaseCluster().getRegionServer(sn);
085    BrokenStoreFileCleaner cleaner = rs.getBrokenStoreFileCleaner();
086
087    // create junk file
088    HStore store = region.getStore(fam);
089    Path cfPath = store.getRegionFileSystem().getStoreDir(store.getColumnFamilyName());
090    Path junkFilePath = new Path(cfPath, junkFileName);
091
092    FSDataOutputStream junkFileOS = store.getFileSystem().create(junkFilePath);
093    junkFileOS.writeUTF("hello");
094    junkFileOS.close();
095
096    int storeFiles = store.getStorefilesCount();
097    assertTrue(storeFiles > 0);
098
099    // verify the file exist before the chore and missing afterwards
100    assertTrue(store.getFileSystem().exists(junkFilePath));
101    cleaner.chore();
102    assertFalse(store.getFileSystem().exists(junkFilePath));
103
104    // verify no storefile got deleted
105    int currentStoreFiles = store.getStorefilesCount();
106    assertEquals(currentStoreFiles, storeFiles);
107
108  }
109
110  @Test
111  public void testSkippingCompactedFiles() throws Exception {
112    tableName = TableName.valueOf(getClass().getSimpleName() + "testSkippningCompactedFiles");
113    createTableWithData(tableName);
114
115    HRegion region = testUtil.getMiniHBaseCluster().getRegions(tableName).get(0);
116
117    ServerName sn = testUtil.getMiniHBaseCluster().getServerHoldingRegion(tableName,
118      region.getRegionInfo().getRegionName());
119    HRegionServer rs = testUtil.getMiniHBaseCluster().getRegionServer(sn);
120    BrokenStoreFileCleaner cleaner = rs.getBrokenStoreFileCleaner();
121
122    // run major compaction to generate compaced files
123    region.compact(true);
124
125    // make sure there are compacted files
126    HStore store = region.getStore(fam);
127    int compactedFiles = store.getCompactedFilesCount();
128    assertTrue(compactedFiles > 0);
129
130    cleaner.chore();
131
132    // verify none of the compacted files were deleted
133    int existingCompactedFiles = store.getCompactedFilesCount();
134    assertEquals(compactedFiles, existingCompactedFiles);
135
136    // verify adding a junk file does not break anything
137    Path cfPath = store.getRegionFileSystem().getStoreDir(store.getColumnFamilyName());
138    Path junkFilePath = new Path(cfPath, junkFileName);
139
140    FSDataOutputStream junkFileOS = store.getFileSystem().create(junkFilePath);
141    junkFileOS.writeUTF("hello");
142    junkFileOS.close();
143
144    assertTrue(store.getFileSystem().exists(junkFilePath));
145    cleaner.setEnabled(true);
146    cleaner.chore();
147    assertFalse(store.getFileSystem().exists(junkFilePath));
148
149    // verify compacted files are still intact
150    existingCompactedFiles = store.getCompactedFilesCount();
151    assertEquals(compactedFiles, existingCompactedFiles);
152  }
153
154  @Test
155  public void testJunkFileTTL() throws Exception {
156    tableName = TableName.valueOf(getClass().getSimpleName() + "testDeletingJunkFile");
157    createTableWithData(tableName);
158
159    HRegion region = testUtil.getMiniHBaseCluster().getRegions(tableName).get(0);
160    ServerName sn = testUtil.getMiniHBaseCluster().getServerHoldingRegion(tableName,
161      region.getRegionInfo().getRegionName());
162    HRegionServer rs = testUtil.getMiniHBaseCluster().getRegionServer(sn);
163
164    // create junk file
165    HStore store = region.getStore(fam);
166    Path cfPath = store.getRegionFileSystem().getStoreDir(store.getColumnFamilyName());
167    Path junkFilePath = new Path(cfPath, junkFileName);
168
169    FSDataOutputStream junkFileOS = store.getFileSystem().create(junkFilePath);
170    junkFileOS.writeUTF("hello");
171    junkFileOS.close();
172
173    int storeFiles = store.getStorefilesCount();
174    assertTrue(storeFiles > 0);
175
176    // verify the file exist before the chore
177    assertTrue(store.getFileSystem().exists(junkFilePath));
178
179    // set a 5 sec ttl
180    rs.getConfiguration().set(BrokenStoreFileCleaner.BROKEN_STOREFILE_CLEANER_TTL, "5000");
181    BrokenStoreFileCleaner cleaner =
182      new BrokenStoreFileCleaner(15000000, 0, rs, rs.getConfiguration(), rs);
183    cleaner.chore();
184    // file is still present after chore run
185    assertTrue(store.getFileSystem().exists(junkFilePath));
186    Thread.sleep(5000);
187    cleaner.chore();
188    assertFalse(store.getFileSystem().exists(junkFilePath));
189
190    // verify no storefile got deleted
191    int currentStoreFiles = store.getStorefilesCount();
192    assertEquals(currentStoreFiles, storeFiles);
193  }
194
195  private Table createTableWithData(TableName tableName) throws IOException {
196    Table table = testUtil.createTable(tableName, fam);
197    try {
198      for (int i = 1; i < 10; i++) {
199        Put p = new Put(Bytes.toBytes("row" + i));
200        p.addColumn(fam, qual1, val);
201        table.put(p);
202      }
203      // flush them
204      testUtil.getAdmin().flush(tableName);
205      for (int i = 11; i < 20; i++) {
206        Put p = new Put(Bytes.toBytes("row" + i));
207        p.addColumn(fam, qual1, val);
208        table.put(p);
209      }
210      // flush them
211      testUtil.getAdmin().flush(tableName);
212      for (int i = 21; i < 30; i++) {
213        Put p = new Put(Bytes.toBytes("row" + i));
214        p.addColumn(fam, qual1, val);
215        table.put(p);
216      }
217      // flush them
218      testUtil.getAdmin().flush(tableName);
219    } catch (IOException e) {
220      table.close();
221      throw e;
222    }
223    return table;
224  }
225}