001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.mob; 019 020import static org.apache.hadoop.hbase.mob.MobConstants.MOB_CLEANER_BATCH_SIZE_UPPER_BOUND; 021import static org.junit.Assert.assertEquals; 022 023import org.apache.hadoop.fs.FileStatus; 024import org.apache.hadoop.fs.Path; 025import org.apache.hadoop.hbase.HBaseClassTestRule; 026import org.apache.hadoop.hbase.HBaseTestingUtil; 027import org.apache.hadoop.hbase.TableName; 028import org.apache.hadoop.hbase.client.Admin; 029import org.apache.hadoop.hbase.client.BufferedMutator; 030import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 031import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 032import org.apache.hadoop.hbase.client.ConnectionFactory; 033import org.apache.hadoop.hbase.client.Put; 034import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 035import org.apache.hadoop.hbase.testclassification.MediumTests; 036import org.apache.hadoop.hbase.util.Bytes; 037import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 038import org.apache.hadoop.util.ToolRunner; 039import org.junit.After; 040import org.junit.AfterClass; 041import org.junit.Before; 042import org.junit.BeforeClass; 043import org.junit.ClassRule; 044import org.junit.Test; 045import org.junit.experimental.categories.Category; 046 047@Category(MediumTests.class) 048public class TestExpiredMobFileCleaner { 049 050 @ClassRule 051 public static final HBaseClassTestRule CLASS_RULE = 052 HBaseClassTestRule.forClass(TestExpiredMobFileCleaner.class); 053 054 private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); 055 private final static TableName tableName = TableName.valueOf("TestExpiredMobFileCleaner"); 056 private final static String family = "family"; 057 private final static byte[] row1 = Bytes.toBytes("row1"); 058 private final static byte[] row2 = Bytes.toBytes("row2"); 059 private final static byte[] row3 = Bytes.toBytes("row3"); 060 private final static byte[] qf = Bytes.toBytes("qf"); 061 062 private static BufferedMutator table; 063 private static Admin admin; 064 065 @BeforeClass 066 public static void setUpBeforeClass() throws Exception { 067 TEST_UTIL.getConfiguration().setInt("hfile.format.version", 3); 068 TEST_UTIL.getConfiguration().setInt(MOB_CLEANER_BATCH_SIZE_UPPER_BOUND, 2); 069 } 070 071 @AfterClass 072 public static void tearDownAfterClass() throws Exception { 073 074 } 075 076 @Before 077 public void setUp() throws Exception { 078 TEST_UTIL.startMiniCluster(1); 079 } 080 081 @After 082 public void tearDown() throws Exception { 083 admin.disableTable(tableName); 084 admin.deleteTable(tableName); 085 admin.close(); 086 TEST_UTIL.shutdownMiniCluster(); 087 TEST_UTIL.getTestFileSystem().delete(TEST_UTIL.getDataTestDir(), true); 088 } 089 090 private void init() throws Exception { 091 TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(tableName); 092 ColumnFamilyDescriptor columnFamilyDescriptor = 093 ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)).setMobEnabled(true) 094 .setMobThreshold(3L).setMaxVersions(4).build(); 095 tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor); 096 097 admin = TEST_UTIL.getAdmin(); 098 admin.createTable(tableDescriptorBuilder.build()); 099 table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()) 100 .getBufferedMutator(tableName); 101 } 102 103 private void modifyColumnExpiryDays(int expireDays) throws Exception { 104 ColumnFamilyDescriptorBuilder columnFamilyDescriptorBuilder = ColumnFamilyDescriptorBuilder 105 .newBuilder(Bytes.toBytes(family)).setMobEnabled(true).setMobThreshold(3L); 106 // change ttl as expire days to make some row expired 107 int timeToLive = expireDays * secondsOfDay(); 108 columnFamilyDescriptorBuilder.setTimeToLive(timeToLive); 109 110 admin.modifyColumnFamily(tableName, columnFamilyDescriptorBuilder.build()); 111 } 112 113 private void putKVAndFlush(BufferedMutator table, byte[] row, byte[] value, long ts) 114 throws Exception { 115 116 Put put = new Put(row, ts); 117 put.addColumn(Bytes.toBytes(family), qf, value); 118 table.mutate(put); 119 120 table.flush(); 121 admin.flush(tableName); 122 } 123 124 /** 125 * Creates a 3 day old hfile and an 1 day old hfile then sets expiry to 2 days. Verifies that the 126 * 3 day old hfile is removed but the 1 day one is still present after the expiry based cleaner is 127 * run. 128 */ 129 @Test 130 public void testCleaner() throws Exception { 131 init(); 132 133 Path mobDirPath = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, family); 134 135 byte[] dummyData = makeDummyData(600); 136 long ts = EnvironmentEdgeManager.currentTime() - 3 * secondsOfDay() * 1000; // 3 days before 137 putKVAndFlush(table, row1, dummyData, ts); 138 FileStatus[] firstFiles = TEST_UTIL.getTestFileSystem().listStatus(mobDirPath); 139 // the first mob file 140 assertEquals("Before cleanup without delay 1", 1, firstFiles.length); 141 String firstFile = firstFiles[0].getPath().getName(); 142 143 ts = EnvironmentEdgeManager.currentTime() - 1 * secondsOfDay() * 1000; // 1 day before 144 putKVAndFlush(table, row2, dummyData, ts); 145 FileStatus[] secondFiles = TEST_UTIL.getTestFileSystem().listStatus(mobDirPath); 146 // now there are 2 mob files 147 assertEquals("Before cleanup without delay 2", 2, secondFiles.length); 148 String f1 = secondFiles[0].getPath().getName(); 149 String f2 = secondFiles[1].getPath().getName(); 150 String secondFile = f1.equals(firstFile) ? f2 : f1; 151 152 ts = EnvironmentEdgeManager.currentTime() - 4 * secondsOfDay() * 1000; // 4 days before 153 putKVAndFlush(table, row3, dummyData, ts); 154 ts = EnvironmentEdgeManager.currentTime() - 4 * secondsOfDay() * 1000; // 4 days before 155 putKVAndFlush(table, row3, dummyData, ts); 156 FileStatus[] thirdFiles = TEST_UTIL.getTestFileSystem().listStatus(mobDirPath); 157 // now there are 4 mob files 158 assertEquals("Before cleanup without delay 3", 4, thirdFiles.length); 159 160 modifyColumnExpiryDays(2); // ttl = 2, make the first row expired 161 162 // run the cleaner 163 String[] args = new String[2]; 164 args[0] = tableName.getNameAsString(); 165 args[1] = family; 166 ToolRunner.run(TEST_UTIL.getConfiguration(), new ExpiredMobFileCleaner(), args); 167 168 FileStatus[] filesAfterClean = TEST_UTIL.getTestFileSystem().listStatus(mobDirPath); 169 String lastFile = filesAfterClean[0].getPath().getName(); 170 // there are 4 mob files in total, but only 3 need to be cleaned 171 assertEquals("After cleanup without delay 1", 1, filesAfterClean.length); 172 assertEquals("After cleanup without delay 2", secondFile, lastFile); 173 } 174 175 private int secondsOfDay() { 176 return 24 * 3600; 177 } 178 179 private byte[] makeDummyData(int size) { 180 byte[] dummyData = new byte[size]; 181 Bytes.random(dummyData); 182 return dummyData; 183 } 184}