001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022import static org.junit.jupiter.api.Assertions.assertNotNull;
023import static org.junit.jupiter.api.Assertions.assertNull;
024import static org.junit.jupiter.api.Assertions.assertTrue;
025
026import java.io.IOException;
027import org.apache.hadoop.fs.FileSystem;
028import org.apache.hadoop.fs.Path;
029import org.apache.hadoop.hbase.HBaseTestingUtil;
030import org.apache.hadoop.hbase.TableName;
031import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
032import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
033import org.apache.hadoop.hbase.client.Put;
034import org.apache.hadoop.hbase.client.Result;
035import org.apache.hadoop.hbase.client.ResultScanner;
036import org.apache.hadoop.hbase.client.Scan;
037import org.apache.hadoop.hbase.client.Table;
038import org.apache.hadoop.hbase.client.TableDescriptor;
039import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
040import org.apache.hadoop.hbase.mob.MobConstants;
041import org.apache.hadoop.hbase.mob.MobUtils;
042import org.apache.hadoop.hbase.testclassification.MediumTests;
043import org.apache.hadoop.hbase.util.Bytes;
044import org.apache.hadoop.hbase.util.CommonFSUtils;
045import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
046import org.apache.hadoop.hbase.util.HFileArchiveUtil;
047import org.junit.jupiter.api.AfterAll;
048import org.junit.jupiter.api.BeforeAll;
049import org.junit.jupiter.api.BeforeEach;
050import org.junit.jupiter.api.Tag;
051import org.junit.jupiter.api.Test;
052import org.junit.jupiter.api.TestInfo;
053
054@Tag(MediumTests.TAG)
055public class TestDeleteMobTable {
056
057  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
058  private final static byte[] FAMILY = Bytes.toBytes("family");
059  private final static byte[] QF = Bytes.toBytes("qualifier");
060  private String name;
061
062  @BeforeEach
063  public void setTestName(TestInfo testInfo) {
064    this.name = testInfo.getTestMethod().get().getName();
065  }
066
067  @BeforeAll
068  public static void setUpBeforeClass() throws Exception {
069    TEST_UTIL.startMiniCluster(1);
070  }
071
072  @AfterAll
073  public static void tearDownAfterClass() throws Exception {
074    TEST_UTIL.shutdownMiniCluster();
075  }
076
077  /**
078   * Generate the mob value. the size of the value
079   * @return the mob value generated
080   */
081  private static byte[] generateMobValue(int size) {
082    byte[] mobVal = new byte[size];
083    Bytes.random(mobVal);
084    return mobVal;
085  }
086
087  private TableDescriptor createTableDescriptor(TableName tableName, boolean hasMob) {
088    ColumnFamilyDescriptorBuilder builder = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY);
089    if (hasMob) {
090      builder.setMobEnabled(true);
091      builder.setMobThreshold(0);
092    }
093    return TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(builder.build()).build();
094  }
095
096  private Table createTableWithOneFile(TableDescriptor tableDescriptor) throws IOException {
097    Table table = TEST_UTIL.createTable(tableDescriptor, null);
098    try {
099      // insert data
100      byte[] value = generateMobValue(10);
101      byte[] row = Bytes.toBytes("row");
102      Put put = new Put(row);
103      put.addColumn(FAMILY, QF, EnvironmentEdgeManager.currentTime(), value);
104      table.put(put);
105
106      // create an hfile
107      TEST_UTIL.getAdmin().flush(tableDescriptor.getTableName());
108    } catch (IOException e) {
109      table.close();
110      throw e;
111    }
112    return table;
113  }
114
115  @Test
116  public void testDeleteMobTable() throws Exception {
117    final TableName tableName = TableName.valueOf(name);
118    TableDescriptor tableDescriptor = createTableDescriptor(tableName, true);
119    ColumnFamilyDescriptor familyDescriptor = tableDescriptor.getColumnFamily(FAMILY);
120
121    String fileName = null;
122    Table table = createTableWithOneFile(tableDescriptor);
123    try {
124      // the mob file exists
125      assertEquals(1, countMobFiles(tableName, familyDescriptor.getNameAsString()));
126      assertEquals(0, countArchiveMobFiles(tableName, familyDescriptor.getNameAsString()));
127      fileName = assertHasOneMobRow(table, tableName, familyDescriptor.getNameAsString());
128      assertFalse(mobArchiveExist(tableName, familyDescriptor.getNameAsString(), fileName));
129      assertTrue(mobTableDirExist(tableName));
130    } finally {
131      table.close();
132      TEST_UTIL.deleteTable(tableName);
133    }
134
135    assertFalse(TEST_UTIL.getAdmin().tableExists(tableName));
136    assertEquals(0, countMobFiles(tableName, familyDescriptor.getNameAsString()));
137    assertEquals(1, countArchiveMobFiles(tableName, familyDescriptor.getNameAsString()));
138    assertTrue(mobArchiveExist(tableName, familyDescriptor.getNameAsString(), fileName));
139    assertFalse(mobTableDirExist(tableName));
140  }
141
142  @Test
143  public void testDeleteNonMobTable() throws Exception {
144    final TableName tableName = TableName.valueOf(name);
145    TableDescriptor htd = createTableDescriptor(tableName, false);
146    ColumnFamilyDescriptor hcd = htd.getColumnFamily(FAMILY);
147
148    Table table = createTableWithOneFile(htd);
149    try {
150      // the mob file doesn't exist
151      assertEquals(0, countMobFiles(tableName, hcd.getNameAsString()));
152      assertEquals(0, countArchiveMobFiles(tableName, hcd.getNameAsString()));
153      assertFalse(mobTableDirExist(tableName));
154    } finally {
155      table.close();
156      TEST_UTIL.deleteTable(tableName);
157    }
158
159    assertFalse(TEST_UTIL.getAdmin().tableExists(tableName));
160    assertEquals(0, countMobFiles(tableName, hcd.getNameAsString()));
161    assertEquals(0, countArchiveMobFiles(tableName, hcd.getNameAsString()));
162    assertFalse(mobTableDirExist(tableName));
163  }
164
165  @Test
166  public void testMobFamilyDelete() throws Exception {
167    final TableName tableName = TableName.valueOf(name);
168    TableDescriptor tableDescriptor = createTableDescriptor(tableName, true);
169    ColumnFamilyDescriptor familyDescriptor = tableDescriptor.getColumnFamily(FAMILY);
170    tableDescriptor = TableDescriptorBuilder.newBuilder(tableDescriptor)
171      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(Bytes.toBytes("family2"))).build();
172
173    Table table = createTableWithOneFile(tableDescriptor);
174    try {
175      // the mob file exists
176      assertEquals(1, countMobFiles(tableName, familyDescriptor.getNameAsString()));
177      assertEquals(0, countArchiveMobFiles(tableName, familyDescriptor.getNameAsString()));
178      String fileName = assertHasOneMobRow(table, tableName, familyDescriptor.getNameAsString());
179      assertFalse(mobArchiveExist(tableName, familyDescriptor.getNameAsString(), fileName));
180      assertTrue(mobTableDirExist(tableName));
181
182      TEST_UTIL.getAdmin().deleteColumnFamily(tableName, FAMILY);
183
184      assertEquals(0, countMobFiles(tableName, familyDescriptor.getNameAsString()));
185      assertEquals(1, countArchiveMobFiles(tableName, familyDescriptor.getNameAsString()));
186      assertTrue(mobArchiveExist(tableName, familyDescriptor.getNameAsString(), fileName));
187      assertFalse(mobColumnFamilyDirExist(tableName, familyDescriptor.getNameAsString()));
188    } finally {
189      table.close();
190      TEST_UTIL.deleteTable(tableName);
191    }
192  }
193
194  private int countMobFiles(TableName tn, String familyName) throws IOException {
195    FileSystem fs = TEST_UTIL.getTestFileSystem();
196    Path mobFileDir = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tn, familyName);
197    if (fs.exists(mobFileDir)) {
198      return fs.listStatus(mobFileDir).length;
199    }
200    return 0;
201  }
202
203  private int countArchiveMobFiles(TableName tn, String familyName) throws IOException {
204    FileSystem fs = TEST_UTIL.getTestFileSystem();
205    Path storePath = HFileArchiveUtil.getStoreArchivePath(TEST_UTIL.getConfiguration(), tn,
206      MobUtils.getMobRegionInfo(tn).getEncodedName(), familyName);
207    if (fs.exists(storePath)) {
208      return fs.listStatus(storePath).length;
209    }
210    return 0;
211  }
212
213  private boolean mobTableDirExist(TableName tn) throws IOException {
214    FileSystem fs = TEST_UTIL.getTestFileSystem();
215    Path tableDir =
216      CommonFSUtils.getTableDir(MobUtils.getMobHome(TEST_UTIL.getConfiguration()), tn);
217    return fs.exists(tableDir);
218  }
219
220  private boolean mobColumnFamilyDirExist(TableName tn, String familyName) throws IOException {
221    FileSystem fs = TEST_UTIL.getTestFileSystem();
222    Path mobFamilyDir = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tn, familyName);
223    return fs.exists(mobFamilyDir);
224  }
225
226  private boolean mobArchiveExist(TableName tn, String familyName, String fileName)
227    throws IOException {
228    FileSystem fs = TEST_UTIL.getTestFileSystem();
229    Path storePath = HFileArchiveUtil.getStoreArchivePath(TEST_UTIL.getConfiguration(), tn,
230      MobUtils.getMobRegionInfo(tn).getEncodedName(), familyName);
231    return fs.exists(new Path(storePath, fileName));
232  }
233
234  private String assertHasOneMobRow(Table table, TableName tn, String familyName)
235    throws IOException {
236    Scan scan = new Scan();
237    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
238    ResultScanner rs = table.getScanner(scan);
239    Result r = rs.next();
240    assertNotNull(r);
241    String fileName = MobUtils.getMobFileName(r.getColumnLatestCell(FAMILY, QF));
242    Path filePath =
243      new Path(MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tn, familyName), fileName);
244    FileSystem fs = TEST_UTIL.getTestFileSystem();
245    assertTrue(fs.exists(filePath));
246    r = rs.next();
247    assertNull(r);
248    return fileName;
249  }
250}