001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.mob;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertTrue;
022import static org.mockito.ArgumentMatchers.any;
023import static org.mockito.Mockito.mock;
024import static org.mockito.Mockito.never;
025import static org.mockito.Mockito.times;
026import static org.mockito.Mockito.verify;
027import static org.mockito.Mockito.when;
028
029import java.io.IOException;
030import java.util.ArrayList;
031import java.util.Arrays;
032import java.util.Date;
033import java.util.List;
034import java.util.UUID;
035import org.apache.hadoop.conf.Configuration;
036import org.apache.hadoop.fs.FSDataOutputStream;
037import org.apache.hadoop.fs.FileStatus;
038import org.apache.hadoop.fs.FileSystem;
039import org.apache.hadoop.fs.Path;
040import org.apache.hadoop.hbase.HBaseConfiguration;
041import org.apache.hadoop.hbase.HBaseTestingUtil;
042import org.apache.hadoop.hbase.HConstants;
043import org.apache.hadoop.hbase.TableName;
044import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
045import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
046import org.apache.hadoop.hbase.client.TableDescriptor;
047import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
048import org.apache.hadoop.hbase.io.HFileLink;
049import org.apache.hadoop.hbase.testclassification.SmallTests;
050import org.apache.hadoop.hbase.util.Bytes;
051import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
052import org.apache.hadoop.hbase.util.HFileArchiveUtil;
053import org.junit.jupiter.api.Tag;
054import org.junit.jupiter.api.Test;
055
056import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet;
057import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSetMultimap;
058
059@Tag(SmallTests.TAG)
060public class TestMobUtils {
061  public static final TableName TEST_TABLE_1 = TableName.valueOf("testTable1");
062  public static final TableName TEST_TABLE_2 = TableName.valueOf("testTable2");
063  public static final TableName TEST_TABLE_3 = TableName.valueOf("testTable3");
064
065  @Test
066  public void testCleanExpiredMobFilesUsesListedStatuses() throws IOException {
067    Configuration conf = HBaseConfiguration.create();
068    conf.set(HConstants.HBASE_DIR, "file:///hbase");
069    conf.setInt(MobConstants.MOB_CLEANER_BATCH_SIZE_UPPER_BOUND, 2);
070    ColumnFamilyDescriptor family = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("f"))
071      .setMobEnabled(true).setTimeToLive(24 * 60 * 60).build();
072    TableDescriptor table =
073      TableDescriptorBuilder.newBuilder(TEST_TABLE_1).setColumnFamily(family).build();
074    Path familyPath = MobUtils.getMobFamilyPath(conf, TEST_TABLE_1, "f");
075    FileStatus expired = mobFileStatus(familyPath, "20000101", false);
076    String legacyName = expired.getPath().getName().split(MobFileName.REGION_SEP)[0];
077    FileStatus legacy = new FileStatus(10, false, 1, 1024, 1, new Path(familyPath, legacyName));
078    FileStatus directory = mobFileStatus(familyPath, "20000101", true);
079    long now = EnvironmentEdgeManager.currentTime();
080    FileStatus current = mobFileStatus(familyPath, MobUtils.formatDate(new Date(now)), false);
081    // HFileLink recognizes the legacy MOB filename without the region suffix.
082    String linkName = HFileLink.createHFileLinkName(TEST_TABLE_2,
083      MobUtils.getMobRegionInfo(TEST_TABLE_2).getEncodedName(), legacyName);
084    FileStatus link = new FileStatus(0, false, 1, 1024, 1, new Path(familyPath, linkName));
085    assertTrue(HFileLink.isHFileLink(link.getPath()));
086    FileSystem fs = mock(FileSystem.class);
087    when(fs.listStatus(familyPath))
088      .thenReturn(new FileStatus[] { expired, directory, legacy, current, link });
089    when(fs.mkdirs(any())).thenReturn(true);
090    when(fs.rename(any(), any())).thenReturn(true);
091
092    MobUtils.cleanExpiredMobFiles(fs, conf, table, family, now);
093
094    Path archiveDir = HFileArchiveUtil.getStoreArchivePath(conf,
095      MobUtils.getMobRegionInfo(TEST_TABLE_1), family.getName());
096    for (FileStatus status : new FileStatus[] { expired, legacy, link }) {
097      // Archive the listed path, including the link itself, rather than its referenced HFile.
098      verify(fs).rename(status.getPath(), new Path(archiveDir, status.getPath().getName()));
099    }
100    verify(fs, times(3)).rename(any(), any());
101    verify(fs, never()).getFileStatus(any());
102    verify(fs, never()).isFile(any());
103  }
104
105  @Test
106  public void testCleanExpiredMobFilesSkipsInvalidNames() throws IOException {
107    HBaseTestingUtil util = new HBaseTestingUtil();
108    Configuration conf = util.getConfiguration();
109    Path rootDir = util.getDataTestDir();
110    conf.set(HConstants.HBASE_DIR, rootDir.toString());
111    ColumnFamilyDescriptor family = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("f"))
112      .setMobEnabled(true).setTimeToLive(24 * 60 * 60).build();
113    TableDescriptor table =
114      TableDescriptorBuilder.newBuilder(TEST_TABLE_1).setColumnFamily(family).build();
115    Path familyPath = MobUtils.getMobFamilyPath(conf, TEST_TABLE_1, "f");
116    Path expired = mobFileStatus(familyPath, "20000101", false).getPath();
117    // Each invalid name contains a parsable, expired date at the expected offset.
118    List<Path> invalidPaths = Arrays.asList(new Path(familyPath, "z".repeat(32) + "20000101junk"),
119      new Path(familyPath, "0".repeat(32) + "20000101junk"),
120      new Path(familyPath, "z".repeat(32) + "20000101" + "0".repeat(32) + "_abc"));
121    List<Path> paths = new ArrayList<>(invalidPaths);
122    paths.add(expired);
123    FileSystem fs = FileSystem.getLocal(conf);
124    try {
125      for (Path path : paths) {
126        try (FSDataOutputStream out = fs.create(path)) {
127          out.write(1);
128        }
129      }
130
131      MobUtils.cleanExpiredMobFiles(fs, conf, table, family, EnvironmentEdgeManager.currentTime());
132
133      Path archiveDir = HFileArchiveUtil.getStoreArchivePath(conf,
134        MobUtils.getMobRegionInfo(TEST_TABLE_1), family.getName());
135      assertTrue(fs.exists(new Path(archiveDir, expired.getName())));
136      for (Path path : invalidPaths) {
137        assertTrue(fs.exists(path), "Invalid name must be left in the MOB directory: " + path);
138      }
139      assertEquals(1, fs.listStatus(archiveDir).length);
140    } finally {
141      fs.delete(rootDir, true);
142    }
143  }
144
145  private FileStatus mobFileStatus(Path familyPath, String date, boolean directory) {
146    String name =
147      MobFileName.create(Bytes.toBytes("row"), date, UUID.randomUUID().toString().replace("-", ""),
148        MobUtils.getMobRegionInfo(TEST_TABLE_1).getEncodedName()).getFileName();
149    return new FileStatus(10, directory, 1, 1024, 1, new Path(familyPath, name));
150  }
151
152  @Test
153  public void serializeSingleMobFileRefs() {
154    ImmutableSetMultimap<TableName, String> mobRefSet =
155      ImmutableSetMultimap.<TableName, String> builder().putAll(TEST_TABLE_1, "file1a").build();
156    byte[] result = MobUtils.serializeMobFileRefs(mobRefSet);
157    assertEquals("testTable1/file1a", Bytes.toString(result));
158  }
159
160  @Test
161  public void serializeMultipleMobFileRefs() {
162    ImmutableSetMultimap<TableName, String> mobRefSet =
163      ImmutableSetMultimap.<TableName, String> builder().putAll(TEST_TABLE_1, "file1a", "file1b")
164        .putAll(TEST_TABLE_2, "file2a").putAll(TEST_TABLE_3, "file3a", "file3b").build();
165    byte[] result = MobUtils.serializeMobFileRefs(mobRefSet);
166    assertEquals("testTable1/file1a,file1b//testTable2/file2a//testTable3/file3a,file3b",
167      Bytes.toString(result));
168  }
169
170  @Test
171  public void deserializeSingleMobFileRefs() {
172    ImmutableSetMultimap<TableName, String> mobRefSet =
173      MobUtils.deserializeMobFileRefs(Bytes.toBytes("testTable1/file1a")).build();
174    assertEquals(1, mobRefSet.size());
175    ImmutableSet<String> testTable1Refs = mobRefSet.get(TEST_TABLE_1);
176    assertEquals(1, testTable1Refs.size());
177    assertTrue(testTable1Refs.contains("file1a"));
178  }
179
180  @Test
181  public void deserializeMultipleMobFileRefs() {
182    ImmutableSetMultimap<TableName,
183      String> mobRefSet = MobUtils
184        .deserializeMobFileRefs(
185          Bytes.toBytes("testTable1/file1a,file1b//testTable2/file2a//testTable3/file3a,file3b"))
186        .build();
187    assertEquals(5, mobRefSet.size());
188    ImmutableSet<String> testTable1Refs = mobRefSet.get(TEST_TABLE_1);
189    ImmutableSet<String> testTable2Refs = mobRefSet.get(TEST_TABLE_2);
190    ImmutableSet<String> testTable3Refs = mobRefSet.get(TEST_TABLE_3);
191    assertEquals(2, testTable1Refs.size());
192    assertEquals(1, testTable2Refs.size());
193    assertEquals(2, testTable3Refs.size());
194    assertTrue(testTable1Refs.contains("file1a"));
195    assertTrue(testTable1Refs.contains("file1b"));
196    assertTrue(testTable2Refs.contains("file2a"));
197    assertTrue(testTable3Refs.contains("file3a"));
198    assertTrue(testTable3Refs.contains("file3b"));
199  }
200
201  public static String getTableName(String testMethodName) {
202    return testMethodName.replace("[", "-").replace("]", "");
203  }
204}