001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertThrows;
022import static org.mockito.ArgumentMatchers.any;
023import static org.mockito.ArgumentMatchers.anyLong;
024import static org.mockito.ArgumentMatchers.argThat;
025import static org.mockito.ArgumentMatchers.eq;
026import static org.mockito.Mockito.atLeast;
027import static org.mockito.Mockito.doThrow;
028import static org.mockito.Mockito.mock;
029import static org.mockito.Mockito.never;
030import static org.mockito.Mockito.times;
031import static org.mockito.Mockito.verify;
032import static org.mockito.Mockito.verifyNoInteractions;
033import static org.mockito.Mockito.when;
034
035import java.io.FileNotFoundException;
036import java.io.IOException;
037import java.util.Arrays;
038import java.util.Collection;
039import java.util.Collections;
040import org.apache.hadoop.conf.Configuration;
041import org.apache.hadoop.fs.FileStatus;
042import org.apache.hadoop.fs.FileSystem;
043import org.apache.hadoop.fs.Path;
044import org.apache.hadoop.hbase.HBaseConfiguration;
045import org.apache.hadoop.hbase.HConstants;
046import org.apache.hadoop.hbase.TableName;
047import org.apache.hadoop.hbase.client.RegionInfo;
048import org.apache.hadoop.hbase.mob.MobUtils;
049import org.apache.hadoop.hbase.testclassification.MiscTests;
050import org.apache.hadoop.hbase.testclassification.SmallTests;
051import org.apache.hadoop.hbase.util.Bytes;
052import org.apache.hadoop.hbase.util.CommonFSUtils;
053import org.apache.hadoop.hbase.util.HFileArchiveUtil;
054import org.junit.jupiter.api.BeforeEach;
055import org.junit.jupiter.api.Tag;
056import org.junit.jupiter.api.Test;
057
058@Tag(SmallTests.TAG)
059@Tag(MiscTests.TAG)
060public class TestHFileArchiverFileStatuses {
061  private static final TableName TABLE_NAME = TableName.valueOf("testArchiveFileStatuses");
062  private static final RegionInfo REGION_INFO = MobUtils.getMobRegionInfo(TABLE_NAME);
063  private static final byte[] FAMILY = Bytes.toBytes("family");
064
065  private Configuration conf;
066  private FileSystem fs;
067  private Path tableDir;
068  private Path archiveDir;
069  private FileStatus source;
070  private Path archivedFile;
071
072  @BeforeEach
073  public void setUp() throws IOException {
074    conf = HBaseConfiguration.create();
075    conf.set(HConstants.HBASE_DIR, "file:///hbase");
076    tableDir = CommonFSUtils.getTableDir(MobUtils.getMobHome(conf), TABLE_NAME);
077    archiveDir = HFileArchiveUtil.getStoreArchivePath(conf, REGION_INFO, tableDir, FAMILY);
078    source = new FileStatus(10, false, 1, 1024, 1,
079      new Path(MobUtils.getMobFamilyPath(conf, TABLE_NAME, "family"), "file"));
080    archivedFile = new Path(archiveDir, source.getPath().getName());
081    fs = mock(FileSystem.class);
082    when(fs.mkdirs(any())).thenReturn(true);
083    when(fs.rename(any(), any())).thenReturn(true);
084  }
085
086  private void archive(Collection<FileStatus> statuses) throws IOException {
087    HFileArchiver.archiveStoreFileStatuses(conf, fs, REGION_INFO, FAMILY, statuses);
088  }
089
090  @Test
091  public void testArchiveWithoutFetchingSourceMetadata() throws IOException {
092    archive(Collections.singletonList(source));
093
094    verify(fs).rename(source.getPath(), archivedFile);
095    verify(fs).setTimes(eq(source.getPath()), anyLong(), eq(-1L));
096    // With no archive conflict, the existing status is enough to identify the source file.
097    verify(fs, never()).getFileStatus(source.getPath());
098    verify(fs, never()).isFile(source.getPath());
099  }
100
101  @Test
102  public void testEmptyInput() throws IOException {
103    archive(Collections.emptyList());
104    verifyNoInteractions(fs);
105  }
106
107  @Test
108  public void testRejectDirectoryBeforeArchivingAnyFiles() {
109    FileStatus directory = new FileStatus(0, true, 1, 0, 0, new Path(tableDir, "directory"));
110    assertThrows(IOException.class, () -> archive(Arrays.asList(source, directory)));
111    verifyNoInteractions(fs);
112  }
113
114  @Test
115  public void testAlreadyArchived() throws IOException {
116    when(fs.exists(archivedFile)).thenReturn(true);
117    archive(Collections.singletonList(source));
118
119    verify(fs, never()).rename(any(), any());
120    verify(fs, never()).getFileStatus(any());
121  }
122
123  @Test
124  public void testArchiveConflictWithSameLength() throws IOException {
125    when(fs.exists(archivedFile)).thenReturn(true);
126    when(fs.exists(source.getPath())).thenReturn(true);
127    when(fs.getFileStatus(source.getPath())).thenReturn(source);
128    when(fs.getFileStatus(archivedFile))
129      .thenReturn(new FileStatus(source.getLen(), false, 1, 1024, 1, archivedFile));
130
131    archive(Collections.singletonList(source));
132
133    verify(fs).rename(eq(archivedFile), argThat(path -> path.getParent().equals(archiveDir)
134      && path.getName().startsWith(source.getPath().getName() + ".")));
135    verify(fs).rename(source.getPath(), archivedFile);
136    verify(fs, never()).delete(any(), eq(false));
137  }
138
139  @Test
140  public void testArchiveConflictWithDifferentLength() throws IOException {
141    when(fs.exists(archivedFile)).thenReturn(true);
142    when(fs.exists(source.getPath())).thenReturn(true);
143    when(fs.getFileStatus(source.getPath())).thenReturn(source);
144    when(fs.getFileStatus(archivedFile))
145      .thenReturn(new FileStatus(source.getLen() + 1, false, 1, 1024, 1, archivedFile));
146
147    FailedArchiveException error =
148      assertThrows(FailedArchiveException.class, () -> archive(Collections.singletonList(source)));
149
150    assertEquals(Collections.singletonList(source.getPath()), error.getFailedFiles());
151    verify(fs, never()).rename(any(), any());
152    verify(fs, never()).delete(any(), eq(false));
153  }
154
155  @Test
156  public void testSourceDisappearsBeforeRename() throws IOException {
157    doThrow(new FileNotFoundException()).when(fs).setTimes(eq(source.getPath()), anyLong(),
158      eq(-1L));
159
160    archive(Collections.singletonList(source));
161
162    verify(fs, never()).rename(any(), any());
163  }
164
165  @Test
166  public void testRetryRename() throws IOException {
167    when(fs.rename(source.getPath(), archivedFile)).thenReturn(false, true);
168
169    archive(Collections.singletonList(source));
170
171    verify(fs, times(2)).rename(source.getPath(), archivedFile);
172  }
173
174  @Test
175  public void testFailureDoesNotPreventOtherFilesFromBeingArchived() throws IOException {
176    FileStatus other =
177      new FileStatus(10, false, 1, 1024, 1, new Path(source.getPath().getParent(), "other"));
178    when(fs.rename(source.getPath(), archivedFile)).thenReturn(false);
179
180    FailedArchiveException error =
181      assertThrows(FailedArchiveException.class, () -> archive(Arrays.asList(source, other)));
182
183    assertEquals(Collections.singletonList(source.getPath()), error.getFailedFiles());
184    verify(fs, atLeast(2)).rename(source.getPath(), archivedFile);
185    verify(fs).rename(other.getPath(), new Path(archiveDir, "other"));
186  }
187
188  @Test
189  public void testCannotCreateArchiveDirectory() throws IOException {
190    when(fs.mkdirs(archiveDir)).thenReturn(false);
191    assertThrows(IOException.class, () -> archive(Collections.singletonList(source)));
192    verify(fs, never()).rename(any(), any());
193  }
194}