001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver.storefiletracker;
019
020import static org.junit.Assert.assertNull;
021import static org.junit.Assert.assertThrows;
022import static org.mockito.Mockito.mock;
023import static org.mockito.Mockito.when;
024
025import java.io.IOException;
026import org.apache.hadoop.fs.FSDataInputStream;
027import org.apache.hadoop.fs.FSDataOutputStream;
028import org.apache.hadoop.fs.FileStatus;
029import org.apache.hadoop.fs.FileSystem;
030import org.apache.hadoop.fs.Path;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
033import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
034import org.apache.hadoop.hbase.regionserver.StoreContext;
035import org.apache.hadoop.hbase.testclassification.RegionServerTests;
036import org.apache.hadoop.hbase.testclassification.SmallTests;
037import org.apache.hadoop.hbase.util.Bytes;
038import org.junit.AfterClass;
039import org.junit.Before;
040import org.junit.ClassRule;
041import org.junit.Rule;
042import org.junit.Test;
043import org.junit.experimental.categories.Category;
044import org.junit.rules.TestName;
045import org.slf4j.Logger;
046import org.slf4j.LoggerFactory;
047
048import org.apache.hbase.thirdparty.com.google.common.io.ByteStreams;
049
050import org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.StoreFileList;
051
052@Category({ RegionServerTests.class, SmallTests.class })
053public class TestStoreFileListFile {
054
055  @ClassRule
056  public static final HBaseClassTestRule CLASS_RULE =
057    HBaseClassTestRule.forClass(TestStoreFileListFile.class);
058
059  private static final Logger LOG = LoggerFactory.getLogger(TestStoreFileListFile.class);
060
061  private static final HBaseCommonTestingUtility UTIL = new HBaseCommonTestingUtility();
062
063  private Path testDir;
064
065  private StoreFileListFile storeFileListFile;
066
067  @Rule
068  public TestName name = new TestName();
069
070  @Before
071  public void setUp() throws IOException {
072    testDir = UTIL.getDataTestDir(name.getMethodName());
073    HRegionFileSystem hfs = mock(HRegionFileSystem.class);
074    when(hfs.getFileSystem()).thenReturn(FileSystem.get(UTIL.getConfiguration()));
075    StoreContext ctx = StoreContext.getBuilder().withFamilyStoreDirectoryPath(testDir)
076      .withRegionFileSystem(hfs).build();
077    storeFileListFile = new StoreFileListFile(ctx);
078  }
079
080  @AfterClass
081  public static void tearDown() {
082    UTIL.cleanupTestDir();
083  }
084
085  @Test
086  public void testEmptyLoad() throws IOException {
087    assertNull(storeFileListFile.load());
088  }
089
090  private FileStatus getOnlyTrackerFile(FileSystem fs) throws IOException {
091    return fs.listStatus(new Path(testDir, StoreFileListFile.TRACK_FILE_DIR))[0];
092  }
093
094  private byte[] readAll(FileSystem fs, Path file) throws IOException {
095    try (FSDataInputStream in = fs.open(file)) {
096      return ByteStreams.toByteArray(in);
097    }
098  }
099
100  private void write(FileSystem fs, Path file, byte[] buf, int off, int len) throws IOException {
101    try (FSDataOutputStream out = fs.create(file, true)) {
102      out.write(buf, off, len);
103    }
104  }
105
106  @Test
107  public void testLoadPartial() throws IOException {
108    StoreFileList.Builder builder = StoreFileList.newBuilder();
109    storeFileListFile.update(builder);
110    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
111    FileStatus trackerFileStatus = getOnlyTrackerFile(fs);
112    // truncate it so we do not have enough data
113    LOG.info("Truncate file {} with size {} to {}", trackerFileStatus.getPath(),
114      trackerFileStatus.getLen(), trackerFileStatus.getLen() / 2);
115    byte[] content = readAll(fs, trackerFileStatus.getPath());
116    write(fs, trackerFileStatus.getPath(), content, 0, content.length / 2);
117    assertNull(storeFileListFile.load());
118  }
119
120  private void writeInt(byte[] buf, int off, int value) {
121    byte[] b = Bytes.toBytes(value);
122    for (int i = 0; i < 4; i++) {
123      buf[off + i] = b[i];
124    }
125  }
126
127  @Test
128  public void testZeroFileLength() throws IOException {
129    StoreFileList.Builder builder = StoreFileList.newBuilder();
130    storeFileListFile.update(builder);
131    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
132    FileStatus trackerFileStatus = getOnlyTrackerFile(fs);
133    // write a zero length
134    byte[] content = readAll(fs, trackerFileStatus.getPath());
135    writeInt(content, 0, 0);
136    write(fs, trackerFileStatus.getPath(), content, 0, content.length);
137    assertThrows(IOException.class, () -> storeFileListFile.load());
138  }
139
140  @Test
141  public void testBigFileLength() throws IOException {
142    StoreFileList.Builder builder = StoreFileList.newBuilder();
143    storeFileListFile.update(builder);
144    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
145    FileStatus trackerFileStatus = getOnlyTrackerFile(fs);
146    // write a large length
147    byte[] content = readAll(fs, trackerFileStatus.getPath());
148    writeInt(content, 0, 128 * 1024 * 1024);
149    write(fs, trackerFileStatus.getPath(), content, 0, content.length);
150    assertThrows(IOException.class, () -> storeFileListFile.load());
151  }
152
153  @Test
154  public void testChecksumMismatch() throws IOException {
155    StoreFileList.Builder builder = StoreFileList.newBuilder();
156    storeFileListFile.update(builder);
157    FileSystem fs = FileSystem.get(UTIL.getConfiguration());
158    FileStatus trackerFileStatus = getOnlyTrackerFile(fs);
159    // flip one byte
160    byte[] content = readAll(fs, trackerFileStatus.getPath());
161    content[5] = (byte) ~content[5];
162    write(fs, trackerFileStatus.getPath(), content, 0, content.length);
163    assertThrows(IOException.class, () -> storeFileListFile.load());
164  }
165}