001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver.storefiletracker; 019 020import static org.junit.Assert.assertNull; 021import static org.junit.Assert.assertThrows; 022import static org.mockito.Mockito.mock; 023import static org.mockito.Mockito.when; 024 025import java.io.IOException; 026import org.apache.hadoop.fs.FSDataInputStream; 027import org.apache.hadoop.fs.FSDataOutputStream; 028import org.apache.hadoop.fs.FileStatus; 029import org.apache.hadoop.fs.FileSystem; 030import org.apache.hadoop.fs.Path; 031import org.apache.hadoop.hbase.HBaseClassTestRule; 032import org.apache.hadoop.hbase.HBaseCommonTestingUtility; 033import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; 034import org.apache.hadoop.hbase.regionserver.StoreContext; 035import org.apache.hadoop.hbase.testclassification.RegionServerTests; 036import org.apache.hadoop.hbase.testclassification.SmallTests; 037import org.apache.hadoop.hbase.util.Bytes; 038import org.junit.AfterClass; 039import org.junit.Before; 040import org.junit.ClassRule; 041import org.junit.Rule; 042import org.junit.Test; 043import org.junit.experimental.categories.Category; 044import org.junit.rules.TestName; 045import org.slf4j.Logger; 046import org.slf4j.LoggerFactory; 047 048import org.apache.hbase.thirdparty.com.google.common.io.ByteStreams; 049 050import org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.StoreFileList; 051 052@Category({ RegionServerTests.class, SmallTests.class }) 053public class TestStoreFileListFile { 054 055 @ClassRule 056 public static final HBaseClassTestRule CLASS_RULE = 057 HBaseClassTestRule.forClass(TestStoreFileListFile.class); 058 059 private static final Logger LOG = LoggerFactory.getLogger(TestStoreFileListFile.class); 060 061 private static final HBaseCommonTestingUtility UTIL = new HBaseCommonTestingUtility(); 062 063 private Path testDir; 064 065 private StoreFileListFile storeFileListFile; 066 067 @Rule 068 public TestName name = new TestName(); 069 070 @Before 071 public void setUp() throws IOException { 072 testDir = UTIL.getDataTestDir(name.getMethodName()); 073 HRegionFileSystem hfs = mock(HRegionFileSystem.class); 074 when(hfs.getFileSystem()).thenReturn(FileSystem.get(UTIL.getConfiguration())); 075 StoreContext ctx = StoreContext.getBuilder().withFamilyStoreDirectoryPath(testDir) 076 .withRegionFileSystem(hfs).build(); 077 storeFileListFile = new StoreFileListFile(ctx); 078 } 079 080 @AfterClass 081 public static void tearDown() { 082 UTIL.cleanupTestDir(); 083 } 084 085 @Test 086 public void testEmptyLoad() throws IOException { 087 assertNull(storeFileListFile.load()); 088 } 089 090 private FileStatus getOnlyTrackerFile(FileSystem fs) throws IOException { 091 return fs.listStatus(new Path(testDir, StoreFileListFile.TRACK_FILE_DIR))[0]; 092 } 093 094 private byte[] readAll(FileSystem fs, Path file) throws IOException { 095 try (FSDataInputStream in = fs.open(file)) { 096 return ByteStreams.toByteArray(in); 097 } 098 } 099 100 private void write(FileSystem fs, Path file, byte[] buf, int off, int len) throws IOException { 101 try (FSDataOutputStream out = fs.create(file, true)) { 102 out.write(buf, off, len); 103 } 104 } 105 106 @Test 107 public void testLoadPartial() throws IOException { 108 StoreFileList.Builder builder = StoreFileList.newBuilder(); 109 storeFileListFile.update(builder); 110 FileSystem fs = FileSystem.get(UTIL.getConfiguration()); 111 FileStatus trackerFileStatus = getOnlyTrackerFile(fs); 112 // truncate it so we do not have enough data 113 LOG.info("Truncate file {} with size {} to {}", trackerFileStatus.getPath(), 114 trackerFileStatus.getLen(), trackerFileStatus.getLen() / 2); 115 byte[] content = readAll(fs, trackerFileStatus.getPath()); 116 write(fs, trackerFileStatus.getPath(), content, 0, content.length / 2); 117 assertNull(storeFileListFile.load()); 118 } 119 120 private void writeInt(byte[] buf, int off, int value) { 121 byte[] b = Bytes.toBytes(value); 122 for (int i = 0; i < 4; i++) { 123 buf[off + i] = b[i]; 124 } 125 } 126 127 @Test 128 public void testZeroFileLength() throws IOException { 129 StoreFileList.Builder builder = StoreFileList.newBuilder(); 130 storeFileListFile.update(builder); 131 FileSystem fs = FileSystem.get(UTIL.getConfiguration()); 132 FileStatus trackerFileStatus = getOnlyTrackerFile(fs); 133 // write a zero length 134 byte[] content = readAll(fs, trackerFileStatus.getPath()); 135 writeInt(content, 0, 0); 136 write(fs, trackerFileStatus.getPath(), content, 0, content.length); 137 assertThrows(IOException.class, () -> storeFileListFile.load()); 138 } 139 140 @Test 141 public void testBigFileLength() throws IOException { 142 StoreFileList.Builder builder = StoreFileList.newBuilder(); 143 storeFileListFile.update(builder); 144 FileSystem fs = FileSystem.get(UTIL.getConfiguration()); 145 FileStatus trackerFileStatus = getOnlyTrackerFile(fs); 146 // write a large length 147 byte[] content = readAll(fs, trackerFileStatus.getPath()); 148 writeInt(content, 0, 128 * 1024 * 1024); 149 write(fs, trackerFileStatus.getPath(), content, 0, content.length); 150 assertThrows(IOException.class, () -> storeFileListFile.load()); 151 } 152 153 @Test 154 public void testChecksumMismatch() throws IOException { 155 StoreFileList.Builder builder = StoreFileList.newBuilder(); 156 storeFileListFile.update(builder); 157 FileSystem fs = FileSystem.get(UTIL.getConfiguration()); 158 FileStatus trackerFileStatus = getOnlyTrackerFile(fs); 159 // flip one byte 160 byte[] content = readAll(fs, trackerFileStatus.getPath()); 161 content[5] = (byte) ~content[5]; 162 write(fs, trackerFileStatus.getPath(), content, 0, content.length); 163 assertThrows(IOException.class, () -> storeFileListFile.load()); 164 } 165}