001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static java.util.Arrays.asList; 021import static org.mockito.ArgumentMatchers.any; 022import static org.mockito.ArgumentMatchers.anyLong; 023import static org.mockito.Mockito.verify; 024import static org.mockito.Mockito.when; 025import static org.mockito.hamcrest.MockitoHamcrest.argThat; 026 027import java.io.FileNotFoundException; 028import java.io.IOException; 029import java.util.ArrayList; 030import java.util.List; 031 032import org.apache.hadoop.fs.Path; 033import org.apache.hadoop.hbase.DoNotRetryIOException; 034import org.apache.hadoop.hbase.HBaseClassTestRule; 035import org.apache.hadoop.hbase.TableName; 036import org.apache.hadoop.hbase.testclassification.SmallTests; 037import org.apache.hadoop.hbase.util.Pair; 038import org.apache.hadoop.hbase.wal.WALEdit; 039import org.apache.hadoop.hbase.wal.WALKeyImpl; 040import org.junit.ClassRule; 041import org.junit.Test; 042import org.junit.experimental.categories.Category; 043import org.mockito.invocation.InvocationOnMock; 044import org.mockito.stubbing.Answer; 045 046 047/** 048 * This class attempts to unit test bulk HLog loading. 049 */ 050@Category(SmallTests.class) 051public class TestBulkLoad extends TestBulkloadBase { 052 053 @ClassRule 054 public static final HBaseClassTestRule CLASS_RULE = 055 HBaseClassTestRule.forClass(TestBulkLoad.class); 056 057 @Test 058 public void verifyBulkLoadEvent() throws IOException { 059 TableName tableName = TableName.valueOf("test", "test"); 060 List<Pair<byte[], String>> familyPaths = withFamilyPathsFor(family1); 061 byte[] familyName = familyPaths.get(0).getFirst(); 062 String storeFileName = familyPaths.get(0).getSecond(); 063 storeFileName = (new Path(storeFileName)).getName(); 064 List<String> storeFileNames = new ArrayList<>(); 065 storeFileNames.add(storeFileName); 066 when(log.appendMarker(any(), any(), 067 argThat(bulkLogWalEdit(WALEdit.BULK_LOAD, tableName.toBytes(), familyName, storeFileNames)))) 068 .thenAnswer(new Answer() { 069 @Override 070 public Object answer(InvocationOnMock invocation) { 071 WALKeyImpl walKey = invocation.getArgument(1); 072 MultiVersionConcurrencyControl mvcc = walKey.getMvcc(); 073 if (mvcc != null) { 074 MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin(); 075 walKey.setWriteEntry(we); 076 } 077 return 01L; 078 }; 079 }); 080 testRegionWithFamiliesAndSpecifiedTableName(tableName, family1).bulkLoadHFiles(familyPaths, 081 false, null); 082 verify(log).sync(anyLong()); 083 } 084 085 @Test 086 public void bulkHLogShouldThrowNoErrorAndWriteMarkerWithBlankInput() throws IOException { 087 testRegionWithFamilies(family1).bulkLoadHFiles(new ArrayList<>(),false, null); 088 } 089 090 @Test 091 public void shouldBulkLoadSingleFamilyHLog() throws IOException { 092 when(log.appendMarker(any(), any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)))) 093 .thenAnswer(new Answer() { 094 @Override 095 public Object answer(InvocationOnMock invocation) { 096 WALKeyImpl walKey = invocation.getArgument(1); 097 MultiVersionConcurrencyControl mvcc = walKey.getMvcc(); 098 if (mvcc != null) { 099 MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin(); 100 walKey.setWriteEntry(we); 101 } 102 return 01L; 103 }; 104 }); 105 testRegionWithFamilies(family1).bulkLoadHFiles(withFamilyPathsFor(family1), false, null); 106 verify(log).sync(anyLong()); 107 } 108 109 @Test 110 public void shouldBulkLoadManyFamilyHLog() throws IOException { 111 when(log.appendMarker(any(), 112 any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)))).thenAnswer(new Answer() { 113 @Override 114 public Object answer(InvocationOnMock invocation) { 115 WALKeyImpl walKey = invocation.getArgument(1); 116 MultiVersionConcurrencyControl mvcc = walKey.getMvcc(); 117 if (mvcc != null) { 118 MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin(); 119 walKey.setWriteEntry(we); 120 } 121 return 01L; 122 }; 123 }); 124 testRegionWithFamilies(family1, family2).bulkLoadHFiles(withFamilyPathsFor(family1, family2), 125 false, null); 126 verify(log).sync(anyLong()); 127 } 128 129 @Test 130 public void shouldBulkLoadManyFamilyHLogEvenWhenTableNameNamespaceSpecified() throws IOException { 131 when(log.appendMarker(any(), any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)))) 132 .thenAnswer(new Answer() { 133 @Override 134 public Object answer(InvocationOnMock invocation) { 135 WALKeyImpl walKey = invocation.getArgument(1); 136 MultiVersionConcurrencyControl mvcc = walKey.getMvcc(); 137 if (mvcc != null) { 138 MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin(); 139 walKey.setWriteEntry(we); 140 } 141 return 01L; 142 }; 143 }); 144 TableName tableName = TableName.valueOf("test", "test"); 145 testRegionWithFamiliesAndSpecifiedTableName(tableName, family1, family2) 146 .bulkLoadHFiles(withFamilyPathsFor(family1, family2), false, null); 147 verify(log).sync(anyLong()); 148 } 149 150 @Test(expected = DoNotRetryIOException.class) 151 public void shouldCrashIfBulkLoadFamiliesNotInTable() throws IOException { 152 testRegionWithFamilies(family1).bulkLoadHFiles(withFamilyPathsFor(family1, family2), false, 153 null); 154 } 155 156 // after HBASE-24021 will throw DoNotRetryIOException, not MultipleIOException 157 @Test(expected = DoNotRetryIOException.class) 158 public void shouldCrashIfBulkLoadMultiFamiliesNotInTable() throws IOException { 159 testRegionWithFamilies(family1).bulkLoadHFiles(withFamilyPathsFor(family1, family2, family3), 160 false, null); 161 } 162 163 @Test(expected = DoNotRetryIOException.class) 164 public void bulkHLogShouldThrowErrorWhenFamilySpecifiedAndHFileExistsButNotInTableDescriptor() 165 throws IOException { 166 testRegionWithFamilies().bulkLoadHFiles(withFamilyPathsFor(family1), false, null); 167 } 168 169 @Test(expected = DoNotRetryIOException.class) 170 public void shouldThrowErrorIfBadFamilySpecifiedAsFamilyPath() throws IOException { 171 testRegionWithFamilies() 172 .bulkLoadHFiles(asList(withInvalidColumnFamilyButProperHFileLocation(family1)), 173 false, null); 174 } 175 176 @Test(expected = FileNotFoundException.class) 177 public void shouldThrowErrorIfHFileDoesNotExist() throws IOException { 178 List<Pair<byte[], String>> list = asList(withMissingHFileForFamily(family1)); 179 testRegionWithFamilies(family1).bulkLoadHFiles(list, false, null); 180 } 181 182 // after HBASE-24021 will throw FileNotFoundException, not MultipleIOException 183 @Test(expected = FileNotFoundException.class) 184 public void shouldThrowErrorIfMultiHFileDoesNotExist() throws IOException { 185 List<Pair<byte[], String>> list = new ArrayList<>(); 186 list.addAll(asList(withMissingHFileForFamily(family1))); 187 list.addAll(asList(withMissingHFileForFamily(family2))); 188 testRegionWithFamilies(family1, family2).bulkLoadHFiles(list, false, null); 189 } 190}