001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.apache.hadoop.hbase.HBaseTestingUtil.COLUMNS; 021import static org.apache.hadoop.hbase.HBaseTestingUtil.fam1; 022import static org.apache.hadoop.hbase.HBaseTestingUtil.fam2; 023import static org.apache.hadoop.hbase.HBaseTestingUtil.fam3; 024import static org.junit.jupiter.api.Assertions.assertArrayEquals; 025import static org.junit.jupiter.api.Assertions.assertEquals; 026import static org.junit.jupiter.api.Assertions.assertFalse; 027import static org.junit.jupiter.api.Assertions.assertInstanceOf; 028import static org.junit.jupiter.api.Assertions.assertNotNull; 029import static org.junit.jupiter.api.Assertions.assertNull; 030import static org.junit.jupiter.api.Assertions.assertThrows; 031import static org.junit.jupiter.api.Assertions.assertTrue; 032import static org.junit.jupiter.api.Assertions.fail; 033import static org.mockito.ArgumentMatchers.any; 034import static org.mockito.ArgumentMatchers.anyLong; 035import static org.mockito.ArgumentMatchers.anyString; 036import static org.mockito.ArgumentMatchers.isA; 037import static org.mockito.Mockito.atLeast; 038import static org.mockito.Mockito.doAnswer; 039import static org.mockito.Mockito.doThrow; 040import static org.mockito.Mockito.mock; 041import static org.mockito.Mockito.never; 042import static org.mockito.Mockito.spy; 043import static org.mockito.Mockito.times; 044import static org.mockito.Mockito.verify; 045import static org.mockito.Mockito.when; 046 047import java.io.IOException; 048import java.lang.reflect.Field; 049import java.math.BigDecimal; 050import java.security.PrivilegedExceptionAction; 051import java.util.ArrayList; 052import java.util.Arrays; 053import java.util.Collection; 054import java.util.HashSet; 055import java.util.List; 056import java.util.Map; 057import java.util.NavigableMap; 058import java.util.Objects; 059import java.util.Set; 060import java.util.TreeMap; 061import java.util.concurrent.Callable; 062import java.util.concurrent.CountDownLatch; 063import java.util.concurrent.ExecutorService; 064import java.util.concurrent.Executors; 065import java.util.concurrent.Future; 066import java.util.concurrent.TimeUnit; 067import java.util.concurrent.atomic.AtomicBoolean; 068import java.util.concurrent.atomic.AtomicReference; 069import java.util.stream.Collectors; 070import org.apache.commons.lang3.RandomStringUtils; 071import org.apache.hadoop.conf.Configuration; 072import org.apache.hadoop.fs.FSDataOutputStream; 073import org.apache.hadoop.fs.FileStatus; 074import org.apache.hadoop.fs.FileSystem; 075import org.apache.hadoop.fs.Path; 076import org.apache.hadoop.hbase.ArrayBackedTag; 077import org.apache.hadoop.hbase.Cell; 078import org.apache.hadoop.hbase.Cell.Type; 079import org.apache.hadoop.hbase.CellBuilderFactory; 080import org.apache.hadoop.hbase.CellBuilderType; 081import org.apache.hadoop.hbase.CellUtil; 082import org.apache.hadoop.hbase.CompareOperator; 083import org.apache.hadoop.hbase.CompatibilitySingletonFactory; 084import org.apache.hadoop.hbase.CompoundConfiguration; 085import org.apache.hadoop.hbase.DoNotRetryIOException; 086import org.apache.hadoop.hbase.DroppedSnapshotException; 087import org.apache.hadoop.hbase.ExtendedCell; 088import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; 089import org.apache.hadoop.hbase.HBaseConfiguration; 090import org.apache.hadoop.hbase.HBaseTestingUtil; 091import org.apache.hadoop.hbase.HConstants; 092import org.apache.hadoop.hbase.HConstants.OperationStatusCode; 093import org.apache.hadoop.hbase.HDFSBlocksDistribution; 094import org.apache.hadoop.hbase.KeyValue; 095import org.apache.hadoop.hbase.MultithreadedTestUtil; 096import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread; 097import org.apache.hadoop.hbase.NotServingRegionException; 098import org.apache.hadoop.hbase.PrivateCellUtil; 099import org.apache.hadoop.hbase.RegionTooBusyException; 100import org.apache.hadoop.hbase.ServerName; 101import org.apache.hadoop.hbase.SingleProcessHBaseCluster; 102import org.apache.hadoop.hbase.StartTestingClusterOption; 103import org.apache.hadoop.hbase.TableName; 104import org.apache.hadoop.hbase.TagType; 105import org.apache.hadoop.hbase.Waiter; 106import org.apache.hadoop.hbase.client.Append; 107import org.apache.hadoop.hbase.client.CheckAndMutate; 108import org.apache.hadoop.hbase.client.CheckAndMutateResult; 109import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 110import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 111import org.apache.hadoop.hbase.client.Delete; 112import org.apache.hadoop.hbase.client.Durability; 113import org.apache.hadoop.hbase.client.Get; 114import org.apache.hadoop.hbase.client.Increment; 115import org.apache.hadoop.hbase.client.Mutation; 116import org.apache.hadoop.hbase.client.Put; 117import org.apache.hadoop.hbase.client.RegionInfo; 118import org.apache.hadoop.hbase.client.RegionInfoBuilder; 119import org.apache.hadoop.hbase.client.Result; 120import org.apache.hadoop.hbase.client.RowMutations; 121import org.apache.hadoop.hbase.client.Scan; 122import org.apache.hadoop.hbase.client.Table; 123import org.apache.hadoop.hbase.client.TableDescriptor; 124import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 125import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; 126import org.apache.hadoop.hbase.coprocessor.MetaTableMetrics; 127import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor; 128import org.apache.hadoop.hbase.coprocessor.RegionObserver; 129import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException; 130import org.apache.hadoop.hbase.filter.BigDecimalComparator; 131import org.apache.hadoop.hbase.filter.BinaryComparator; 132import org.apache.hadoop.hbase.filter.ColumnCountGetFilter; 133import org.apache.hadoop.hbase.filter.Filter; 134import org.apache.hadoop.hbase.filter.FilterBase; 135import org.apache.hadoop.hbase.filter.FilterList; 136import org.apache.hadoop.hbase.filter.NullComparator; 137import org.apache.hadoop.hbase.filter.PrefixFilter; 138import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter; 139import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; 140import org.apache.hadoop.hbase.filter.SubstringComparator; 141import org.apache.hadoop.hbase.filter.ValueFilter; 142import org.apache.hadoop.hbase.io.TimeRange; 143import org.apache.hadoop.hbase.io.hfile.HFile; 144import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler; 145import org.apache.hadoop.hbase.monitoring.MonitoredTask; 146import org.apache.hadoop.hbase.monitoring.TaskMonitor; 147import org.apache.hadoop.hbase.regionserver.Region.Operation; 148import org.apache.hadoop.hbase.regionserver.Region.RowLock; 149import org.apache.hadoop.hbase.regionserver.TestHStore.FaultyFileSystem; 150import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl; 151import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; 152import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; 153import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL; 154import org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL; 155import org.apache.hadoop.hbase.regionserver.wal.FSHLog; 156import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource; 157import org.apache.hadoop.hbase.regionserver.wal.WALUtil; 158import org.apache.hadoop.hbase.replication.regionserver.ReplicationObserver; 159import org.apache.hadoop.hbase.security.User; 160import org.apache.hadoop.hbase.security.access.RegionReadOnlyController; 161import org.apache.hadoop.hbase.test.MetricsAssertHelper; 162import org.apache.hadoop.hbase.testclassification.LargeTests; 163import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; 164import org.apache.hadoop.hbase.util.Bytes; 165import org.apache.hadoop.hbase.util.CommonFSUtils; 166import org.apache.hadoop.hbase.util.CoprocessorConfigurationUtil; 167import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 168import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; 169import org.apache.hadoop.hbase.util.HFileArchiveUtil; 170import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge; 171import org.apache.hadoop.hbase.util.ManualEnvironmentEdge; 172import org.apache.hadoop.hbase.util.Threads; 173import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; 174import org.apache.hadoop.hbase.wal.FaultyFSLog; 175import org.apache.hadoop.hbase.wal.NettyAsyncFSWALConfigHelper; 176import org.apache.hadoop.hbase.wal.WAL; 177import org.apache.hadoop.hbase.wal.WALEdit; 178import org.apache.hadoop.hbase.wal.WALEditInternalHelper; 179import org.apache.hadoop.hbase.wal.WALFactory; 180import org.apache.hadoop.hbase.wal.WALKeyImpl; 181import org.apache.hadoop.hbase.wal.WALProvider; 182import org.apache.hadoop.hbase.wal.WALProvider.Writer; 183import org.apache.hadoop.hbase.wal.WALSplitUtil; 184import org.apache.hadoop.hbase.wal.WALStreamReader; 185import org.junit.jupiter.api.AfterEach; 186import org.junit.jupiter.api.BeforeEach; 187import org.junit.jupiter.api.Disabled; 188import org.junit.jupiter.api.Tag; 189import org.junit.jupiter.api.Test; 190import org.junit.jupiter.api.TestInfo; 191import org.mockito.ArgumentCaptor; 192import org.mockito.ArgumentMatcher; 193import org.mockito.invocation.InvocationOnMock; 194import org.mockito.stubbing.Answer; 195import org.slf4j.Logger; 196import org.slf4j.LoggerFactory; 197 198import org.apache.hbase.thirdparty.com.google.common.collect.Lists; 199import org.apache.hbase.thirdparty.com.google.protobuf.ByteString; 200import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup; 201import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup; 202import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel; 203 204import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 205import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor; 206import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor; 207import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.FlushAction; 208import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor; 209import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor; 210import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.StoreDescriptor; 211 212/** 213 * Basic stand-alone testing of HRegion. No clusters! A lot of the meta information for an HRegion 214 * now lives inside other HRegions or in the HBaseMaster, so only basic testing is possible. 215 */ 216@Tag(VerySlowRegionServerTests.TAG) 217@Tag(LargeTests.TAG) 218@SuppressWarnings("deprecation") 219public class TestHRegion { 220 221 // Do not spin up clusters in here. If you need to spin up a cluster, do it 222 // over in TestHRegionOnCluster. 223 private static final Logger LOG = LoggerFactory.getLogger(TestHRegion.class); 224 private String name; 225 private static final String COLUMN_FAMILY = "MyCF"; 226 private static final byte[] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY); 227 private static final EventLoopGroup GROUP = new NioEventLoopGroup(); 228 229 HRegion region = null; 230 // Do not run unit tests in parallel (? Why not? It don't work? Why not? St.Ack) 231 protected static HBaseTestingUtil TEST_UTIL; 232 public static Configuration CONF; 233 private String dir; 234 private final int MAX_VERSIONS = 2; 235 236 // Test names 237 protected TableName tableName; 238 protected String method; 239 protected final byte[] qual = Bytes.toBytes("qual"); 240 protected final byte[] qual1 = Bytes.toBytes("qual1"); 241 protected final byte[] qual2 = Bytes.toBytes("qual2"); 242 protected final byte[] qual3 = Bytes.toBytes("qual3"); 243 protected final byte[] value = Bytes.toBytes("value"); 244 protected final byte[] value1 = Bytes.toBytes("value1"); 245 protected final byte[] value2 = Bytes.toBytes("value2"); 246 protected final byte[] row = Bytes.toBytes("rowA"); 247 protected final byte[] row2 = Bytes.toBytes("rowB"); 248 249 protected final MetricsAssertHelper metricsAssertHelper = 250 CompatibilitySingletonFactory.getInstance(MetricsAssertHelper.class); 251 252 @BeforeEach 253 public void setup(TestInfo testInfo) throws IOException { 254 this.name = testInfo.getTestMethod().get().getName(); 255 TEST_UTIL = new HBaseTestingUtil(); 256 CONF = TEST_UTIL.getConfiguration(); 257 NettyAsyncFSWALConfigHelper.setEventLoopConfig(CONF, GROUP, NioSocketChannel.class); 258 dir = TEST_UTIL.getDataTestDir("TestHRegion").toString(); 259 method = name; 260 tableName = TableName.valueOf(method); 261 CONF.set(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, String.valueOf(0.09)); 262 CONF.setLong(AbstractFSWAL.WAL_SYNC_TIMEOUT_MS, 10000); 263 } 264 265 @AfterEach 266 public void tearDown() throws IOException { 267 // Region may have been closed, but it is still no harm if we close it again here using HTU. 268 HBaseTestingUtil.closeRegionAndWAL(region); 269 EnvironmentEdgeManagerTestHelper.reset(); 270 LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir()); 271 TEST_UTIL.cleanupTestDir(); 272 } 273 274 /** 275 * Test that I can use the max flushed sequence id after the close. 276 */ 277 278 @Test 279 public void testSequenceId() throws IOException { 280 region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES); 281 assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId()); 282 // Weird. This returns 0 if no store files or no edits. Afraid to change it. 283 assertEquals(0, (long) region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES)); 284 HBaseTestingUtil.closeRegionAndWAL(this.region); 285 assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId()); 286 assertEquals(0, (long) region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES)); 287 HRegion oldRegion = region; 288 try { 289 // Open region again. 290 region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES); 291 byte[] value = Bytes.toBytes(method); 292 // Make a random put against our cf. 293 Put put = new Put(value); 294 put.addColumn(COLUMN_FAMILY_BYTES, null, value); 295 region.put(put); 296 // No flush yet so init numbers should still be in place. 297 assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId()); 298 assertEquals(0, (long) region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES)); 299 region.flush(true); 300 long max = region.getMaxFlushedSeqId(); 301 HBaseTestingUtil.closeRegionAndWAL(this.region); 302 assertEquals(max, region.getMaxFlushedSeqId()); 303 this.region = null; 304 } finally { 305 HBaseTestingUtil.closeRegionAndWAL(oldRegion); 306 } 307 } 308 309 /** 310 * Test for Bug 2 of HBASE-10466. "Bug 2: Conditions for the first flush of region close 311 * (so-called pre-flush) If memstoreSize is smaller than a certain value, or when region close 312 * starts a flush is ongoing, the first flush is skipped and only the second flush takes place. 313 * However, two flushes are required in case previous flush fails and leaves some data in 314 * snapshot. The bug could cause loss of data in current memstore. The fix is removing all 315 * conditions except abort check so we ensure 2 flushes for region close." 316 */ 317 @Test 318 public void testCloseCarryingSnapshot() throws IOException { 319 region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES); 320 HStore store = region.getStore(COLUMN_FAMILY_BYTES); 321 // Get some random bytes. 322 byte[] value = Bytes.toBytes(method); 323 // Make a random put against our cf. 324 Put put = new Put(value); 325 put.addColumn(COLUMN_FAMILY_BYTES, null, value); 326 // First put something in current memstore, which will be in snapshot after flusher.prepare() 327 region.put(put); 328 StoreFlushContext storeFlushCtx = store.createFlushContext(12345, FlushLifeCycleTracker.DUMMY); 329 storeFlushCtx.prepare(); 330 // Second put something in current memstore 331 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value); 332 region.put(put); 333 // Close with something in memstore and something in the snapshot. Make sure all is cleared. 334 HBaseTestingUtil.closeRegionAndWAL(region); 335 assertEquals(0, region.getMemStoreDataSize()); 336 region = null; 337 } 338 339 /* 340 * This test is for verifying memstore snapshot size is correctly updated in case of rollback See 341 * HBASE-10845 342 */ 343 @Test 344 public void testMemstoreSnapshotSize() throws IOException { 345 class MyFaultyFSLog extends FaultyFSLog { 346 StoreFlushContext storeFlushCtx; 347 348 public MyFaultyFSLog(FileSystem fs, Path rootDir, String logName, Configuration conf) 349 throws IOException { 350 super(fs, rootDir, logName, conf); 351 } 352 353 void setStoreFlushCtx(StoreFlushContext storeFlushCtx) { 354 this.storeFlushCtx = storeFlushCtx; 355 } 356 357 @Override 358 protected void doSync(long txid, boolean forceSync) throws IOException { 359 storeFlushCtx.prepare(); 360 super.doSync(txid, forceSync); 361 } 362 } 363 364 FileSystem fs = FileSystem.get(CONF); 365 Path rootDir = new Path(dir + method); 366 fs.mkdirs(new Path(rootDir, method)); 367 MyFaultyFSLog faultyLog = new MyFaultyFSLog(fs, rootDir, method, CONF); 368 faultyLog.init(); 369 region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, faultyLog, 370 COLUMN_FAMILY_BYTES); 371 372 HStore store = region.getStore(COLUMN_FAMILY_BYTES); 373 // Get some random bytes. 374 byte[] value = Bytes.toBytes(method); 375 faultyLog.setStoreFlushCtx(store.createFlushContext(12345, FlushLifeCycleTracker.DUMMY)); 376 377 Put put = new Put(value); 378 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value); 379 faultyLog.setFailureType(FaultyFSLog.FailureType.SYNC); 380 boolean threwIOE = false; 381 try { 382 region.put(put); 383 } catch (IOException ioe) { 384 threwIOE = true; 385 } finally { 386 assertTrue(threwIOE, "The regionserver should have thrown an exception"); 387 } 388 MemStoreSize mss = store.getFlushableSize(); 389 assertTrue(mss.getDataSize() == 0, "flushable size should be zero, but it is " + mss); 390 } 391 392 /** 393 * Create a WAL outside of the usual helper in 394 * {@link HBaseTestingUtil#createWal(Configuration, Path, RegionInfo)} because that method doesn't 395 * play nicely with FaultyFileSystem. Call this method before overriding {@code fs.file.impl}. 396 * @param callingMethod a unique component for the path, probably the name of the test method. 397 */ 398 private static WAL createWALCompatibleWithFaultyFileSystem(String callingMethod, 399 Configuration conf, TableName tableName) throws IOException { 400 final Path logDir = TEST_UTIL.getDataTestDirOnTestFS(callingMethod + ".log"); 401 final Configuration walConf = new Configuration(conf); 402 CommonFSUtils.setRootDir(walConf, logDir); 403 return new WALFactory(walConf, callingMethod) 404 .getWAL(RegionInfoBuilder.newBuilder(tableName).build()); 405 } 406 407 @Test 408 public void testMemstoreSizeAccountingWithFailedPostBatchMutate() throws IOException { 409 FileSystem fs = FileSystem.get(CONF); 410 Path rootDir = new Path(dir + method); 411 fs.mkdirs(new Path(rootDir, method)); 412 FSHLog hLog = new FSHLog(fs, rootDir, method, CONF); 413 hLog.init(); 414 region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, hLog, 415 COLUMN_FAMILY_BYTES); 416 HStore store = region.getStore(COLUMN_FAMILY_BYTES); 417 assertEquals(0, region.getMemStoreDataSize()); 418 419 // Put one value 420 byte[] value = Bytes.toBytes(method); 421 Put put = new Put(value); 422 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value); 423 region.put(put); 424 long onePutSize = region.getMemStoreDataSize(); 425 assertTrue(onePutSize > 0); 426 427 RegionCoprocessorHost mockedCPHost = mock(RegionCoprocessorHost.class); 428 doThrow(new IOException()).when(mockedCPHost).postBatchMutate(any()); 429 region.setCoprocessorHost(mockedCPHost); 430 431 put = new Put(value); 432 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("dfg"), value); 433 try { 434 region.put(put); 435 fail("Should have failed with IOException"); 436 } catch (IOException expected) { 437 } 438 long expectedSize = onePutSize * 2; 439 assertEquals(expectedSize, region.getMemStoreDataSize(), "memstoreSize should be incremented"); 440 assertEquals(expectedSize, store.getFlushableSize().getDataSize(), 441 "flushable size should be incremented"); 442 443 region.setCoprocessorHost(null); 444 } 445 446 /** 447 * A test case of HBASE-21041 448 */ 449 @Test 450 public void testFlushAndMemstoreSizeCounting() throws Exception { 451 byte[] family = Bytes.toBytes("family"); 452 this.region = initHRegion(tableName, method, CONF, family); 453 for (byte[] row : HBaseTestingUtil.ROWS) { 454 Put put = new Put(row); 455 put.addColumn(family, family, row); 456 region.put(put); 457 } 458 region.flush(true); 459 // After flush, data size should be zero 460 assertEquals(0, region.getMemStoreDataSize()); 461 // After flush, a new active mutable segment is created, so the heap size 462 // should equal to MutableSegment.DEEP_OVERHEAD 463 assertEquals(MutableSegment.DEEP_OVERHEAD, region.getMemStoreHeapSize()); 464 // After flush, offheap should be zero 465 assertEquals(0, region.getMemStoreOffHeapSize()); 466 } 467 468 /** 469 * Test we do not lose data if we fail a flush and then close. Part of HBase-10466. Tests the 470 * following from the issue description: "Bug 1: Wrong calculation of HRegion.memstoreSize: When a 471 * flush fails, data to be flushed is kept in each MemStore's snapshot and wait for next flush 472 * attempt to continue on it. But when the next flush succeeds, the counter of total memstore size 473 * in HRegion is always deduced by the sum of current memstore sizes instead of snapshots left 474 * from previous failed flush. This calculation is problematic that almost every time there is 475 * failed flush, HRegion.memstoreSize gets reduced by a wrong value. If region flush could not 476 * proceed for a couple cycles, the size in current memstore could be much larger than the 477 * snapshot. It's likely to drift memstoreSize much smaller than expected. In extreme case, if the 478 * error accumulates to even bigger than HRegion's memstore size limit, any further flush is 479 * skipped because flush does not do anything if memstoreSize is not larger than 0." 480 */ 481 @Test 482 public void testFlushSizeAccounting() throws Exception { 483 final Configuration conf = HBaseConfiguration.create(CONF); 484 final WAL wal = createWALCompatibleWithFaultyFileSystem(method, conf, tableName); 485 // Only retry once. 486 conf.setInt("hbase.hstore.flush.retries.number", 1); 487 final User user = User.createUserForTesting(conf, method, new String[] { "foo" }); 488 // Inject our faulty LocalFileSystem 489 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class); 490 user.runAs(new PrivilegedExceptionAction<Object>() { 491 @Override 492 public Object run() throws Exception { 493 // Make sure it worked (above is sensitive to caching details in hadoop core) 494 FileSystem fs = FileSystem.get(conf); 495 assertEquals(FaultyFileSystem.class, fs.getClass()); 496 FaultyFileSystem ffs = (FaultyFileSystem) fs; 497 HRegion region = null; 498 try { 499 // Initialize region 500 region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, wal, 501 COLUMN_FAMILY_BYTES); 502 long size = region.getMemStoreDataSize(); 503 assertEquals(0, size); 504 // Put one item into memstore. Measure the size of one item in memstore. 505 Put p1 = new Put(row); 506 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[]) null)); 507 region.put(p1); 508 final long sizeOfOnePut = region.getMemStoreDataSize(); 509 // Fail a flush which means the current memstore will hang out as memstore 'snapshot'. 510 try { 511 LOG.info("Flushing"); 512 region.flush(true); 513 fail("Didn't bubble up IOE!"); 514 } catch (DroppedSnapshotException dse) { 515 // What we are expecting 516 region.closing.set(false); // this is needed for the rest of the test to work 517 } 518 // Make it so all writes succeed from here on out 519 ffs.fault.set(false); 520 // Check sizes. Should still be the one entry. 521 assertEquals(sizeOfOnePut, region.getMemStoreDataSize()); 522 // Now add two entries so that on this next flush that fails, we can see if we 523 // subtract the right amount, the snapshot size only. 524 Put p2 = new Put(row); 525 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[]) null)); 526 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[]) null)); 527 region.put(p2); 528 long expectedSize = sizeOfOnePut * 3; 529 assertEquals(expectedSize, region.getMemStoreDataSize()); 530 // Do a successful flush. It will clear the snapshot only. Thats how flushes work. 531 // If already a snapshot, we clear it else we move the memstore to be snapshot and flush 532 // it 533 region.flush(true); 534 // Make sure our memory accounting is right. 535 assertEquals(sizeOfOnePut * 2, region.getMemStoreDataSize()); 536 } finally { 537 HBaseTestingUtil.closeRegionAndWAL(region); 538 } 539 return null; 540 } 541 }); 542 FileSystem.closeAllForUGI(user.getUGI()); 543 } 544 545 @Test 546 public void testCloseWithFailingFlush() throws Exception { 547 final Configuration conf = HBaseConfiguration.create(CONF); 548 final WAL wal = createWALCompatibleWithFaultyFileSystem(method, conf, tableName); 549 // Only retry once. 550 conf.setInt("hbase.hstore.flush.retries.number", 1); 551 final User user = User.createUserForTesting(conf, this.method, new String[] { "foo" }); 552 // Inject our faulty LocalFileSystem 553 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class); 554 user.runAs(new PrivilegedExceptionAction<Object>() { 555 @Override 556 public Object run() throws Exception { 557 // Make sure it worked (above is sensitive to caching details in hadoop core) 558 FileSystem fs = FileSystem.get(conf); 559 assertEquals(FaultyFileSystem.class, fs.getClass()); 560 FaultyFileSystem ffs = (FaultyFileSystem) fs; 561 HRegion region = null; 562 try { 563 // Initialize region 564 region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, wal, 565 COLUMN_FAMILY_BYTES); 566 long size = region.getMemStoreDataSize(); 567 assertEquals(0, size); 568 // Put one item into memstore. Measure the size of one item in memstore. 569 Put p1 = new Put(row); 570 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[]) null)); 571 region.put(p1); 572 // Manufacture an outstanding snapshot -- fake a failed flush by doing prepare step only. 573 HStore store = region.getStore(COLUMN_FAMILY_BYTES); 574 StoreFlushContext storeFlushCtx = 575 store.createFlushContext(12345, FlushLifeCycleTracker.DUMMY); 576 storeFlushCtx.prepare(); 577 // Now add two entries to the foreground memstore. 578 Put p2 = new Put(row); 579 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[]) null)); 580 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[]) null)); 581 region.put(p2); 582 // Now try close on top of a failing flush. 583 HBaseTestingUtil.closeRegionAndWAL(region); 584 region = null; 585 fail(); 586 } catch (DroppedSnapshotException dse) { 587 // Expected 588 LOG.info("Expected DroppedSnapshotException"); 589 } finally { 590 // Make it so all writes succeed from here on out so can close clean 591 ffs.fault.set(false); 592 HBaseTestingUtil.closeRegionAndWAL(region); 593 } 594 return null; 595 } 596 }); 597 FileSystem.closeAllForUGI(user.getUGI()); 598 } 599 600 @Test 601 public void testCompactionAffectedByScanners() throws Exception { 602 byte[] family = Bytes.toBytes("family"); 603 this.region = initHRegion(tableName, method, CONF, family); 604 605 Put put = new Put(Bytes.toBytes("r1")); 606 put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); 607 region.put(put); 608 region.flush(true); 609 610 Scan scan = new Scan(); 611 scan.readVersions(3); 612 // open the first scanner 613 try (RegionScanner scanner1 = region.getScanner(scan)) { 614 Delete delete = new Delete(Bytes.toBytes("r1")); 615 region.delete(delete); 616 region.flush(true); 617 // open the second scanner 618 try (RegionScanner scanner2 = region.getScanner(scan)) { 619 List<Cell> results = new ArrayList<>(); 620 621 LOG.info("Smallest read point:" + region.getSmallestReadPoint()); 622 623 // make a major compaction 624 region.compact(true); 625 626 // open the third scanner 627 try (RegionScanner scanner3 = region.getScanner(scan)) { 628 // get data from scanner 1, 2, 3 after major compaction 629 scanner1.next(results); 630 LOG.info(results.toString()); 631 assertEquals(1, results.size()); 632 633 results.clear(); 634 scanner2.next(results); 635 LOG.info(results.toString()); 636 assertEquals(0, results.size()); 637 638 results.clear(); 639 scanner3.next(results); 640 LOG.info(results.toString()); 641 assertEquals(0, results.size()); 642 } 643 } 644 } 645 } 646 647 @Test 648 public void testToShowNPEOnRegionScannerReseek() throws Exception { 649 byte[] family = Bytes.toBytes("family"); 650 this.region = initHRegion(tableName, method, CONF, family); 651 652 Put put = new Put(Bytes.toBytes("r1")); 653 put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); 654 region.put(put); 655 put = new Put(Bytes.toBytes("r2")); 656 put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); 657 region.put(put); 658 region.flush(true); 659 660 Scan scan = new Scan(); 661 scan.readVersions(3); 662 // open the first scanner 663 try (RegionScanner scanner1 = region.getScanner(scan)) { 664 LOG.info("Smallest read point:" + region.getSmallestReadPoint()); 665 666 region.compact(true); 667 668 scanner1.reseek(Bytes.toBytes("r2")); 669 List<Cell> results = new ArrayList<>(); 670 scanner1.next(results); 671 Cell keyValue = results.get(0); 672 assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0); 673 scanner1.close(); 674 } 675 } 676 677 @Test 678 public void testArchiveRecoveredEditsReplay() throws Exception { 679 byte[] family = Bytes.toBytes("family"); 680 this.region = initHRegion(tableName, method, CONF, family); 681 final WALFactory wals = new WALFactory(CONF, method); 682 try { 683 Path regiondir = region.getRegionFileSystem().getRegionDir(); 684 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 685 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 686 687 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 688 689 long maxSeqId = 1050; 690 long minSeqId = 1000; 691 692 for (long i = minSeqId; i <= maxSeqId; i += 10) { 693 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); 694 fs.create(recoveredEdits); 695 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 696 697 long time = System.nanoTime(); 698 WALEdit edit = new WALEdit(); 699 WALEditInternalHelper.addExtendedCell(edit, 700 new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i))); 701 writer.append(new WAL.Entry( 702 new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); 703 704 writer.close(); 705 } 706 MonitoredTask status = TaskMonitor.get().createStatus(method); 707 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 708 for (HStore store : region.getStores()) { 709 maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId - 1); 710 } 711 CONF.set("hbase.region.archive.recovered.edits", "true"); 712 CONF.set(CommonFSUtils.HBASE_WAL_DIR, "/custom_wal_dir"); 713 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status); 714 assertEquals(maxSeqId, seqId); 715 region.getMVCC().advanceTo(seqId); 716 String fakeFamilyName = recoveredEditsDir.getName(); 717 Path rootDir = new Path(CONF.get(HConstants.HBASE_DIR)); 718 Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePathForRootDir(rootDir, 719 region.getRegionInfo(), Bytes.toBytes(fakeFamilyName)); 720 FileStatus[] list = TEST_UTIL.getTestFileSystem().listStatus(storeArchiveDir); 721 assertEquals(6, list.length); 722 } finally { 723 CONF.set("hbase.region.archive.recovered.edits", "false"); 724 CONF.set(CommonFSUtils.HBASE_WAL_DIR, ""); 725 HBaseTestingUtil.closeRegionAndWAL(this.region); 726 this.region = null; 727 wals.close(); 728 } 729 } 730 731 @Test 732 public void testSkipRecoveredEditsReplay() throws Exception { 733 byte[] family = Bytes.toBytes("family"); 734 this.region = initHRegion(tableName, method, CONF, family); 735 final WALFactory wals = new WALFactory(CONF, method); 736 try { 737 Path regiondir = region.getRegionFileSystem().getRegionDir(); 738 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 739 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 740 741 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 742 743 long maxSeqId = 1050; 744 long minSeqId = 1000; 745 746 for (long i = minSeqId; i <= maxSeqId; i += 10) { 747 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); 748 fs.create(recoveredEdits); 749 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 750 751 long time = System.nanoTime(); 752 WALEdit edit = new WALEdit(); 753 WALEditInternalHelper.addExtendedCell(edit, 754 new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i))); 755 writer.append(new WAL.Entry( 756 new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); 757 758 writer.close(); 759 } 760 MonitoredTask status = TaskMonitor.get().createStatus(method); 761 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 762 for (HStore store : region.getStores()) { 763 maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId - 1); 764 } 765 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status); 766 assertEquals(maxSeqId, seqId); 767 region.getMVCC().advanceTo(seqId); 768 Get get = new Get(row); 769 Result result = region.get(get); 770 for (long i = minSeqId; i <= maxSeqId; i += 10) { 771 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i)); 772 assertEquals(1, kvs.size()); 773 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0))); 774 } 775 } finally { 776 HBaseTestingUtil.closeRegionAndWAL(this.region); 777 this.region = null; 778 wals.close(); 779 } 780 } 781 782 @Test 783 public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception { 784 byte[] family = Bytes.toBytes("family"); 785 this.region = initHRegion(tableName, method, CONF, family); 786 final WALFactory wals = new WALFactory(CONF, method); 787 try { 788 Path regiondir = region.getRegionFileSystem().getRegionDir(); 789 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 790 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 791 792 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 793 794 long maxSeqId = 1050; 795 long minSeqId = 1000; 796 797 for (long i = minSeqId; i <= maxSeqId; i += 10) { 798 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); 799 fs.create(recoveredEdits); 800 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 801 802 long time = System.nanoTime(); 803 WALEdit edit = new WALEdit(); 804 WALEditInternalHelper.addExtendedCell(edit, 805 new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i))); 806 writer.append(new WAL.Entry( 807 new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); 808 809 writer.close(); 810 } 811 long recoverSeqId = 1030; 812 MonitoredTask status = TaskMonitor.get().createStatus(method); 813 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 814 for (HStore store : region.getStores()) { 815 maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), recoverSeqId - 1); 816 } 817 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status); 818 assertEquals(maxSeqId, seqId); 819 region.getMVCC().advanceTo(seqId); 820 Get get = new Get(row); 821 Result result = region.get(get); 822 for (long i = minSeqId; i <= maxSeqId; i += 10) { 823 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i)); 824 if (i < recoverSeqId) { 825 assertEquals(0, kvs.size()); 826 } else { 827 assertEquals(1, kvs.size()); 828 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0))); 829 } 830 } 831 } finally { 832 HBaseTestingUtil.closeRegionAndWAL(this.region); 833 this.region = null; 834 wals.close(); 835 } 836 } 837 838 @Test 839 public void testSkipRecoveredEditsReplayAllIgnored() throws Exception { 840 byte[] family = Bytes.toBytes("family"); 841 this.region = initHRegion(tableName, method, CONF, family); 842 Path regiondir = region.getRegionFileSystem().getRegionDir(); 843 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 844 845 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 846 for (int i = 1000; i < 1050; i += 10) { 847 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); 848 FSDataOutputStream dos = fs.create(recoveredEdits); 849 dos.writeInt(i); 850 dos.close(); 851 } 852 long minSeqId = 2000; 853 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1)); 854 FSDataOutputStream dos = fs.create(recoveredEdits); 855 dos.close(); 856 857 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 858 for (HStore store : region.getStores()) { 859 maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId); 860 } 861 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, null); 862 assertEquals(minSeqId, seqId); 863 } 864 865 @Test 866 public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception { 867 byte[] family = Bytes.toBytes("family"); 868 this.region = initHRegion(tableName, method, CONF, family); 869 final WALFactory wals = new WALFactory(CONF, method); 870 try { 871 Path regiondir = region.getRegionFileSystem().getRegionDir(); 872 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 873 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 874 byte[][] columns = region.getTableDescriptor().getColumnFamilyNames().toArray(new byte[0][]); 875 876 assertEquals(0, region.getStoreFileList(columns).size()); 877 878 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 879 880 long maxSeqId = 1050; 881 long minSeqId = 1000; 882 883 for (long i = minSeqId; i <= maxSeqId; i += 10) { 884 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); 885 fs.create(recoveredEdits); 886 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 887 888 long time = System.nanoTime(); 889 WALEdit edit = null; 890 if (i == maxSeqId) { 891 edit = WALEdit.createCompaction(region.getRegionInfo(), 892 CompactionDescriptor.newBuilder().setTableName(ByteString.copyFrom(tableName.getName())) 893 .setFamilyName(ByteString.copyFrom(regionName)) 894 .setEncodedRegionName(ByteString.copyFrom(regionName)) 895 .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString()))) 896 .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName())).build()); 897 } else { 898 edit = new WALEdit(); 899 WALEditInternalHelper.addExtendedCell(edit, 900 new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i))); 901 } 902 writer.append(new WAL.Entry( 903 new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); 904 writer.close(); 905 } 906 907 long recoverSeqId = 1030; 908 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 909 MonitoredTask status = TaskMonitor.get().createStatus(method); 910 for (HStore store : region.getStores()) { 911 maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), recoverSeqId - 1); 912 } 913 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status); 914 assertEquals(maxSeqId, seqId); 915 916 // assert that the files are flushed 917 assertEquals(1, region.getStoreFileList(columns).size()); 918 919 } finally { 920 HBaseTestingUtil.closeRegionAndWAL(this.region); 921 this.region = null; 922 wals.close(); 923 } 924 } 925 926 @Test 927 public void testRecoveredEditsReplayCompaction() throws Exception { 928 testRecoveredEditsReplayCompaction(false); 929 testRecoveredEditsReplayCompaction(true); 930 } 931 932 public void testRecoveredEditsReplayCompaction(boolean mismatchedRegionName) throws Exception { 933 CONF.setClass(HConstants.REGION_IMPL, HRegionForTesting.class, Region.class); 934 byte[] family = Bytes.toBytes("family"); 935 this.region = initHRegion(tableName, method, CONF, family); 936 final WALFactory wals = new WALFactory(CONF, method); 937 try { 938 Path regiondir = region.getRegionFileSystem().getRegionDir(); 939 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 940 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 941 942 long maxSeqId = 3; 943 long minSeqId = 0; 944 945 for (long i = minSeqId; i < maxSeqId; i++) { 946 Put put = new Put(Bytes.toBytes(i)); 947 put.addColumn(family, Bytes.toBytes(i), Bytes.toBytes(i)); 948 region.put(put); 949 region.flush(true); 950 } 951 952 // this will create a region with 3 files 953 assertEquals(3, region.getStore(family).getStorefilesCount()); 954 List<Path> storeFiles = new ArrayList<>(3); 955 for (HStoreFile sf : region.getStore(family).getStorefiles()) { 956 storeFiles.add(sf.getPath()); 957 } 958 959 // disable compaction completion 960 CONF.setBoolean("hbase.hstore.compaction.complete", false); 961 region.compactStores(); 962 963 // ensure that nothing changed 964 assertEquals(3, region.getStore(family).getStorefilesCount()); 965 966 // now find the compacted file, and manually add it to the recovered edits 967 Path tmpDir = new Path(region.getRegionFileSystem().getTempDir(), Bytes.toString(family)); 968 FileStatus[] files = CommonFSUtils.listStatus(fs, tmpDir); 969 String errorMsg = "Expected to find 1 file in the region temp directory " 970 + "from the compaction, could not find any"; 971 assertNotNull(files, errorMsg); 972 assertEquals(1, files.length, errorMsg); 973 // move the file inside region dir 974 Path newFile = 975 region.getRegionFileSystem().commitStoreFile(Bytes.toString(family), files[0].getPath()); 976 977 byte[] encodedNameAsBytes = this.region.getRegionInfo().getEncodedNameAsBytes(); 978 byte[] fakeEncodedNameAsBytes = new byte[encodedNameAsBytes.length]; 979 for (int i = 0; i < encodedNameAsBytes.length; i++) { 980 // Mix the byte array to have a new encodedName 981 fakeEncodedNameAsBytes[i] = (byte) (encodedNameAsBytes[i] + 1); 982 } 983 984 CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor( 985 this.region.getRegionInfo(), mismatchedRegionName ? fakeEncodedNameAsBytes : null, family, 986 storeFiles, Lists.newArrayList(newFile), 987 region.getRegionFileSystem().getStoreDir(Bytes.toString(family))); 988 989 WALUtil.writeCompactionMarker(region.getWAL(), this.region.getReplicationScope(), 990 this.region.getRegionInfo(), compactionDescriptor, region.getMVCC(), null); 991 992 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 993 994 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000)); 995 fs.create(recoveredEdits); 996 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 997 998 long time = System.nanoTime(); 999 1000 writer.append(new WAL.Entry( 1001 new WALKeyImpl(regionName, tableName, 10, time, HConstants.DEFAULT_CLUSTER_ID), 1002 WALEdit.createCompaction(region.getRegionInfo(), compactionDescriptor))); 1003 writer.close(); 1004 1005 // close the region now, and reopen again 1006 region.getTableDescriptor(); 1007 region.getRegionInfo(); 1008 HBaseTestingUtil.closeRegionAndWAL(this.region); 1009 try { 1010 region = HRegion.openHRegion(region, null); 1011 } catch (WrongRegionException wre) { 1012 fail("Matching encoded region name should not have produced WrongRegionException"); 1013 } 1014 1015 // now check whether we have only one store file, the compacted one 1016 Collection<HStoreFile> sfs = region.getStore(family).getStorefiles(); 1017 for (HStoreFile sf : sfs) { 1018 LOG.info(Objects.toString(sf.getPath())); 1019 } 1020 if (!mismatchedRegionName) { 1021 assertEquals(1, region.getStore(family).getStorefilesCount()); 1022 } 1023 files = CommonFSUtils.listStatus(fs, tmpDir); 1024 assertTrue(files == null || files.length == 0, "Expected to find 0 files inside " + tmpDir); 1025 1026 for (long i = minSeqId; i < maxSeqId; i++) { 1027 Get get = new Get(Bytes.toBytes(i)); 1028 Result result = region.get(get); 1029 byte[] value = result.getValue(family, Bytes.toBytes(i)); 1030 assertArrayEquals(Bytes.toBytes(i), value); 1031 } 1032 } finally { 1033 HBaseTestingUtil.closeRegionAndWAL(this.region); 1034 this.region = null; 1035 wals.close(); 1036 CONF.setClass(HConstants.REGION_IMPL, HRegion.class, Region.class); 1037 } 1038 } 1039 1040 @Test 1041 public void testFlushMarkers() throws Exception { 1042 // tests that flush markers are written to WAL and handled at recovered edits 1043 byte[] family = Bytes.toBytes("family"); 1044 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + ".log"); 1045 final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration()); 1046 CommonFSUtils.setRootDir(walConf, logDir); 1047 final WALFactory wals = new WALFactory(walConf, method); 1048 final WAL wal = wals.getWAL(RegionInfoBuilder.newBuilder(tableName).build()); 1049 1050 this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF, 1051 false, Durability.USE_DEFAULT, wal, family); 1052 try { 1053 Path regiondir = region.getRegionFileSystem().getRegionDir(); 1054 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 1055 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 1056 1057 long maxSeqId = 3; 1058 long minSeqId = 0; 1059 1060 for (long i = minSeqId; i < maxSeqId; i++) { 1061 Put put = new Put(Bytes.toBytes(i)); 1062 put.addColumn(family, Bytes.toBytes(i), Bytes.toBytes(i)); 1063 region.put(put); 1064 region.flush(true); 1065 } 1066 1067 // this will create a region with 3 files from flush 1068 assertEquals(3, region.getStore(family).getStorefilesCount()); 1069 List<String> storeFiles = new ArrayList<>(3); 1070 for (HStoreFile sf : region.getStore(family).getStorefiles()) { 1071 storeFiles.add(sf.getPath().getName()); 1072 } 1073 1074 // now verify that the flush markers are written 1075 wal.shutdown(); 1076 WALStreamReader reader = WALFactory.createStreamReader(fs, 1077 AbstractFSWALProvider.getCurrentFileName(wal), TEST_UTIL.getConfiguration()); 1078 try { 1079 List<WAL.Entry> flushDescriptors = new ArrayList<>(); 1080 long lastFlushSeqId = -1; 1081 while (true) { 1082 WAL.Entry entry = reader.next(); 1083 if (entry == null) { 1084 break; 1085 } 1086 Cell cell = entry.getEdit().getCells().get(0); 1087 if (WALEdit.isMetaEditFamily(cell)) { 1088 FlushDescriptor flushDesc = WALEdit.getFlushDescriptor(cell); 1089 assertNotNull(flushDesc); 1090 assertArrayEquals(tableName.getName(), flushDesc.getTableName().toByteArray()); 1091 if (flushDesc.getAction() == FlushAction.START_FLUSH) { 1092 assertTrue(flushDesc.getFlushSequenceNumber() > lastFlushSeqId); 1093 } else if (flushDesc.getAction() == FlushAction.COMMIT_FLUSH) { 1094 assertTrue(flushDesc.getFlushSequenceNumber() == lastFlushSeqId); 1095 } 1096 lastFlushSeqId = flushDesc.getFlushSequenceNumber(); 1097 assertArrayEquals(regionName, flushDesc.getEncodedRegionName().toByteArray()); 1098 assertEquals(1, flushDesc.getStoreFlushesCount()); // only one store 1099 StoreFlushDescriptor storeFlushDesc = flushDesc.getStoreFlushes(0); 1100 assertArrayEquals(family, storeFlushDesc.getFamilyName().toByteArray()); 1101 assertEquals("family", storeFlushDesc.getStoreHomeDir()); 1102 if (flushDesc.getAction() == FlushAction.START_FLUSH) { 1103 assertEquals(0, storeFlushDesc.getFlushOutputCount()); 1104 } else { 1105 assertEquals(1, storeFlushDesc.getFlushOutputCount()); // only one file from flush 1106 assertTrue(storeFiles.contains(storeFlushDesc.getFlushOutput(0))); 1107 } 1108 1109 flushDescriptors.add(entry); 1110 } 1111 } 1112 1113 assertEquals(3 * 2, flushDescriptors.size()); // START_FLUSH and COMMIT_FLUSH per flush 1114 1115 // now write those markers to the recovered edits again. 1116 1117 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 1118 1119 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000)); 1120 fs.create(recoveredEdits); 1121 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 1122 1123 for (WAL.Entry entry : flushDescriptors) { 1124 writer.append(entry); 1125 } 1126 writer.close(); 1127 } finally { 1128 reader.close(); 1129 } 1130 1131 // close the region now, and reopen again 1132 HBaseTestingUtil.closeRegionAndWAL(this.region); 1133 region = HRegion.openHRegion(region, null); 1134 1135 // now check whether we have can read back the data from region 1136 for (long i = minSeqId; i < maxSeqId; i++) { 1137 Get get = new Get(Bytes.toBytes(i)); 1138 Result result = region.get(get); 1139 byte[] value = result.getValue(family, Bytes.toBytes(i)); 1140 assertArrayEquals(Bytes.toBytes(i), value); 1141 } 1142 } finally { 1143 HBaseTestingUtil.closeRegionAndWAL(this.region); 1144 this.region = null; 1145 wals.close(); 1146 } 1147 } 1148 1149 static class IsFlushWALMarker implements ArgumentMatcher<WALEdit> { 1150 volatile FlushAction[] actions; 1151 1152 public IsFlushWALMarker(FlushAction... actions) { 1153 this.actions = actions; 1154 } 1155 1156 @Override 1157 public boolean matches(WALEdit edit) { 1158 List<Cell> cells = edit.getCells(); 1159 if (cells.isEmpty()) { 1160 return false; 1161 } 1162 if (WALEdit.isMetaEditFamily(cells.get(0))) { 1163 FlushDescriptor desc; 1164 try { 1165 desc = WALEdit.getFlushDescriptor(cells.get(0)); 1166 } catch (IOException e) { 1167 LOG.warn(e.toString(), e); 1168 return false; 1169 } 1170 if (desc != null) { 1171 for (FlushAction action : actions) { 1172 if (desc.getAction() == action) { 1173 return true; 1174 } 1175 } 1176 } 1177 } 1178 return false; 1179 } 1180 1181 public IsFlushWALMarker set(FlushAction... actions) { 1182 this.actions = actions; 1183 return this; 1184 } 1185 } 1186 1187 @Test 1188 public void testFlushMarkersWALFail() throws Exception { 1189 // test the cases where the WAL append for flush markers fail. 1190 byte[] family = Bytes.toBytes("family"); 1191 1192 // spy an actual WAL implementation to throw exception (was not able to mock) 1193 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + "log"); 1194 1195 final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration()); 1196 CommonFSUtils.setRootDir(walConf, logDir); 1197 // Make up a WAL that we can manipulate at append time. 1198 class FailAppendFlushMarkerWAL extends FSHLog { 1199 volatile FlushAction[] flushActions = null; 1200 1201 public FailAppendFlushMarkerWAL(FileSystem fs, Path root, String logDir, Configuration conf) 1202 throws IOException { 1203 super(fs, root, logDir, conf); 1204 } 1205 1206 @Override 1207 protected Writer createWriterInstance(FileSystem fs, Path path) throws IOException { 1208 final Writer w = super.createWriterInstance(fs, path); 1209 return new Writer() { 1210 @Override 1211 public void close() throws IOException { 1212 w.close(); 1213 } 1214 1215 @Override 1216 public void sync(boolean forceSync) throws IOException { 1217 w.sync(forceSync); 1218 } 1219 1220 @Override 1221 public void append(Entry entry) throws IOException { 1222 List<Cell> cells = entry.getEdit().getCells(); 1223 if (WALEdit.isMetaEditFamily(cells.get(0))) { 1224 FlushDescriptor desc = WALEdit.getFlushDescriptor(cells.get(0)); 1225 if (desc != null) { 1226 for (FlushAction flushAction : flushActions) { 1227 if (desc.getAction().equals(flushAction)) { 1228 throw new IOException("Failed to append flush marker! " + flushAction); 1229 } 1230 } 1231 } 1232 } 1233 w.append(entry); 1234 } 1235 1236 @Override 1237 public long getLength() { 1238 return w.getLength(); 1239 } 1240 1241 @Override 1242 public long getSyncedLength() { 1243 return w.getSyncedLength(); 1244 } 1245 }; 1246 } 1247 } 1248 FileSystem fs = FileSystem.get(walConf); 1249 Path rootDir = CommonFSUtils.getRootDir(walConf); 1250 fs.mkdirs(new Path(rootDir, method)); 1251 FailAppendFlushMarkerWAL wal = new FailAppendFlushMarkerWAL(fs, rootDir, method, walConf); 1252 wal.init(); 1253 this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF, 1254 false, Durability.USE_DEFAULT, wal, family); 1255 int i = 0; 1256 Put put = new Put(Bytes.toBytes(i)); 1257 put.setDurability(Durability.SKIP_WAL); // have to skip mocked wal 1258 put.addColumn(family, Bytes.toBytes(i), Bytes.toBytes(i)); 1259 region.put(put); 1260 1261 // 1. Test case where START_FLUSH throws exception 1262 wal.flushActions = new FlushAction[] { FlushAction.START_FLUSH }; 1263 1264 // start cache flush will throw exception 1265 try { 1266 region.flush(true); 1267 fail("This should have thrown exception"); 1268 } catch (DroppedSnapshotException unexpected) { 1269 // this should not be a dropped snapshot exception. Meaning that RS will not abort 1270 throw unexpected; 1271 } catch (IOException expected) { 1272 // expected 1273 } 1274 // The WAL is hosed now. It has two edits appended. We cannot roll the log without it 1275 // throwing a DroppedSnapshotException to force an abort. Just clean up the mess. 1276 region.close(true); 1277 wal.close(); 1278 // release the snapshot and active segment, so netty will not report memory leak 1279 for (HStore store : region.getStores()) { 1280 AbstractMemStore memstore = (AbstractMemStore) store.memstore; 1281 memstore.doClearSnapShot(); 1282 memstore.close(); 1283 } 1284 1285 // 2. Test case where START_FLUSH succeeds but COMMIT_FLUSH will throw exception 1286 wal.flushActions = new FlushAction[] { FlushAction.COMMIT_FLUSH }; 1287 wal = new FailAppendFlushMarkerWAL(FileSystem.get(walConf), CommonFSUtils.getRootDir(walConf), 1288 method, walConf); 1289 wal.init(); 1290 this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF, 1291 false, Durability.USE_DEFAULT, wal, family); 1292 region.put(put); 1293 // 3. Test case where ABORT_FLUSH will throw exception. 1294 // Even if ABORT_FLUSH throws exception, we should not fail with IOE, but continue with 1295 // DroppedSnapshotException. Below COMMIT_FLUSH will cause flush to abort 1296 wal.flushActions = new FlushAction[] { FlushAction.COMMIT_FLUSH, FlushAction.ABORT_FLUSH }; 1297 1298 // we expect this exception, since we were able to write the snapshot, but failed to 1299 // write the flush marker to WAL 1300 assertThrows(DroppedSnapshotException.class, () -> region.flush(true)); 1301 1302 region.close(true); 1303 // release the snapshot and active segment, so netty will not report memory leak 1304 for (HStore store : region.getStores()) { 1305 AbstractMemStore memstore = (AbstractMemStore) store.memstore; 1306 memstore.doClearSnapShot(); 1307 memstore.close(); 1308 } 1309 region = null; 1310 } 1311 1312 /* 1313 * An involved filter test. Has multiple column families and deletes in mix. 1314 */ 1315 @Test 1316 public void testWeirdCacheBehaviour() throws Exception { 1317 final TableName tableName = TableName.valueOf(name); 1318 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"), 1319 Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") }; 1320 this.region = initHRegion(tableName, method, CONF, FAMILIES); 1321 String value = "this is the value"; 1322 String value2 = "this is some other value"; 1323 String keyPrefix1 = "prefix1"; 1324 String keyPrefix2 = "prefix2"; 1325 String keyPrefix3 = "prefix3"; 1326 putRows(this.region, 3, value, keyPrefix1); 1327 putRows(this.region, 3, value, keyPrefix2); 1328 putRows(this.region, 3, value, keyPrefix3); 1329 putRows(this.region, 3, value2, keyPrefix1); 1330 putRows(this.region, 3, value2, keyPrefix2); 1331 putRows(this.region, 3, value2, keyPrefix3); 1332 System.out.println("Checking values for key: " + keyPrefix1); 1333 assertEquals(3, getNumberOfRows(keyPrefix1, value2, this.region), 1334 "Got back incorrect number of rows from scan"); 1335 System.out.println("Checking values for key: " + keyPrefix2); 1336 assertEquals(3, getNumberOfRows(keyPrefix2, value2, this.region), 1337 "Got back incorrect number of rows from scan"); 1338 System.out.println("Checking values for key: " + keyPrefix3); 1339 assertEquals(3, getNumberOfRows(keyPrefix3, value2, this.region), 1340 "Got back incorrect number of rows from scan"); 1341 deleteColumns(this.region, value2, keyPrefix1); 1342 deleteColumns(this.region, value2, keyPrefix2); 1343 deleteColumns(this.region, value2, keyPrefix3); 1344 System.out.println("Starting important checks....."); 1345 assertEquals(0, getNumberOfRows(keyPrefix1, value2, this.region), 1346 "Got back incorrect number of rows from scan: " + keyPrefix1); 1347 assertEquals(0, getNumberOfRows(keyPrefix2, value2, this.region), 1348 "Got back incorrect number of rows from scan: " + keyPrefix2); 1349 assertEquals(0, getNumberOfRows(keyPrefix3, value2, this.region), 1350 "Got back incorrect number of rows from scan: " + keyPrefix3); 1351 } 1352 1353 @Test 1354 public void testAppendWithReadOnlyTable() throws Exception { 1355 final TableName tableName = TableName.valueOf(name); 1356 this.region = initHRegion(tableName, method, CONF, true, Bytes.toBytes("somefamily")); 1357 boolean exceptionCaught = false; 1358 Append append = new Append(Bytes.toBytes("somerow")); 1359 append.setDurability(Durability.SKIP_WAL); 1360 append.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1361 Bytes.toBytes("somevalue")); 1362 try { 1363 region.append(append); 1364 } catch (IOException e) { 1365 exceptionCaught = true; 1366 } 1367 assertTrue(exceptionCaught == true); 1368 } 1369 1370 @Test 1371 public void testIncrWithReadOnlyTable() throws Exception { 1372 final TableName tableName = TableName.valueOf(name); 1373 this.region = initHRegion(tableName, method, CONF, true, Bytes.toBytes("somefamily")); 1374 boolean exceptionCaught = false; 1375 Increment inc = new Increment(Bytes.toBytes("somerow")); 1376 inc.setDurability(Durability.SKIP_WAL); 1377 inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L); 1378 try { 1379 region.increment(inc); 1380 } catch (IOException e) { 1381 exceptionCaught = true; 1382 } 1383 assertTrue(exceptionCaught == true); 1384 } 1385 1386 private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException { 1387 int count = 0; 1388 try (InternalScanner scanner = buildScanner(keyPrefix, value, r)) { 1389 boolean more = false; 1390 List<Cell> results = new ArrayList<>(); 1391 do { 1392 more = scanner.next(results); 1393 if (results != null && !results.isEmpty()) { 1394 count++; 1395 } else { 1396 break; 1397 } 1398 Delete delete = new Delete(CellUtil.cloneRow(results.get(0))); 1399 delete.addColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2")); 1400 r.delete(delete); 1401 results.clear(); 1402 } while (more); 1403 } 1404 assertEquals(3, count, "Did not perform correct number of deletes"); 1405 } 1406 1407 private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception { 1408 try (InternalScanner resultScanner = buildScanner(keyPrefix, value, r)) { 1409 int numberOfResults = 0; 1410 List<Cell> results = new ArrayList<>(); 1411 boolean more = false; 1412 do { 1413 more = resultScanner.next(results); 1414 if (results != null && !results.isEmpty()) { 1415 numberOfResults++; 1416 } else { 1417 break; 1418 } 1419 for (Cell kv : results) { 1420 LOG.info("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv))); 1421 } 1422 results.clear(); 1423 } while (more); 1424 return numberOfResults; 1425 } 1426 } 1427 1428 private InternalScanner buildScanner(String keyPrefix, String value, HRegion r) 1429 throws IOException { 1430 // Defaults FilterList.Operator.MUST_PASS_ALL. 1431 FilterList allFilters = new FilterList(); 1432 allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix))); 1433 // Only return rows where this column value exists in the row. 1434 SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"), 1435 Bytes.toBytes("qual2"), CompareOperator.EQUAL, Bytes.toBytes(value)); 1436 filter.setFilterIfMissing(true); 1437 allFilters.addFilter(filter); 1438 Scan scan = new Scan(); 1439 scan.addFamily(Bytes.toBytes("trans-blob")); 1440 scan.addFamily(Bytes.toBytes("trans-type")); 1441 scan.addFamily(Bytes.toBytes("trans-date")); 1442 scan.addFamily(Bytes.toBytes("trans-tags")); 1443 scan.addFamily(Bytes.toBytes("trans-group")); 1444 scan.setFilter(allFilters); 1445 return r.getScanner(scan); 1446 } 1447 1448 private void putRows(HRegion r, int numRows, String value, String key) throws IOException { 1449 for (int i = 0; i < numRows; i++) { 1450 String row = key + "_" + i/* UUID.randomUUID().toString() */; 1451 System.out.println(String.format("Saving row: %s, with value %s", row, value)); 1452 Put put = new Put(Bytes.toBytes(row)); 1453 put.setDurability(Durability.SKIP_WAL); 1454 put.addColumn(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob")); 1455 put.addColumn(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement")); 1456 put.addColumn(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999")); 1457 put.addColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value)); 1458 put.addColumn(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId")); 1459 r.put(put); 1460 } 1461 } 1462 1463 @Test 1464 public void testFamilyWithAndWithoutColon() throws Exception { 1465 byte[] cf = Bytes.toBytes(COLUMN_FAMILY); 1466 this.region = initHRegion(tableName, method, CONF, cf); 1467 Put p = new Put(tableName.toBytes()); 1468 byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":"); 1469 p.addColumn(cfwithcolon, cfwithcolon, cfwithcolon); 1470 boolean exception = false; 1471 try { 1472 this.region.put(p); 1473 } catch (NoSuchColumnFamilyException e) { 1474 exception = true; 1475 } 1476 assertTrue(exception); 1477 } 1478 1479 @Test 1480 public void testBatchPut_whileNoRowLocksHeld() throws IOException { 1481 final Put[] puts = new Put[10]; 1482 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); 1483 long syncs = prepareRegionForBachPut(puts, source, false); 1484 1485 OperationStatus[] codes = this.region.batchMutate(puts); 1486 assertEquals(10, codes.length); 1487 for (int i = 0; i < 10; i++) { 1488 assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode()); 1489 } 1490 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source); 1491 1492 LOG.info("Next a batch put with one invalid family"); 1493 puts[5].addColumn(Bytes.toBytes("BAD_CF"), qual, value); 1494 codes = this.region.batchMutate(puts); 1495 assertEquals(10, codes.length); 1496 for (int i = 0; i < 10; i++) { 1497 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS, 1498 codes[i].getOperationStatusCode()); 1499 } 1500 1501 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source); 1502 } 1503 1504 @Test 1505 public void testBatchPut_whileMultipleRowLocksHeld() throws Exception { 1506 final Put[] puts = new Put[10]; 1507 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); 1508 long syncs = prepareRegionForBachPut(puts, source, false); 1509 1510 puts[5].addColumn(Bytes.toBytes("BAD_CF"), qual, value); 1511 1512 LOG.info("batchPut will have to break into four batches to avoid row locks"); 1513 RowLock rowLock1 = region.getRowLock(Bytes.toBytes("row_2")); 1514 RowLock rowLock2 = region.getRowLock(Bytes.toBytes("row_1")); 1515 RowLock rowLock3 = region.getRowLock(Bytes.toBytes("row_3")); 1516 RowLock rowLock4 = region.getRowLock(Bytes.toBytes("row_3"), true); 1517 1518 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF); 1519 final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<>(); 1520 final CountDownLatch startingPuts = new CountDownLatch(1); 1521 final CountDownLatch startingClose = new CountDownLatch(1); 1522 TestThread putter = new TestThread(ctx) { 1523 @Override 1524 public void doWork() throws IOException { 1525 startingPuts.countDown(); 1526 retFromThread.set(region.batchMutate(puts)); 1527 } 1528 }; 1529 LOG.info("...starting put thread while holding locks"); 1530 ctx.addThread(putter); 1531 ctx.startThreads(); 1532 1533 // Now attempt to close the region from another thread. Prior to HBASE-12565 1534 // this would cause the in-progress batchMutate operation to to fail with 1535 // exception because it use to release and re-acquire the close-guard lock 1536 // between batches. Caller then didn't get status indicating which writes succeeded. 1537 // We now expect this thread to block until the batchMutate call finishes. 1538 Thread regionCloseThread = new TestThread(ctx) { 1539 @Override 1540 public void doWork() { 1541 try { 1542 startingPuts.await(); 1543 // Give some time for the batch mutate to get in. 1544 // We don't want to race with the mutate 1545 Thread.sleep(10); 1546 startingClose.countDown(); 1547 HBaseTestingUtil.closeRegionAndWAL(region); 1548 region = null; 1549 } catch (IOException e) { 1550 throw new RuntimeException(e); 1551 } catch (InterruptedException e) { 1552 throw new RuntimeException(e); 1553 } 1554 } 1555 }; 1556 regionCloseThread.start(); 1557 1558 startingClose.await(); 1559 startingPuts.await(); 1560 Thread.sleep(100); 1561 LOG.info("...releasing row lock 1, which should let put thread continue"); 1562 rowLock1.release(); 1563 rowLock2.release(); 1564 rowLock3.release(); 1565 waitForCounter(source, "syncTimeNumOps", syncs + 1); 1566 1567 LOG.info("...joining on put thread"); 1568 ctx.stop(); 1569 regionCloseThread.join(); 1570 1571 OperationStatus[] codes = retFromThread.get(); 1572 for (int i = 0; i < codes.length; i++) { 1573 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS, 1574 codes[i].getOperationStatusCode()); 1575 } 1576 rowLock4.release(); 1577 } 1578 1579 private void waitForCounter(MetricsWALSource source, String metricName, long expectedCount) 1580 throws InterruptedException { 1581 long startWait = EnvironmentEdgeManager.currentTime(); 1582 long currentCount; 1583 while ((currentCount = metricsAssertHelper.getCounter(metricName, source)) < expectedCount) { 1584 Thread.sleep(100); 1585 if (EnvironmentEdgeManager.currentTime() - startWait > 10000) { 1586 fail(String.format("Timed out waiting for '%s' >= '%s', currentCount=%s", metricName, 1587 expectedCount, currentCount)); 1588 } 1589 } 1590 } 1591 1592 @Test 1593 public void testAtomicBatchPut() throws IOException { 1594 final Put[] puts = new Put[10]; 1595 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); 1596 long syncs = prepareRegionForBachPut(puts, source, false); 1597 1598 // 1. Straight forward case, should succeed 1599 OperationStatus[] codes = this.region.batchMutate(puts, true); 1600 assertEquals(10, codes.length); 1601 for (int i = 0; i < 10; i++) { 1602 assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode()); 1603 } 1604 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source); 1605 1606 // 2. Failed to get lock 1607 RowLock lock = region.getRowLock(Bytes.toBytes("row_" + 3)); 1608 // Method {@link HRegion#getRowLock(byte[])} is reentrant. As 'row_3' is locked in this 1609 // thread, need to run {@link HRegion#batchMutate(HRegion.BatchOperation)} in different thread 1610 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF); 1611 final AtomicReference<IOException> retFromThread = new AtomicReference<>(); 1612 final CountDownLatch finishedPuts = new CountDownLatch(1); 1613 TestThread putter = new TestThread(ctx) { 1614 @Override 1615 public void doWork() throws IOException { 1616 try { 1617 region.batchMutate(puts, true); 1618 } catch (IOException ioe) { 1619 LOG.error("test failed!", ioe); 1620 retFromThread.set(ioe); 1621 } 1622 finishedPuts.countDown(); 1623 } 1624 }; 1625 LOG.info("...starting put thread while holding locks"); 1626 ctx.addThread(putter); 1627 ctx.startThreads(); 1628 LOG.info("...waiting for batch puts while holding locks"); 1629 try { 1630 finishedPuts.await(); 1631 } catch (InterruptedException e) { 1632 LOG.error("Interrupted!", e); 1633 } finally { 1634 if (lock != null) { 1635 lock.release(); 1636 } 1637 } 1638 assertNotNull(retFromThread.get()); 1639 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source); 1640 1641 // 3. Exception thrown in validation 1642 LOG.info("Next a batch put with one invalid family"); 1643 puts[5].addColumn(Bytes.toBytes("BAD_CF"), qual, value); 1644 assertThrows(NoSuchColumnFamilyException.class, () -> this.region.batchMutate(puts, true)); 1645 } 1646 1647 @Test 1648 public void testBatchPutWithTsSlop() throws Exception { 1649 // add data with a timestamp that is too recent for range. Ensure assert 1650 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000); 1651 final Put[] puts = new Put[10]; 1652 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); 1653 1654 long syncs = prepareRegionForBachPut(puts, source, true); 1655 1656 OperationStatus[] codes = this.region.batchMutate(puts); 1657 assertEquals(10, codes.length); 1658 for (int i = 0; i < 10; i++) { 1659 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode()); 1660 } 1661 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source); 1662 } 1663 1664 /** Returns syncs initial syncTimeNumOps */ 1665 private long prepareRegionForBachPut(final Put[] puts, final MetricsWALSource source, 1666 boolean slop) throws IOException { 1667 this.region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES); 1668 1669 LOG.info("First a batch put with all valid puts"); 1670 for (int i = 0; i < puts.length; i++) { 1671 puts[i] = slop 1672 ? new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100) 1673 : new Put(Bytes.toBytes("row_" + i)); 1674 puts[i].addColumn(COLUMN_FAMILY_BYTES, qual, value); 1675 } 1676 1677 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source); 1678 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source); 1679 return syncs; 1680 } 1681 1682 // //////////////////////////////////////////////////////////////////////////// 1683 // checkAndMutate tests 1684 // //////////////////////////////////////////////////////////////////////////// 1685 @Test 1686 @Deprecated 1687 public void testCheckAndMutate_WithEmptyRowValue() throws IOException { 1688 byte[] row1 = Bytes.toBytes("row1"); 1689 byte[] fam1 = Bytes.toBytes("fam1"); 1690 byte[] qf1 = Bytes.toBytes("qualifier"); 1691 byte[] emptyVal = new byte[] {}; 1692 byte[] val1 = Bytes.toBytes("value1"); 1693 byte[] val2 = Bytes.toBytes("value2"); 1694 1695 // Setting up region 1696 this.region = initHRegion(tableName, method, CONF, fam1); 1697 // Putting empty data in key 1698 Put put = new Put(row1); 1699 put.addColumn(fam1, qf1, emptyVal); 1700 1701 // checkAndPut with empty value 1702 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1703 new BinaryComparator(emptyVal), put); 1704 assertTrue(res); 1705 1706 // Putting data in key 1707 put = new Put(row1); 1708 put.addColumn(fam1, qf1, val1); 1709 1710 // checkAndPut with correct value 1711 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1712 new BinaryComparator(emptyVal), put); 1713 assertTrue(res); 1714 1715 // not empty anymore 1716 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1717 new BinaryComparator(emptyVal), put); 1718 assertFalse(res); 1719 1720 Delete delete = new Delete(row1); 1721 delete.addColumn(fam1, qf1); 1722 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1723 new BinaryComparator(emptyVal), delete); 1724 assertFalse(res); 1725 1726 put = new Put(row1); 1727 put.addColumn(fam1, qf1, val2); 1728 // checkAndPut with correct value 1729 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val1), 1730 put); 1731 assertTrue(res); 1732 1733 // checkAndDelete with correct value 1734 delete = new Delete(row1); 1735 delete.addColumn(fam1, qf1); 1736 delete.addColumn(fam1, qf1); 1737 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val2), 1738 delete); 1739 assertTrue(res); 1740 1741 delete = new Delete(row1); 1742 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1743 new BinaryComparator(emptyVal), delete); 1744 assertTrue(res); 1745 1746 // checkAndPut looking for a null value 1747 put = new Put(row1); 1748 put.addColumn(fam1, qf1, val1); 1749 1750 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new NullComparator(), put); 1751 assertTrue(res); 1752 } 1753 1754 @Test 1755 @Deprecated 1756 public void testCheckAndMutate_WithWrongValue() throws IOException { 1757 byte[] row1 = Bytes.toBytes("row1"); 1758 byte[] fam1 = Bytes.toBytes("fam1"); 1759 byte[] qf1 = Bytes.toBytes("qualifier"); 1760 byte[] val1 = Bytes.toBytes("value1"); 1761 byte[] val2 = Bytes.toBytes("value2"); 1762 BigDecimal bd1 = new BigDecimal(Double.MAX_VALUE); 1763 BigDecimal bd2 = new BigDecimal(Double.MIN_VALUE); 1764 1765 // Setting up region 1766 this.region = initHRegion(tableName, method, CONF, fam1); 1767 // Putting data in key 1768 Put put = new Put(row1); 1769 put.addColumn(fam1, qf1, val1); 1770 region.put(put); 1771 1772 // checkAndPut with wrong value 1773 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1774 new BinaryComparator(val2), put); 1775 assertEquals(false, res); 1776 1777 // checkAndDelete with wrong value 1778 Delete delete = new Delete(row1); 1779 delete.addFamily(fam1); 1780 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val2), 1781 put); 1782 assertEquals(false, res); 1783 1784 // Putting data in key 1785 put = new Put(row1); 1786 put.addColumn(fam1, qf1, Bytes.toBytes(bd1)); 1787 region.put(put); 1788 1789 // checkAndPut with wrong value 1790 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1791 new BigDecimalComparator(bd2), put); 1792 assertEquals(false, res); 1793 1794 // checkAndDelete with wrong value 1795 delete = new Delete(row1); 1796 delete.addFamily(fam1); 1797 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1798 new BigDecimalComparator(bd2), put); 1799 assertEquals(false, res); 1800 } 1801 1802 @Test 1803 @Deprecated 1804 public void testCheckAndMutate_WithCorrectValue() throws IOException { 1805 byte[] row1 = Bytes.toBytes("row1"); 1806 byte[] fam1 = Bytes.toBytes("fam1"); 1807 byte[] qf1 = Bytes.toBytes("qualifier"); 1808 byte[] val1 = Bytes.toBytes("value1"); 1809 BigDecimal bd1 = new BigDecimal(Double.MIN_VALUE); 1810 1811 // Setting up region 1812 this.region = initHRegion(tableName, method, CONF, fam1); 1813 // Putting data in key 1814 long now = EnvironmentEdgeManager.currentTime(); 1815 Put put = new Put(row1); 1816 put.addColumn(fam1, qf1, now, val1); 1817 region.put(put); 1818 1819 // checkAndPut with correct value 1820 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1821 new BinaryComparator(val1), put); 1822 assertEquals(true, res, "First"); 1823 1824 // checkAndDelete with correct value 1825 Delete delete = new Delete(row1, now + 1); 1826 delete.addColumn(fam1, qf1); 1827 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val1), 1828 delete); 1829 assertEquals(true, res, "Delete"); 1830 1831 // Putting data in key 1832 put = new Put(row1); 1833 put.addColumn(fam1, qf1, now + 2, Bytes.toBytes(bd1)); 1834 region.put(put); 1835 1836 // checkAndPut with correct value 1837 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1838 new BigDecimalComparator(bd1), put); 1839 assertEquals(true, res, "Second put"); 1840 1841 // checkAndDelete with correct value 1842 delete = new Delete(row1, now + 3); 1843 delete.addColumn(fam1, qf1); 1844 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1845 new BigDecimalComparator(bd1), delete); 1846 assertEquals(true, res, "Second delete"); 1847 } 1848 1849 @Test 1850 @Deprecated 1851 public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException { 1852 byte[] row1 = Bytes.toBytes("row1"); 1853 byte[] fam1 = Bytes.toBytes("fam1"); 1854 byte[] qf1 = Bytes.toBytes("qualifier"); 1855 byte[] val1 = Bytes.toBytes("value1"); 1856 byte[] val2 = Bytes.toBytes("value2"); 1857 byte[] val3 = Bytes.toBytes("value3"); 1858 byte[] val4 = Bytes.toBytes("value4"); 1859 1860 // Setting up region 1861 this.region = initHRegion(tableName, method, CONF, fam1); 1862 // Putting val3 in key 1863 Put put = new Put(row1); 1864 put.addColumn(fam1, qf1, val3); 1865 region.put(put); 1866 1867 // Test CompareOp.LESS: original = val3, compare with val3, fail 1868 boolean res = 1869 region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS, new BinaryComparator(val3), put); 1870 assertEquals(false, res); 1871 1872 // Test CompareOp.LESS: original = val3, compare with val4, fail 1873 res = 1874 region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS, new BinaryComparator(val4), put); 1875 assertEquals(false, res); 1876 1877 // Test CompareOp.LESS: original = val3, compare with val2, 1878 // succeed (now value = val2) 1879 put = new Put(row1); 1880 put.addColumn(fam1, qf1, val2); 1881 res = 1882 region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS, new BinaryComparator(val2), put); 1883 assertEquals(true, res); 1884 1885 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val3, fail 1886 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS_OR_EQUAL, 1887 new BinaryComparator(val3), put); 1888 assertEquals(false, res); 1889 1890 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val2, 1891 // succeed (value still = val2) 1892 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS_OR_EQUAL, 1893 new BinaryComparator(val2), put); 1894 assertEquals(true, res); 1895 1896 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val1, 1897 // succeed (now value = val3) 1898 put = new Put(row1); 1899 put.addColumn(fam1, qf1, val3); 1900 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS_OR_EQUAL, 1901 new BinaryComparator(val1), put); 1902 assertEquals(true, res); 1903 1904 // Test CompareOp.GREATER: original = val3, compare with val3, fail 1905 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER, 1906 new BinaryComparator(val3), put); 1907 assertEquals(false, res); 1908 1909 // Test CompareOp.GREATER: original = val3, compare with val2, fail 1910 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER, 1911 new BinaryComparator(val2), put); 1912 assertEquals(false, res); 1913 1914 // Test CompareOp.GREATER: original = val3, compare with val4, 1915 // succeed (now value = val2) 1916 put = new Put(row1); 1917 put.addColumn(fam1, qf1, val2); 1918 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER, 1919 new BinaryComparator(val4), put); 1920 assertEquals(true, res); 1921 1922 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val1, fail 1923 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER_OR_EQUAL, 1924 new BinaryComparator(val1), put); 1925 assertEquals(false, res); 1926 1927 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val2, 1928 // succeed (value still = val2) 1929 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER_OR_EQUAL, 1930 new BinaryComparator(val2), put); 1931 assertEquals(true, res); 1932 1933 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val3, succeed 1934 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER_OR_EQUAL, 1935 new BinaryComparator(val3), put); 1936 assertEquals(true, res); 1937 } 1938 1939 @Test 1940 @Deprecated 1941 public void testCheckAndPut_ThatPutWasWritten() throws IOException { 1942 byte[] row1 = Bytes.toBytes("row1"); 1943 byte[] fam1 = Bytes.toBytes("fam1"); 1944 byte[] fam2 = Bytes.toBytes("fam2"); 1945 byte[] qf1 = Bytes.toBytes("qualifier"); 1946 byte[] val1 = Bytes.toBytes("value1"); 1947 byte[] val2 = Bytes.toBytes("value2"); 1948 1949 byte[][] families = { fam1, fam2 }; 1950 1951 // Setting up region 1952 this.region = initHRegion(tableName, method, CONF, families); 1953 // Putting data in the key to check 1954 Put put = new Put(row1); 1955 put.addColumn(fam1, qf1, val1); 1956 region.put(put); 1957 1958 // Creating put to add 1959 long ts = EnvironmentEdgeManager.currentTime(); 1960 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2); 1961 put = new Put(row1); 1962 put.add(kv); 1963 1964 // checkAndPut with wrong value 1965 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1966 new BinaryComparator(val1), put); 1967 assertEquals(true, res); 1968 1969 Get get = new Get(row1); 1970 get.addColumn(fam2, qf1); 1971 Cell[] actual = region.get(get).rawCells(); 1972 1973 Cell[] expected = { kv }; 1974 1975 assertEquals(expected.length, actual.length); 1976 for (int i = 0; i < actual.length; i++) { 1977 assertEquals(expected[i], actual[i]); 1978 } 1979 } 1980 1981 @Test 1982 @Deprecated 1983 public void testCheckAndPut_wrongRowInPut() throws IOException { 1984 this.region = initHRegion(tableName, method, CONF, COLUMNS); 1985 Put put = new Put(row2); 1986 put.addColumn(fam1, qual1, value1); 1987 try { 1988 region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(value2), 1989 put); 1990 fail(); 1991 } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) { 1992 // expected exception. 1993 } 1994 } 1995 1996 @Test 1997 @Deprecated 1998 public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException { 1999 byte[] row1 = Bytes.toBytes("row1"); 2000 byte[] fam1 = Bytes.toBytes("fam1"); 2001 byte[] fam2 = Bytes.toBytes("fam2"); 2002 byte[] qf1 = Bytes.toBytes("qualifier1"); 2003 byte[] qf2 = Bytes.toBytes("qualifier2"); 2004 byte[] qf3 = Bytes.toBytes("qualifier3"); 2005 byte[] val1 = Bytes.toBytes("value1"); 2006 byte[] val2 = Bytes.toBytes("value2"); 2007 byte[] val3 = Bytes.toBytes("value3"); 2008 byte[] emptyVal = new byte[] {}; 2009 2010 byte[][] families = { fam1, fam2 }; 2011 2012 // Setting up region 2013 this.region = initHRegion(tableName, method, CONF, families); 2014 // Put content 2015 Put put = new Put(row1); 2016 put.addColumn(fam1, qf1, val1); 2017 region.put(put); 2018 Threads.sleep(2); 2019 2020 put = new Put(row1); 2021 put.addColumn(fam1, qf1, val2); 2022 put.addColumn(fam2, qf1, val3); 2023 put.addColumn(fam2, qf2, val2); 2024 put.addColumn(fam2, qf3, val1); 2025 put.addColumn(fam1, qf3, val1); 2026 region.put(put); 2027 2028 LOG.info("get={}", region.get(new Get(row1).addColumn(fam1, qf1)).toString()); 2029 2030 // Multi-column delete 2031 Delete delete = new Delete(row1); 2032 delete.addColumn(fam1, qf1); 2033 delete.addColumn(fam2, qf1); 2034 delete.addColumn(fam1, qf3); 2035 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 2036 new BinaryComparator(val2), delete); 2037 assertEquals(true, res); 2038 2039 Get get = new Get(row1); 2040 get.addColumn(fam1, qf1); 2041 get.addColumn(fam1, qf3); 2042 get.addColumn(fam2, qf2); 2043 Result r = region.get(get); 2044 assertEquals(2, r.size()); 2045 assertArrayEquals(val1, r.getValue(fam1, qf1)); 2046 assertArrayEquals(val2, r.getValue(fam2, qf2)); 2047 2048 // Family delete 2049 delete = new Delete(row1); 2050 delete.addFamily(fam2); 2051 res = region.checkAndMutate(row1, fam2, qf1, CompareOperator.EQUAL, 2052 new BinaryComparator(emptyVal), delete); 2053 assertEquals(true, res); 2054 2055 get = new Get(row1); 2056 r = region.get(get); 2057 assertEquals(1, r.size()); 2058 assertArrayEquals(val1, r.getValue(fam1, qf1)); 2059 2060 // Row delete 2061 delete = new Delete(row1); 2062 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val1), 2063 delete); 2064 assertEquals(true, res); 2065 get = new Get(row1); 2066 r = region.get(get); 2067 assertEquals(0, r.size()); 2068 } 2069 2070 @Test 2071 @Deprecated 2072 public void testCheckAndMutate_WithFilters() throws Throwable { 2073 final byte[] FAMILY = Bytes.toBytes("fam"); 2074 2075 // Setting up region 2076 this.region = initHRegion(tableName, method, CONF, FAMILY); 2077 2078 // Put one row 2079 Put put = new Put(row); 2080 put.addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")); 2081 put.addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")); 2082 put.addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")); 2083 region.put(put); 2084 2085 // Put with success 2086 boolean ok = region.checkAndMutate(row, 2087 new FilterList( 2088 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2089 Bytes.toBytes("a")), 2090 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2091 Bytes.toBytes("b"))), 2092 new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d"))); 2093 assertTrue(ok); 2094 2095 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))); 2096 assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D")))); 2097 2098 // Put with failure 2099 ok = region.checkAndMutate(row, 2100 new FilterList( 2101 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2102 Bytes.toBytes("a")), 2103 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2104 Bytes.toBytes("c"))), 2105 new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e"))); 2106 assertFalse(ok); 2107 2108 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E"))).isEmpty()); 2109 2110 // Delete with success 2111 ok = region.checkAndMutate(row, 2112 new FilterList( 2113 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2114 Bytes.toBytes("a")), 2115 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2116 Bytes.toBytes("b"))), 2117 new Delete(row).addColumns(FAMILY, Bytes.toBytes("D"))); 2118 assertTrue(ok); 2119 2120 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))).isEmpty()); 2121 2122 // Mutate with success 2123 ok = region.checkAndRowMutate(row, 2124 new FilterList( 2125 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2126 Bytes.toBytes("a")), 2127 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2128 Bytes.toBytes("b"))), 2129 new RowMutations(row) 2130 .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e"))) 2131 .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A")))); 2132 assertTrue(ok); 2133 2134 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E"))); 2135 assertEquals("e", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("E")))); 2136 2137 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty()); 2138 } 2139 2140 @Test 2141 @Deprecated 2142 public void testCheckAndMutate_WithFiltersAndTimeRange() throws Throwable { 2143 final byte[] FAMILY = Bytes.toBytes("fam"); 2144 2145 // Setting up region 2146 this.region = initHRegion(tableName, method, CONF, FAMILY); 2147 2148 // Put with specifying the timestamp 2149 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), 100, Bytes.toBytes("a"))); 2150 2151 // Put with success 2152 boolean ok = region.checkAndMutate(row, 2153 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2154 Bytes.toBytes("a")), 2155 TimeRange.between(0, 101), 2156 new Put(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b"))); 2157 assertTrue(ok); 2158 2159 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2160 assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2161 2162 // Put with failure 2163 ok = region.checkAndMutate(row, 2164 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2165 Bytes.toBytes("a")), 2166 TimeRange.between(0, 100), 2167 new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c"))); 2168 assertFalse(ok); 2169 2170 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("C"))).isEmpty()); 2171 2172 // Mutate with success 2173 ok = region.checkAndRowMutate(row, 2174 new SingleColumnValueFilter( 2175 FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, Bytes.toBytes("a")), 2176 TimeRange.between(0, 101), 2177 new RowMutations(row) 2178 .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d"))) 2179 .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A")))); 2180 assertTrue(ok); 2181 2182 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))); 2183 assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D")))); 2184 2185 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty()); 2186 } 2187 2188 @Test 2189 @Deprecated 2190 public void testCheckAndMutate_wrongMutationType() throws Throwable { 2191 // Setting up region 2192 this.region = initHRegion(tableName, method, CONF, fam1); 2193 2194 try { 2195 region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(value1), 2196 new Increment(row).addColumn(fam1, qual1, 1)); 2197 fail("should throw DoNotRetryIOException"); 2198 } catch (DoNotRetryIOException e) { 2199 assertEquals("Unsupported mutate type: INCREMENT", e.getMessage()); 2200 } 2201 2202 try { 2203 region.checkAndMutate(row, 2204 new SingleColumnValueFilter(fam1, qual1, CompareOperator.EQUAL, value1), 2205 new Increment(row).addColumn(fam1, qual1, 1)); 2206 fail("should throw DoNotRetryIOException"); 2207 } catch (DoNotRetryIOException e) { 2208 assertEquals("Unsupported mutate type: INCREMENT", e.getMessage()); 2209 } 2210 } 2211 2212 @Test 2213 @Deprecated 2214 public void testCheckAndMutate_wrongRow() throws Throwable { 2215 final byte[] wrongRow = Bytes.toBytes("wrongRow"); 2216 2217 // Setting up region 2218 this.region = initHRegion(tableName, method, CONF, fam1); 2219 2220 try { 2221 region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(value1), 2222 new Put(wrongRow).addColumn(fam1, qual1, value1)); 2223 fail("should throw DoNotRetryIOException"); 2224 } catch (DoNotRetryIOException e) { 2225 assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>", 2226 e.getMessage()); 2227 } 2228 2229 try { 2230 region.checkAndMutate(row, 2231 new SingleColumnValueFilter(fam1, qual1, CompareOperator.EQUAL, value1), 2232 new Put(wrongRow).addColumn(fam1, qual1, value1)); 2233 fail("should throw DoNotRetryIOException"); 2234 } catch (DoNotRetryIOException e) { 2235 assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>", 2236 e.getMessage()); 2237 } 2238 2239 try { 2240 region.checkAndRowMutate(row, fam1, qual1, CompareOperator.EQUAL, 2241 new BinaryComparator(value1), 2242 new RowMutations(wrongRow).add((Mutation) new Put(wrongRow).addColumn(fam1, qual1, value1)) 2243 .add((Mutation) new Delete(wrongRow).addColumns(fam1, qual2))); 2244 fail("should throw DoNotRetryIOException"); 2245 } catch (DoNotRetryIOException e) { 2246 assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>", 2247 e.getMessage()); 2248 } 2249 2250 try { 2251 region.checkAndRowMutate(row, 2252 new SingleColumnValueFilter(fam1, qual1, CompareOperator.EQUAL, value1), 2253 new RowMutations(wrongRow).add((Mutation) new Put(wrongRow).addColumn(fam1, qual1, value1)) 2254 .add((Mutation) new Delete(wrongRow).addColumns(fam1, qual2))); 2255 fail("should throw DoNotRetryIOException"); 2256 } catch (DoNotRetryIOException e) { 2257 assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>", 2258 e.getMessage()); 2259 } 2260 } 2261 2262 @Test 2263 public void testCheckAndMutateWithEmptyRowValue() throws IOException { 2264 byte[] row1 = Bytes.toBytes("row1"); 2265 byte[] fam1 = Bytes.toBytes("fam1"); 2266 byte[] qf1 = Bytes.toBytes("qualifier"); 2267 byte[] emptyVal = new byte[] {}; 2268 byte[] val1 = Bytes.toBytes("value1"); 2269 byte[] val2 = Bytes.toBytes("value2"); 2270 2271 // Setting up region 2272 this.region = initHRegion(tableName, method, CONF, fam1); 2273 // Putting empty data in key 2274 Put put = new Put(row1); 2275 put.addColumn(fam1, qf1, emptyVal); 2276 2277 // checkAndPut with empty value 2278 CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2279 .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(put)); 2280 assertTrue(res.isSuccess()); 2281 assertNull(res.getResult()); 2282 2283 // Putting data in key 2284 put = new Put(row1); 2285 put.addColumn(fam1, qf1, val1); 2286 2287 // checkAndPut with correct value 2288 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2289 .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(put)); 2290 assertTrue(res.isSuccess()); 2291 assertNull(res.getResult()); 2292 2293 // not empty anymore 2294 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2295 .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(put)); 2296 assertFalse(res.isSuccess()); 2297 assertNull(res.getResult()); 2298 2299 Delete delete = new Delete(row1); 2300 delete.addColumn(fam1, qf1); 2301 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2302 .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(delete)); 2303 assertFalse(res.isSuccess()); 2304 assertNull(res.getResult()); 2305 2306 put = new Put(row1); 2307 put.addColumn(fam1, qf1, val2); 2308 // checkAndPut with correct value 2309 res = region.checkAndMutate( 2310 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(put)); 2311 assertTrue(res.isSuccess()); 2312 assertNull(res.getResult()); 2313 2314 // checkAndDelete with correct value 2315 delete = new Delete(row1); 2316 delete.addColumn(fam1, qf1); 2317 delete.addColumn(fam1, qf1); 2318 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2319 .ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(delete)); 2320 assertTrue(res.isSuccess()); 2321 assertNull(res.getResult()); 2322 2323 delete = new Delete(row1); 2324 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2325 .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(delete)); 2326 assertTrue(res.isSuccess()); 2327 assertNull(res.getResult()); 2328 2329 // checkAndPut looking for a null value 2330 put = new Put(row1); 2331 put.addColumn(fam1, qf1, val1); 2332 2333 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1).ifNotExists(fam1, qf1).build(put)); 2334 assertTrue(res.isSuccess()); 2335 assertNull(res.getResult()); 2336 } 2337 2338 @Test 2339 public void testCheckAndMutateWithWrongValue() throws IOException { 2340 byte[] row1 = Bytes.toBytes("row1"); 2341 byte[] fam1 = Bytes.toBytes("fam1"); 2342 byte[] qf1 = Bytes.toBytes("qualifier"); 2343 byte[] val1 = Bytes.toBytes("value1"); 2344 byte[] val2 = Bytes.toBytes("value2"); 2345 BigDecimal bd1 = new BigDecimal(Double.MAX_VALUE); 2346 BigDecimal bd2 = new BigDecimal(Double.MIN_VALUE); 2347 2348 // Setting up region 2349 this.region = initHRegion(tableName, method, CONF, fam1); 2350 // Putting data in key 2351 Put put = new Put(row1); 2352 put.addColumn(fam1, qf1, val1); 2353 region.put(put); 2354 2355 // checkAndPut with wrong value 2356 CheckAndMutateResult res = region.checkAndMutate( 2357 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(put)); 2358 assertFalse(res.isSuccess()); 2359 assertNull(res.getResult()); 2360 2361 // checkAndDelete with wrong value 2362 Delete delete = new Delete(row1); 2363 delete.addFamily(fam1); 2364 res = region.checkAndMutate( 2365 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(put)); 2366 assertFalse(res.isSuccess()); 2367 assertNull(res.getResult()); 2368 2369 // Putting data in key 2370 put = new Put(row1); 2371 put.addColumn(fam1, qf1, Bytes.toBytes(bd1)); 2372 region.put(put); 2373 2374 // checkAndPut with wrong value 2375 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2376 .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd2)).build(put)); 2377 assertFalse(res.isSuccess()); 2378 assertNull(res.getResult()); 2379 2380 // checkAndDelete with wrong value 2381 delete = new Delete(row1); 2382 delete.addFamily(fam1); 2383 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2384 .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd2)).build(delete)); 2385 assertFalse(res.isSuccess()); 2386 assertNull(res.getResult()); 2387 } 2388 2389 @Test 2390 public void testCheckAndMutateWithCorrectValue() throws IOException { 2391 byte[] row1 = Bytes.toBytes("row1"); 2392 byte[] fam1 = Bytes.toBytes("fam1"); 2393 byte[] qf1 = Bytes.toBytes("qualifier"); 2394 byte[] val1 = Bytes.toBytes("value1"); 2395 BigDecimal bd1 = new BigDecimal(Double.MIN_VALUE); 2396 2397 // Setting up region 2398 this.region = initHRegion(tableName, method, CONF, fam1); 2399 // Putting data in key 2400 long now = EnvironmentEdgeManager.currentTime(); 2401 Put put = new Put(row1); 2402 put.addColumn(fam1, qf1, now, val1); 2403 region.put(put); 2404 2405 // checkAndPut with correct value 2406 CheckAndMutateResult res = region.checkAndMutate( 2407 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(put)); 2408 assertTrue(res.isSuccess(), "First"); 2409 2410 // checkAndDelete with correct value 2411 Delete delete = new Delete(row1, now + 1); 2412 delete.addColumn(fam1, qf1); 2413 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2414 .ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(delete)); 2415 assertTrue(res.isSuccess(), "Delete"); 2416 assertNull(res.getResult()); 2417 2418 // Putting data in key 2419 put = new Put(row1); 2420 put.addColumn(fam1, qf1, now + 2, Bytes.toBytes(bd1)); 2421 region.put(put); 2422 2423 // checkAndPut with correct value 2424 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2425 .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd1)).build(put)); 2426 assertTrue(res.isSuccess(), "Second put"); 2427 assertNull(res.getResult()); 2428 2429 // checkAndDelete with correct value 2430 delete = new Delete(row1, now + 3); 2431 delete.addColumn(fam1, qf1); 2432 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2433 .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd1)).build(delete)); 2434 assertTrue(res.isSuccess(), "Second delete"); 2435 assertNull(res.getResult()); 2436 } 2437 2438 @Test 2439 public void testCheckAndMutateWithNonEqualCompareOp() throws IOException { 2440 byte[] row1 = Bytes.toBytes("row1"); 2441 byte[] fam1 = Bytes.toBytes("fam1"); 2442 byte[] qf1 = Bytes.toBytes("qualifier"); 2443 byte[] val1 = Bytes.toBytes("value1"); 2444 byte[] val2 = Bytes.toBytes("value2"); 2445 byte[] val3 = Bytes.toBytes("value3"); 2446 byte[] val4 = Bytes.toBytes("value4"); 2447 2448 // Setting up region 2449 this.region = initHRegion(tableName, method, CONF, fam1); 2450 // Putting val3 in key 2451 Put put = new Put(row1); 2452 put.addColumn(fam1, qf1, val3); 2453 region.put(put); 2454 2455 // Test CompareOp.LESS: original = val3, compare with val3, fail 2456 CheckAndMutateResult res = region.checkAndMutate( 2457 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.LESS, val3).build(put)); 2458 assertFalse(res.isSuccess()); 2459 assertNull(res.getResult()); 2460 2461 // Test CompareOp.LESS: original = val3, compare with val4, fail 2462 res = region.checkAndMutate( 2463 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.LESS, val4).build(put)); 2464 assertFalse(res.isSuccess()); 2465 assertNull(res.getResult()); 2466 2467 // Test CompareOp.LESS: original = val3, compare with val2, 2468 // succeed (now value = val2) 2469 put = new Put(row1); 2470 put.addColumn(fam1, qf1, val2); 2471 res = region.checkAndMutate( 2472 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.LESS, val2).build(put)); 2473 assertTrue(res.isSuccess()); 2474 assertNull(res.getResult()); 2475 2476 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val3, fail 2477 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2478 .ifMatches(fam1, qf1, CompareOperator.LESS_OR_EQUAL, val3).build(put)); 2479 assertFalse(res.isSuccess()); 2480 assertNull(res.getResult()); 2481 2482 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val2, 2483 // succeed (value still = val2) 2484 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2485 .ifMatches(fam1, qf1, CompareOperator.LESS_OR_EQUAL, val2).build(put)); 2486 assertTrue(res.isSuccess()); 2487 assertNull(res.getResult()); 2488 2489 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val1, 2490 // succeed (now value = val3) 2491 put = new Put(row1); 2492 put.addColumn(fam1, qf1, val3); 2493 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2494 .ifMatches(fam1, qf1, CompareOperator.LESS_OR_EQUAL, val1).build(put)); 2495 assertTrue(res.isSuccess()); 2496 assertNull(res.getResult()); 2497 2498 // Test CompareOp.GREATER: original = val3, compare with val3, fail 2499 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2500 .ifMatches(fam1, qf1, CompareOperator.GREATER, val3).build(put)); 2501 assertFalse(res.isSuccess()); 2502 assertNull(res.getResult()); 2503 2504 // Test CompareOp.GREATER: original = val3, compare with val2, fail 2505 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2506 .ifMatches(fam1, qf1, CompareOperator.GREATER, val2).build(put)); 2507 assertFalse(res.isSuccess()); 2508 assertNull(res.getResult()); 2509 2510 // Test CompareOp.GREATER: original = val3, compare with val4, 2511 // succeed (now value = val2) 2512 put = new Put(row1); 2513 put.addColumn(fam1, qf1, val2); 2514 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2515 .ifMatches(fam1, qf1, CompareOperator.GREATER, val4).build(put)); 2516 assertTrue(res.isSuccess()); 2517 assertNull(res.getResult()); 2518 2519 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val1, fail 2520 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2521 .ifMatches(fam1, qf1, CompareOperator.GREATER_OR_EQUAL, val1).build(put)); 2522 assertFalse(res.isSuccess()); 2523 assertNull(res.getResult()); 2524 2525 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val2, 2526 // succeed (value still = val2) 2527 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2528 .ifMatches(fam1, qf1, CompareOperator.GREATER_OR_EQUAL, val2).build(put)); 2529 assertTrue(res.isSuccess()); 2530 assertNull(res.getResult()); 2531 2532 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val3, succeed 2533 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2534 .ifMatches(fam1, qf1, CompareOperator.GREATER_OR_EQUAL, val3).build(put)); 2535 assertTrue(res.isSuccess()); 2536 assertNull(res.getResult()); 2537 } 2538 2539 @Test 2540 public void testCheckAndPutThatPutWasWritten() throws IOException { 2541 byte[] row1 = Bytes.toBytes("row1"); 2542 byte[] fam1 = Bytes.toBytes("fam1"); 2543 byte[] fam2 = Bytes.toBytes("fam2"); 2544 byte[] qf1 = Bytes.toBytes("qualifier"); 2545 byte[] val1 = Bytes.toBytes("value1"); 2546 byte[] val2 = Bytes.toBytes("value2"); 2547 2548 byte[][] families = { fam1, fam2 }; 2549 2550 // Setting up region 2551 this.region = initHRegion(tableName, method, CONF, families); 2552 // Putting data in the key to check 2553 Put put = new Put(row1); 2554 put.addColumn(fam1, qf1, val1); 2555 region.put(put); 2556 2557 // Creating put to add 2558 long ts = EnvironmentEdgeManager.currentTime(); 2559 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2); 2560 put = new Put(row1); 2561 put.add(kv); 2562 2563 // checkAndPut with wrong value 2564 CheckAndMutateResult res = region.checkAndMutate( 2565 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(put)); 2566 assertTrue(res.isSuccess()); 2567 assertNull(res.getResult()); 2568 2569 Get get = new Get(row1); 2570 get.addColumn(fam2, qf1); 2571 Cell[] actual = region.get(get).rawCells(); 2572 2573 Cell[] expected = { kv }; 2574 2575 assertEquals(expected.length, actual.length); 2576 for (int i = 0; i < actual.length; i++) { 2577 assertEquals(expected[i], actual[i]); 2578 } 2579 } 2580 2581 @Test 2582 public void testCheckAndDeleteThatDeleteWasWritten() throws IOException { 2583 byte[] row1 = Bytes.toBytes("row1"); 2584 byte[] fam1 = Bytes.toBytes("fam1"); 2585 byte[] fam2 = Bytes.toBytes("fam2"); 2586 byte[] qf1 = Bytes.toBytes("qualifier1"); 2587 byte[] qf2 = Bytes.toBytes("qualifier2"); 2588 byte[] qf3 = Bytes.toBytes("qualifier3"); 2589 byte[] val1 = Bytes.toBytes("value1"); 2590 byte[] val2 = Bytes.toBytes("value2"); 2591 byte[] val3 = Bytes.toBytes("value3"); 2592 byte[] emptyVal = new byte[] {}; 2593 2594 byte[][] families = { fam1, fam2 }; 2595 2596 // Setting up region 2597 this.region = initHRegion(tableName, method, CONF, families); 2598 // Put content 2599 Put put = new Put(row1); 2600 put.addColumn(fam1, qf1, val1); 2601 region.put(put); 2602 Threads.sleep(2); 2603 2604 put = new Put(row1); 2605 put.addColumn(fam1, qf1, val2); 2606 put.addColumn(fam2, qf1, val3); 2607 put.addColumn(fam2, qf2, val2); 2608 put.addColumn(fam2, qf3, val1); 2609 put.addColumn(fam1, qf3, val1); 2610 region.put(put); 2611 2612 LOG.info("get={}", region.get(new Get(row1).addColumn(fam1, qf1)).toString()); 2613 2614 // Multi-column delete 2615 Delete delete = new Delete(row1); 2616 delete.addColumn(fam1, qf1); 2617 delete.addColumn(fam2, qf1); 2618 delete.addColumn(fam1, qf3); 2619 CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2620 .ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(delete)); 2621 assertTrue(res.isSuccess()); 2622 assertNull(res.getResult()); 2623 2624 Get get = new Get(row1); 2625 get.addColumn(fam1, qf1); 2626 get.addColumn(fam1, qf3); 2627 get.addColumn(fam2, qf2); 2628 Result r = region.get(get); 2629 assertEquals(2, r.size()); 2630 assertArrayEquals(val1, r.getValue(fam1, qf1)); 2631 assertArrayEquals(val2, r.getValue(fam2, qf2)); 2632 2633 // Family delete 2634 delete = new Delete(row1); 2635 delete.addFamily(fam2); 2636 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2637 .ifMatches(fam2, qf1, CompareOperator.EQUAL, emptyVal).build(delete)); 2638 assertTrue(res.isSuccess()); 2639 assertNull(res.getResult()); 2640 2641 get = new Get(row1); 2642 r = region.get(get); 2643 assertEquals(1, r.size()); 2644 assertArrayEquals(val1, r.getValue(fam1, qf1)); 2645 2646 // Row delete 2647 delete = new Delete(row1); 2648 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2649 .ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(delete)); 2650 assertTrue(res.isSuccess()); 2651 assertNull(res.getResult()); 2652 2653 get = new Get(row1); 2654 r = region.get(get); 2655 assertEquals(0, r.size()); 2656 } 2657 2658 @Test 2659 public void testCheckAndMutateWithFilters() throws Throwable { 2660 final byte[] FAMILY = Bytes.toBytes("fam"); 2661 2662 // Setting up region 2663 this.region = initHRegion(tableName, method, CONF, FAMILY); 2664 2665 // Put one row 2666 Put put = new Put(row); 2667 put.addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")); 2668 put.addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")); 2669 put.addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")); 2670 region.put(put); 2671 2672 // Put with success 2673 CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2674 .ifMatches(new FilterList( 2675 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2676 Bytes.toBytes("a")), 2677 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2678 Bytes.toBytes("b")))) 2679 .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d")))); 2680 assertTrue(res.isSuccess()); 2681 assertNull(res.getResult()); 2682 2683 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))); 2684 assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D")))); 2685 2686 // Put with failure 2687 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2688 .ifMatches(new FilterList( 2689 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2690 Bytes.toBytes("a")), 2691 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2692 Bytes.toBytes("c")))) 2693 .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e")))); 2694 assertFalse(res.isSuccess()); 2695 assertNull(res.getResult()); 2696 2697 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E"))).isEmpty()); 2698 2699 // Delete with success 2700 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2701 .ifMatches(new FilterList( 2702 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2703 Bytes.toBytes("a")), 2704 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2705 Bytes.toBytes("b")))) 2706 .build(new Delete(row).addColumns(FAMILY, Bytes.toBytes("D")))); 2707 assertTrue(res.isSuccess()); 2708 assertNull(res.getResult()); 2709 2710 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))).isEmpty()); 2711 2712 // Mutate with success 2713 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2714 .ifMatches(new FilterList( 2715 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2716 Bytes.toBytes("a")), 2717 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2718 Bytes.toBytes("b")))) 2719 .build(new RowMutations(row) 2720 .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e"))) 2721 .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A"))))); 2722 assertTrue(res.isSuccess()); 2723 assertNull(res.getResult()); 2724 2725 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E"))); 2726 assertEquals("e", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("E")))); 2727 2728 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty()); 2729 } 2730 2731 @Test 2732 public void testCheckAndMutateWithFiltersAndTimeRange() throws Throwable { 2733 final byte[] FAMILY = Bytes.toBytes("fam"); 2734 2735 // Setting up region 2736 this.region = initHRegion(tableName, method, CONF, FAMILY); 2737 2738 // Put with specifying the timestamp 2739 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), 100, Bytes.toBytes("a"))); 2740 2741 // Put with success 2742 CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2743 .ifMatches(new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2744 Bytes.toBytes("a"))) 2745 .timeRange(TimeRange.between(0, 101)) 2746 .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")))); 2747 assertTrue(res.isSuccess()); 2748 assertNull(res.getResult()); 2749 2750 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2751 assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2752 2753 // Put with failure 2754 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2755 .ifMatches(new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2756 Bytes.toBytes("a"))) 2757 .timeRange(TimeRange.between(0, 100)) 2758 .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")))); 2759 assertFalse(res.isSuccess()); 2760 assertNull(res.getResult()); 2761 2762 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("C"))).isEmpty()); 2763 2764 // RowMutations with success 2765 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2766 .ifMatches(new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2767 Bytes.toBytes("a"))) 2768 .timeRange(TimeRange.between(0, 101)) 2769 .build(new RowMutations(row) 2770 .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d"))) 2771 .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A"))))); 2772 assertTrue(res.isSuccess()); 2773 assertNull(res.getResult()); 2774 2775 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))); 2776 assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D")))); 2777 2778 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty()); 2779 } 2780 2781 @Test 2782 public void testCheckAndIncrement() throws Throwable { 2783 final byte[] FAMILY = Bytes.toBytes("fam"); 2784 2785 // Setting up region 2786 this.region = initHRegion(tableName, method, CONF, FAMILY); 2787 2788 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a"))); 2789 2790 // CheckAndIncrement with correct value 2791 CheckAndMutateResult res = region.checkAndMutate( 2792 CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")) 2793 .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 1))); 2794 assertTrue(res.isSuccess()); 2795 assertEquals(1, Bytes.toLong(res.getResult().getValue(FAMILY, Bytes.toBytes("B")))); 2796 2797 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2798 assertEquals(1, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B")))); 2799 2800 // CheckAndIncrement with wrong value 2801 res = region.checkAndMutate( 2802 CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("b")) 2803 .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 1))); 2804 assertFalse(res.isSuccess()); 2805 assertNull(res.getResult()); 2806 2807 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2808 assertEquals(1, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B")))); 2809 2810 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c"))); 2811 2812 // CheckAndIncrement with a filter and correct value 2813 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2814 .ifMatches(new FilterList( 2815 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2816 Bytes.toBytes("a")), 2817 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL, 2818 Bytes.toBytes("c")))) 2819 .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 2))); 2820 assertTrue(res.isSuccess()); 2821 assertEquals(3, Bytes.toLong(res.getResult().getValue(FAMILY, Bytes.toBytes("B")))); 2822 2823 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2824 assertEquals(3, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B")))); 2825 2826 // CheckAndIncrement with a filter and correct value 2827 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2828 .ifMatches(new FilterList( 2829 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2830 Bytes.toBytes("b")), 2831 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL, 2832 Bytes.toBytes("d")))) 2833 .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 2))); 2834 assertFalse(res.isSuccess()); 2835 assertNull(res.getResult()); 2836 2837 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2838 assertEquals(3, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B")))); 2839 } 2840 2841 @Test 2842 public void testCheckAndAppend() throws Throwable { 2843 final byte[] FAMILY = Bytes.toBytes("fam"); 2844 2845 // Setting up region 2846 this.region = initHRegion(tableName, method, CONF, FAMILY); 2847 2848 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a"))); 2849 2850 // CheckAndAppend with correct value 2851 CheckAndMutateResult res = region.checkAndMutate( 2852 CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")) 2853 .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")))); 2854 assertTrue(res.isSuccess()); 2855 assertEquals("b", Bytes.toString(res.getResult().getValue(FAMILY, Bytes.toBytes("B")))); 2856 2857 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2858 assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2859 2860 // CheckAndAppend with wrong value 2861 res = region.checkAndMutate( 2862 CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("b")) 2863 .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")))); 2864 assertFalse(res.isSuccess()); 2865 assertNull(res.getResult()); 2866 2867 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2868 assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2869 2870 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c"))); 2871 2872 // CheckAndAppend with a filter and correct value 2873 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2874 .ifMatches(new FilterList( 2875 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2876 Bytes.toBytes("a")), 2877 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL, 2878 Bytes.toBytes("c")))) 2879 .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("bb")))); 2880 assertTrue(res.isSuccess()); 2881 assertEquals("bbb", Bytes.toString(res.getResult().getValue(FAMILY, Bytes.toBytes("B")))); 2882 2883 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2884 assertEquals("bbb", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2885 2886 // CheckAndAppend with a filter and wrong value 2887 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2888 .ifMatches(new FilterList( 2889 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2890 Bytes.toBytes("b")), 2891 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL, 2892 Bytes.toBytes("d")))) 2893 .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("bb")))); 2894 assertFalse(res.isSuccess()); 2895 assertNull(res.getResult()); 2896 2897 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2898 assertEquals("bbb", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2899 } 2900 2901 @Test 2902 public void testCheckAndIncrementAndAppend() throws Throwable { 2903 // Setting up region 2904 this.region = initHRegion(tableName, method, CONF, fam1); 2905 2906 // CheckAndMutate with Increment and Append 2907 CheckAndMutate checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, qual) 2908 .build(new RowMutations(row).add((Mutation) new Increment(row).addColumn(fam1, qual1, 1L)) 2909 .add((Mutation) new Append(row).addColumn(fam1, qual2, Bytes.toBytes("a")))); 2910 2911 CheckAndMutateResult result = region.checkAndMutate(checkAndMutate); 2912 assertTrue(result.isSuccess()); 2913 assertEquals(1L, Bytes.toLong(result.getResult().getValue(fam1, qual1))); 2914 assertEquals("a", Bytes.toString(result.getResult().getValue(fam1, qual2))); 2915 2916 Result r = region.get(new Get(row)); 2917 assertEquals(1L, Bytes.toLong(r.getValue(fam1, qual1))); 2918 assertEquals("a", Bytes.toString(r.getValue(fam1, qual2))); 2919 2920 // Set return results to false 2921 checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, qual) 2922 .build(new RowMutations(row) 2923 .add((Mutation) new Increment(row).addColumn(fam1, qual1, 1L).setReturnResults(false)) 2924 .add((Mutation) new Append(row).addColumn(fam1, qual2, Bytes.toBytes("a")) 2925 .setReturnResults(false))); 2926 2927 result = region.checkAndMutate(checkAndMutate); 2928 assertTrue(result.isSuccess()); 2929 assertNull(result.getResult().getValue(fam1, qual1)); 2930 assertNull(result.getResult().getValue(fam1, qual2)); 2931 2932 r = region.get(new Get(row)); 2933 assertEquals(2L, Bytes.toLong(r.getValue(fam1, qual1))); 2934 assertEquals("aa", Bytes.toString(r.getValue(fam1, qual2))); 2935 2936 checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, qual) 2937 .build(new RowMutations(row).add((Mutation) new Increment(row).addColumn(fam1, qual1, 1L)) 2938 .add((Mutation) new Append(row).addColumn(fam1, qual2, Bytes.toBytes("a")) 2939 .setReturnResults(false))); 2940 2941 result = region.checkAndMutate(checkAndMutate); 2942 assertTrue(result.isSuccess()); 2943 assertEquals(3L, Bytes.toLong(result.getResult().getValue(fam1, qual1))); 2944 assertNull(result.getResult().getValue(fam1, qual2)); 2945 2946 r = region.get(new Get(row)); 2947 assertEquals(3L, Bytes.toLong(r.getValue(fam1, qual1))); 2948 assertEquals("aaa", Bytes.toString(r.getValue(fam1, qual2))); 2949 } 2950 2951 @Test 2952 public void testCheckAndRowMutations() throws Throwable { 2953 final byte[] row = Bytes.toBytes("row"); 2954 final byte[] q1 = Bytes.toBytes("q1"); 2955 final byte[] q2 = Bytes.toBytes("q2"); 2956 final byte[] q3 = Bytes.toBytes("q3"); 2957 final byte[] q4 = Bytes.toBytes("q4"); 2958 final String v1 = "v1"; 2959 2960 region = initHRegion(tableName, method, CONF, fam1); 2961 2962 // Initial values 2963 region 2964 .batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q2, Bytes.toBytes("toBeDeleted")), 2965 new Put(row).addColumn(fam1, q3, Bytes.toBytes(5L)), 2966 new Put(row).addColumn(fam1, q4, Bytes.toBytes("a")), }); 2967 2968 // Do CheckAndRowMutations 2969 CheckAndMutate checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, q1).build( 2970 new RowMutations(row).add(Arrays.asList(new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1)), 2971 new Delete(row).addColumns(fam1, q2), new Increment(row).addColumn(fam1, q3, 1), 2972 new Append(row).addColumn(fam1, q4, Bytes.toBytes("b"))))); 2973 2974 CheckAndMutateResult result = region.checkAndMutate(checkAndMutate); 2975 assertTrue(result.isSuccess()); 2976 assertEquals(6L, Bytes.toLong(result.getResult().getValue(fam1, q3))); 2977 assertEquals("ab", Bytes.toString(result.getResult().getValue(fam1, q4))); 2978 2979 // Verify the value 2980 Result r = region.get(new Get(row)); 2981 assertEquals(v1, Bytes.toString(r.getValue(fam1, q1))); 2982 assertNull(r.getValue(fam1, q2)); 2983 assertEquals(6L, Bytes.toLong(r.getValue(fam1, q3))); 2984 assertEquals("ab", Bytes.toString(r.getValue(fam1, q4))); 2985 2986 // Do CheckAndRowMutations again 2987 checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, q1) 2988 .build(new RowMutations(row).add(Arrays.asList(new Delete(row).addColumns(fam1, q1), 2989 new Put(row).addColumn(fam1, q2, Bytes.toBytes(v1)), 2990 new Increment(row).addColumn(fam1, q3, 1), 2991 new Append(row).addColumn(fam1, q4, Bytes.toBytes("b"))))); 2992 2993 result = region.checkAndMutate(checkAndMutate); 2994 assertFalse(result.isSuccess()); 2995 assertNull(result.getResult()); 2996 2997 // Verify the value 2998 r = region.get(new Get(row)); 2999 assertEquals(v1, Bytes.toString(r.getValue(fam1, q1))); 3000 assertNull(r.getValue(fam1, q2)); 3001 assertEquals(6L, Bytes.toLong(r.getValue(fam1, q3))); 3002 assertEquals("ab", Bytes.toString(r.getValue(fam1, q4))); 3003 } 3004 3005 // //////////////////////////////////////////////////////////////////////////// 3006 // Delete tests 3007 // //////////////////////////////////////////////////////////////////////////// 3008 @Test 3009 public void testDelete_multiDeleteColumn() throws IOException { 3010 byte[] row1 = Bytes.toBytes("row1"); 3011 byte[] fam1 = Bytes.toBytes("fam1"); 3012 byte[] qual = Bytes.toBytes("qualifier"); 3013 byte[] value = Bytes.toBytes("value"); 3014 3015 Put put = new Put(row1); 3016 put.addColumn(fam1, qual, 1, value); 3017 put.addColumn(fam1, qual, 2, value); 3018 3019 this.region = initHRegion(tableName, method, CONF, fam1); 3020 region.put(put); 3021 3022 // We do support deleting more than 1 'latest' version 3023 Delete delete = new Delete(row1); 3024 delete.addColumn(fam1, qual); 3025 delete.addColumn(fam1, qual); 3026 region.delete(delete); 3027 3028 Get get = new Get(row1); 3029 get.addFamily(fam1); 3030 Result r = region.get(get); 3031 assertEquals(0, r.size()); 3032 } 3033 3034 @Test 3035 public void testDelete_CheckFamily() throws IOException { 3036 byte[] row1 = Bytes.toBytes("row1"); 3037 byte[] fam1 = Bytes.toBytes("fam1"); 3038 byte[] fam2 = Bytes.toBytes("fam2"); 3039 byte[] fam3 = Bytes.toBytes("fam3"); 3040 byte[] fam4 = Bytes.toBytes("fam4"); 3041 3042 // Setting up region 3043 this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3); 3044 List<Cell> kvs = new ArrayList<>(); 3045 kvs.add(new KeyValue(row1, fam4, null, null)); 3046 3047 byte[] forUnitTestsOnly = Bytes.toBytes("ForUnitTestsOnly"); 3048 3049 // testing existing family 3050 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<>(Bytes.BYTES_COMPARATOR); 3051 deleteMap.put(fam2, kvs); 3052 region.delete(new Delete(forUnitTestsOnly, HConstants.LATEST_TIMESTAMP, deleteMap)); 3053 3054 // testing non existing family 3055 NavigableMap<byte[], List<Cell>> deleteMap2 = new TreeMap<>(Bytes.BYTES_COMPARATOR); 3056 deleteMap2.put(fam4, kvs); 3057 assertThrows(NoSuchColumnFamilyException.class, 3058 () -> region.delete(new Delete(forUnitTestsOnly, HConstants.LATEST_TIMESTAMP, deleteMap2)), 3059 "Family " + Bytes.toString(fam4) + " does exist"); 3060 } 3061 3062 @Test 3063 public void testDelete_mixed() throws IOException, InterruptedException { 3064 byte[] fam = Bytes.toBytes("info"); 3065 byte[][] families = { fam }; 3066 this.region = initHRegion(tableName, method, CONF, families); 3067 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge()); 3068 3069 byte[] row = Bytes.toBytes("table_name"); 3070 // column names 3071 byte[] serverinfo = Bytes.toBytes("serverinfo"); 3072 byte[] splitA = Bytes.toBytes("splitA"); 3073 byte[] splitB = Bytes.toBytes("splitB"); 3074 3075 // add some data: 3076 Put put = new Put(row); 3077 put.addColumn(fam, splitA, Bytes.toBytes("reference_A")); 3078 region.put(put); 3079 3080 put = new Put(row); 3081 put.addColumn(fam, splitB, Bytes.toBytes("reference_B")); 3082 region.put(put); 3083 3084 put = new Put(row); 3085 put.addColumn(fam, serverinfo, Bytes.toBytes("ip_address")); 3086 region.put(put); 3087 3088 // ok now delete a split: 3089 Delete delete = new Delete(row); 3090 delete.addColumns(fam, splitA); 3091 region.delete(delete); 3092 3093 // assert some things: 3094 Get get = new Get(row).addColumn(fam, serverinfo); 3095 Result result = region.get(get); 3096 assertEquals(1, result.size()); 3097 3098 get = new Get(row).addColumn(fam, splitA); 3099 result = region.get(get); 3100 assertEquals(0, result.size()); 3101 3102 get = new Get(row).addColumn(fam, splitB); 3103 result = region.get(get); 3104 assertEquals(1, result.size()); 3105 3106 // Assert that after a delete, I can put. 3107 put = new Put(row); 3108 put.addColumn(fam, splitA, Bytes.toBytes("reference_A")); 3109 region.put(put); 3110 get = new Get(row); 3111 result = region.get(get); 3112 assertEquals(3, result.size()); 3113 3114 // Now delete all... then test I can add stuff back 3115 delete = new Delete(row); 3116 region.delete(delete); 3117 assertEquals(0, region.get(get).size()); 3118 3119 region.put(new Put(row).addColumn(fam, splitA, Bytes.toBytes("reference_A"))); 3120 result = region.get(get); 3121 assertEquals(1, result.size()); 3122 } 3123 3124 @Test 3125 public void testDeleteRowWithFutureTs() throws IOException { 3126 byte[] fam = Bytes.toBytes("info"); 3127 byte[][] families = { fam }; 3128 this.region = initHRegion(tableName, method, CONF, families); 3129 byte[] row = Bytes.toBytes("table_name"); 3130 // column names 3131 byte[] serverinfo = Bytes.toBytes("serverinfo"); 3132 3133 // add data in the far future 3134 Put put = new Put(row); 3135 put.addColumn(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value")); 3136 region.put(put); 3137 3138 // now delete something in the present 3139 Delete delete = new Delete(row); 3140 region.delete(delete); 3141 3142 // make sure we still see our data 3143 Get get = new Get(row).addColumn(fam, serverinfo); 3144 Result result = region.get(get); 3145 assertEquals(1, result.size()); 3146 3147 // delete the future row 3148 delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3); 3149 region.delete(delete); 3150 3151 // make sure it is gone 3152 get = new Get(row).addColumn(fam, serverinfo); 3153 result = region.get(get); 3154 assertEquals(0, result.size()); 3155 } 3156 3157 /** 3158 * Tests that the special LATEST_TIMESTAMP option for puts gets replaced by the actual timestamp 3159 */ 3160 @Test 3161 public void testPutWithLatestTS() throws IOException { 3162 byte[] fam = Bytes.toBytes("info"); 3163 byte[][] families = { fam }; 3164 this.region = initHRegion(tableName, method, CONF, families); 3165 byte[] row = Bytes.toBytes("row1"); 3166 // column names 3167 byte[] qual = Bytes.toBytes("qual"); 3168 3169 // add data with LATEST_TIMESTAMP, put without WAL 3170 Put put = new Put(row); 3171 put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value")); 3172 region.put(put); 3173 3174 // Make sure it shows up with an actual timestamp 3175 Get get = new Get(row).addColumn(fam, qual); 3176 Result result = region.get(get); 3177 assertEquals(1, result.size()); 3178 Cell kv = result.rawCells()[0]; 3179 LOG.info("Got: " + kv); 3180 assertTrue(kv.getTimestamp() != HConstants.LATEST_TIMESTAMP, 3181 "LATEST_TIMESTAMP was not replaced with real timestamp"); 3182 3183 // Check same with WAL enabled (historically these took different 3184 // code paths, so check both) 3185 row = Bytes.toBytes("row2"); 3186 put = new Put(row); 3187 put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value")); 3188 region.put(put); 3189 3190 // Make sure it shows up with an actual timestamp 3191 get = new Get(row).addColumn(fam, qual); 3192 result = region.get(get); 3193 assertEquals(1, result.size()); 3194 kv = result.rawCells()[0]; 3195 LOG.info("Got: " + kv); 3196 assertTrue(kv.getTimestamp() != HConstants.LATEST_TIMESTAMP, 3197 "LATEST_TIMESTAMP was not replaced with real timestamp"); 3198 } 3199 3200 /** 3201 * Tests that there is server-side filtering for invalid timestamp upper bound. Note that the 3202 * timestamp lower bound is automatically handled for us by the TTL field. 3203 */ 3204 @Test 3205 public void testPutWithTsSlop() throws IOException { 3206 byte[] fam = Bytes.toBytes("info"); 3207 byte[][] families = { fam }; 3208 3209 // add data with a timestamp that is too recent for range. Ensure assert 3210 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000); 3211 this.region = initHRegion(tableName, method, CONF, families); 3212 boolean caughtExcep = false; 3213 try { 3214 // no TS specified == use latest. should not error 3215 region.put(new Put(row).addColumn(fam, Bytes.toBytes("qual"), Bytes.toBytes("value"))); 3216 // TS out of range. should error 3217 region.put(new Put(row).addColumn(fam, Bytes.toBytes("qual"), 3218 EnvironmentEdgeManager.currentTime() + 2000, Bytes.toBytes("value"))); 3219 fail("Expected IOE for TS out of configured timerange"); 3220 } catch (FailedSanityCheckException ioe) { 3221 LOG.debug("Received expected exception", ioe); 3222 caughtExcep = true; 3223 } 3224 assertTrue(caughtExcep, "Should catch FailedSanityCheckException"); 3225 } 3226 3227 @Test 3228 public void testScanner_DeleteOneFamilyNotAnother() throws IOException { 3229 byte[] fam1 = Bytes.toBytes("columnA"); 3230 byte[] fam2 = Bytes.toBytes("columnB"); 3231 this.region = initHRegion(tableName, method, CONF, fam1, fam2); 3232 byte[] rowA = Bytes.toBytes("rowA"); 3233 byte[] rowB = Bytes.toBytes("rowB"); 3234 3235 byte[] value = Bytes.toBytes("value"); 3236 3237 Delete delete = new Delete(rowA); 3238 delete.addFamily(fam1); 3239 3240 region.delete(delete); 3241 3242 // now create data. 3243 Put put = new Put(rowA); 3244 put.addColumn(fam2, null, value); 3245 region.put(put); 3246 3247 put = new Put(rowB); 3248 put.addColumn(fam1, null, value); 3249 put.addColumn(fam2, null, value); 3250 region.put(put); 3251 3252 Scan scan = new Scan(); 3253 scan.addFamily(fam1).addFamily(fam2); 3254 try (InternalScanner s = region.getScanner(scan)) { 3255 List<Cell> results = new ArrayList<>(); 3256 s.next(results); 3257 assertTrue(CellUtil.matchingRows(results.get(0), rowA)); 3258 3259 results.clear(); 3260 s.next(results); 3261 assertTrue(CellUtil.matchingRows(results.get(0), rowB)); 3262 } 3263 } 3264 3265 @Test 3266 public void testDataInMemoryWithoutWAL() throws IOException { 3267 FileSystem fs = FileSystem.get(CONF); 3268 Path rootDir = new Path(dir + method); 3269 fs.mkdirs(new Path(rootDir, method)); 3270 FSHLog hLog = new FSHLog(fs, rootDir, method, CONF); 3271 hLog.init(); 3272 // This chunk creation is done throughout the code base. Do we want to move it into core? 3273 // It is missing from this test. W/o it we NPE. 3274 region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, hLog, 3275 COLUMN_FAMILY_BYTES); 3276 3277 Cell originalCell = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row) 3278 .setFamily(COLUMN_FAMILY_BYTES).setQualifier(qual1) 3279 .setTimestamp(EnvironmentEdgeManager.currentTime()).setType(KeyValue.Type.Put.getCode()) 3280 .setValue(value1).build(); 3281 final long originalSize = originalCell.getSerializedSize(); 3282 3283 Cell addCell = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row) 3284 .setFamily(COLUMN_FAMILY_BYTES).setQualifier(qual1) 3285 .setTimestamp(EnvironmentEdgeManager.currentTime()).setType(KeyValue.Type.Put.getCode()) 3286 .setValue(Bytes.toBytes("xxxxxxxxxx")).build(); 3287 final long addSize = addCell.getSerializedSize(); 3288 3289 LOG.info("originalSize:" + originalSize + ", addSize:" + addSize); 3290 // start test. We expect that the addPut's durability will be replaced 3291 // by originalPut's durability. 3292 3293 // case 1: 3294 testDataInMemoryWithoutWAL(region, 3295 new Put(row).add(originalCell).setDurability(Durability.SKIP_WAL), 3296 new Put(row).add(addCell).setDurability(Durability.SKIP_WAL), originalSize + addSize); 3297 3298 // case 2: 3299 testDataInMemoryWithoutWAL(region, 3300 new Put(row).add(originalCell).setDurability(Durability.SKIP_WAL), 3301 new Put(row).add(addCell).setDurability(Durability.SYNC_WAL), originalSize + addSize); 3302 3303 // case 3: 3304 testDataInMemoryWithoutWAL(region, 3305 new Put(row).add(originalCell).setDurability(Durability.SYNC_WAL), 3306 new Put(row).add(addCell).setDurability(Durability.SKIP_WAL), 0); 3307 3308 // case 4: 3309 testDataInMemoryWithoutWAL(region, 3310 new Put(row).add(originalCell).setDurability(Durability.SYNC_WAL), 3311 new Put(row).add(addCell).setDurability(Durability.SYNC_WAL), 0); 3312 } 3313 3314 private static void testDataInMemoryWithoutWAL(HRegion region, Put originalPut, final Put addPut, 3315 long delta) throws IOException { 3316 final long initSize = region.getDataInMemoryWithoutWAL(); 3317 // save normalCPHost and replaced by mockedCPHost 3318 RegionCoprocessorHost normalCPHost = region.getCoprocessorHost(); 3319 RegionCoprocessorHost mockedCPHost = mock(RegionCoprocessorHost.class); 3320 // Because the preBatchMutate returns void, we can't do usual Mockito when...then form. Must 3321 // do below format (from Mockito doc). 3322 doAnswer(new Answer<Void>() { 3323 @Override 3324 public Void answer(InvocationOnMock invocation) throws Throwable { 3325 MiniBatchOperationInProgress<Mutation> mb = invocation.getArgument(0); 3326 mb.addOperationsFromCP(0, new Mutation[] { addPut }); 3327 return null; 3328 } 3329 }).when(mockedCPHost).preBatchMutate(isA(MiniBatchOperationInProgress.class)); 3330 ColumnFamilyDescriptorBuilder builder = 3331 ColumnFamilyDescriptorBuilder.newBuilder(COLUMN_FAMILY_BYTES); 3332 ScanInfo info = new ScanInfo(CONF, builder.build(), Long.MAX_VALUE, Long.MAX_VALUE, 3333 region.getCellComparator()); 3334 when(mockedCPHost.preFlushScannerOpen(any(HStore.class), any())).thenReturn(info); 3335 3336 when(mockedCPHost.preFlush(any(), any(StoreScanner.class), any())) 3337 .thenAnswer(i -> i.getArgument(1)); 3338 region.setCoprocessorHost(mockedCPHost); 3339 3340 region.put(originalPut); 3341 region.setCoprocessorHost(normalCPHost); 3342 final long finalSize = region.getDataInMemoryWithoutWAL(); 3343 assertEquals(finalSize, initSize + delta, 3344 "finalSize:" + finalSize + ", initSize:" + initSize + ", delta:" + delta); 3345 } 3346 3347 @Test 3348 public void testDeleteColumns_PostInsert() throws IOException, InterruptedException { 3349 Delete delete = new Delete(row); 3350 delete.addColumns(fam1, qual1); 3351 doTestDelete_AndPostInsert(delete); 3352 } 3353 3354 @Test 3355 public void testaddFamily_PostInsert() throws IOException, InterruptedException { 3356 Delete delete = new Delete(row); 3357 delete.addFamily(fam1); 3358 doTestDelete_AndPostInsert(delete); 3359 } 3360 3361 public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException { 3362 this.region = initHRegion(tableName, method, CONF, fam1); 3363 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge()); 3364 Put put = new Put(row); 3365 put.addColumn(fam1, qual1, value1); 3366 region.put(put); 3367 3368 // now delete the value: 3369 region.delete(delete); 3370 3371 // ok put data: 3372 put = new Put(row); 3373 put.addColumn(fam1, qual1, value2); 3374 region.put(put); 3375 3376 // ok get: 3377 Get get = new Get(row); 3378 get.addColumn(fam1, qual1); 3379 3380 Result r = region.get(get); 3381 assertEquals(1, r.size()); 3382 assertArrayEquals(value2, r.getValue(fam1, qual1)); 3383 3384 // next: 3385 Scan scan = new Scan().withStartRow(row); 3386 scan.addColumn(fam1, qual1); 3387 try (InternalScanner s = region.getScanner(scan)) { 3388 List<Cell> results = new ArrayList<>(); 3389 assertEquals(false, s.next(results)); 3390 assertEquals(1, results.size()); 3391 Cell kv = results.get(0); 3392 3393 assertArrayEquals(value2, CellUtil.cloneValue(kv)); 3394 assertArrayEquals(fam1, CellUtil.cloneFamily(kv)); 3395 assertArrayEquals(qual1, CellUtil.cloneQualifier(kv)); 3396 assertArrayEquals(row, CellUtil.cloneRow(kv)); 3397 } 3398 } 3399 3400 @Test 3401 public void testDelete_CheckTimestampUpdated() throws IOException { 3402 byte[] row1 = Bytes.toBytes("row1"); 3403 byte[] col1 = Bytes.toBytes("col1"); 3404 byte[] col2 = Bytes.toBytes("col2"); 3405 byte[] col3 = Bytes.toBytes("col3"); 3406 3407 byte[] forUnitTestsOnly = Bytes.toBytes("ForUnitTestsOnly"); 3408 3409 // Setting up region 3410 this.region = initHRegion(tableName, method, CONF, fam1); 3411 // Building checkerList 3412 List<Cell> kvs = new ArrayList<>(); 3413 kvs.add(new KeyValue(row1, fam1, col1, null)); 3414 kvs.add(new KeyValue(row1, fam1, col2, null)); 3415 kvs.add(new KeyValue(row1, fam1, col3, null)); 3416 3417 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<>(Bytes.BYTES_COMPARATOR); 3418 deleteMap.put(fam1, kvs); 3419 region.delete(new Delete(forUnitTestsOnly, HConstants.LATEST_TIMESTAMP, deleteMap)); 3420 3421 // extract the key values out the memstore: 3422 // This is kinda hacky, but better than nothing... 3423 long now = EnvironmentEdgeManager.currentTime(); 3424 AbstractMemStore memstore = (AbstractMemStore) region.getStore(fam1).memstore; 3425 Cell firstCell = memstore.getActive().first(); 3426 assertTrue(firstCell.getTimestamp() <= now); 3427 now = firstCell.getTimestamp(); 3428 for (Cell cell : memstore.getActive().getCellSet()) { 3429 assertTrue(cell.getTimestamp() <= now); 3430 now = cell.getTimestamp(); 3431 } 3432 } 3433 3434 // //////////////////////////////////////////////////////////////////////////// 3435 // Get tests 3436 // //////////////////////////////////////////////////////////////////////////// 3437 @Test 3438 public void testGet_FamilyChecker() throws IOException { 3439 byte[] row1 = Bytes.toBytes("row1"); 3440 byte[] fam1 = Bytes.toBytes("fam1"); 3441 byte[] fam2 = Bytes.toBytes("False"); 3442 byte[] col1 = Bytes.toBytes("col1"); 3443 3444 // Setting up region 3445 this.region = initHRegion(tableName, method, CONF, fam1); 3446 Get get = new Get(row1); 3447 get.addColumn(fam2, col1); 3448 3449 // Test 3450 try { 3451 region.get(get); 3452 fail("Expecting DoNotRetryIOException in get but did not get any"); 3453 } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) { 3454 LOG.info("Got expected DoNotRetryIOException successfully"); 3455 } 3456 } 3457 3458 @Test 3459 public void testGet_Basic() throws IOException { 3460 byte[] row1 = Bytes.toBytes("row1"); 3461 byte[] fam1 = Bytes.toBytes("fam1"); 3462 byte[] col1 = Bytes.toBytes("col1"); 3463 byte[] col2 = Bytes.toBytes("col2"); 3464 byte[] col3 = Bytes.toBytes("col3"); 3465 byte[] col4 = Bytes.toBytes("col4"); 3466 byte[] col5 = Bytes.toBytes("col5"); 3467 3468 // Setting up region 3469 this.region = initHRegion(tableName, method, CONF, fam1); 3470 // Add to memstore 3471 Put put = new Put(row1); 3472 put.addColumn(fam1, col1, null); 3473 put.addColumn(fam1, col2, null); 3474 put.addColumn(fam1, col3, null); 3475 put.addColumn(fam1, col4, null); 3476 put.addColumn(fam1, col5, null); 3477 region.put(put); 3478 3479 Get get = new Get(row1); 3480 get.addColumn(fam1, col2); 3481 get.addColumn(fam1, col4); 3482 // Expected result 3483 KeyValue kv1 = new KeyValue(row1, fam1, col2); 3484 KeyValue kv2 = new KeyValue(row1, fam1, col4); 3485 KeyValue[] expected = { kv1, kv2 }; 3486 3487 // Test 3488 Result res = region.get(get); 3489 assertEquals(expected.length, res.size()); 3490 for (int i = 0; i < res.size(); i++) { 3491 assertTrue(CellUtil.matchingRows(expected[i], res.rawCells()[i])); 3492 assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i])); 3493 assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i])); 3494 } 3495 3496 // Test using a filter on a Get 3497 Get g = new Get(row1); 3498 final int count = 2; 3499 g.setFilter(new ColumnCountGetFilter(count)); 3500 res = region.get(g); 3501 assertEquals(count, res.size()); 3502 } 3503 3504 @Test 3505 public void testGet_Empty() throws IOException { 3506 byte[] row = Bytes.toBytes("row"); 3507 byte[] fam = Bytes.toBytes("fam"); 3508 3509 this.region = initHRegion(tableName, method, CONF, fam); 3510 Get get = new Get(row); 3511 get.addFamily(fam); 3512 Result r = region.get(get); 3513 3514 assertTrue(r.isEmpty()); 3515 } 3516 3517 @Test 3518 public void testGetWithFilter() throws IOException, InterruptedException { 3519 byte[] row1 = Bytes.toBytes("row1"); 3520 byte[] fam1 = Bytes.toBytes("fam1"); 3521 byte[] col1 = Bytes.toBytes("col1"); 3522 byte[] value1 = Bytes.toBytes("value1"); 3523 byte[] value2 = Bytes.toBytes("value2"); 3524 3525 final int maxVersions = 3; 3526 TableDescriptor tableDescriptor = 3527 TableDescriptorBuilder.newBuilder(TableName.valueOf("testFilterAndColumnTracker")) 3528 .setColumnFamily( 3529 ColumnFamilyDescriptorBuilder.newBuilder(fam1).setMaxVersions(maxVersions).build()) 3530 .build(); 3531 ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null, 3532 MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT); 3533 RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 3534 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + ".log"); 3535 final WAL wal = HBaseTestingUtil.createWal(TEST_UTIL.getConfiguration(), logDir, info); 3536 this.region = TEST_UTIL.createLocalHRegion(info, CONF, tableDescriptor, wal); 3537 3538 // Put 4 version to memstore 3539 long ts = 0; 3540 Put put = new Put(row1, ts); 3541 put.addColumn(fam1, col1, value1); 3542 region.put(put); 3543 put = new Put(row1, ts + 1); 3544 put.addColumn(fam1, col1, Bytes.toBytes("filter1")); 3545 region.put(put); 3546 put = new Put(row1, ts + 2); 3547 put.addColumn(fam1, col1, Bytes.toBytes("filter2")); 3548 region.put(put); 3549 put = new Put(row1, ts + 3); 3550 put.addColumn(fam1, col1, value2); 3551 region.put(put); 3552 3553 Get get = new Get(row1); 3554 get.readAllVersions(); 3555 Result res = region.get(get); 3556 // Get 3 versions, the oldest version has gone from user view 3557 assertEquals(maxVersions, res.size()); 3558 3559 get.setFilter(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("value"))); 3560 res = region.get(get); 3561 // When use value filter, the oldest version should still gone from user view and it 3562 // should only return one key vaule 3563 assertEquals(1, res.size()); 3564 assertTrue(CellUtil.matchingValue(new KeyValue(row1, fam1, col1, value2), res.rawCells()[0])); 3565 assertEquals(ts + 3, res.rawCells()[0].getTimestamp()); 3566 3567 region.flush(true); 3568 region.compact(true); 3569 Thread.sleep(1000); 3570 res = region.get(get); 3571 // After flush and compact, the result should be consistent with previous result 3572 assertEquals(1, res.size()); 3573 assertTrue(CellUtil.matchingValue(new KeyValue(row1, fam1, col1, value2), res.rawCells()[0])); 3574 } 3575 3576 // //////////////////////////////////////////////////////////////////////////// 3577 // Scanner tests 3578 // //////////////////////////////////////////////////////////////////////////// 3579 @Test 3580 public void testGetScanner_WithOkFamilies() throws IOException { 3581 byte[] fam1 = Bytes.toBytes("fam1"); 3582 byte[] fam2 = Bytes.toBytes("fam2"); 3583 3584 byte[][] families = { fam1, fam2 }; 3585 3586 // Setting up region 3587 this.region = initHRegion(tableName, method, CONF, families); 3588 Scan scan = new Scan(); 3589 scan.addFamily(fam1); 3590 scan.addFamily(fam2); 3591 region.getScanner(scan).close(); 3592 } 3593 3594 @Test 3595 public void testGetScanner_WithNotOkFamilies() throws IOException { 3596 byte[] fam1 = Bytes.toBytes("fam1"); 3597 byte[] fam2 = Bytes.toBytes("fam2"); 3598 3599 byte[][] families = { fam1 }; 3600 3601 // Setting up region 3602 this.region = initHRegion(tableName, method, CONF, families); 3603 Scan scan = new Scan(); 3604 scan.addFamily(fam2); 3605 assertThrows(NoSuchColumnFamilyException.class, () -> region.getScanner(scan)); 3606 } 3607 3608 @Test 3609 public void testGetScanner_WithNoFamilies() throws IOException { 3610 byte[] row1 = Bytes.toBytes("row1"); 3611 byte[] fam1 = Bytes.toBytes("fam1"); 3612 byte[] fam2 = Bytes.toBytes("fam2"); 3613 byte[] fam3 = Bytes.toBytes("fam3"); 3614 byte[] fam4 = Bytes.toBytes("fam4"); 3615 3616 byte[][] families = { fam1, fam2, fam3, fam4 }; 3617 3618 // Setting up region 3619 this.region = initHRegion(tableName, method, CONF, families); 3620 // Putting data in Region 3621 Put put = new Put(row1); 3622 put.addColumn(fam1, null, null); 3623 put.addColumn(fam2, null, null); 3624 put.addColumn(fam3, null, null); 3625 put.addColumn(fam4, null, null); 3626 region.put(put); 3627 3628 Scan scan = null; 3629 3630 // Testing to see how many scanners that is produced by getScanner, starting with known number, 3631 // 2 - current = 1 3632 scan = new Scan(); 3633 scan.addFamily(fam2); 3634 scan.addFamily(fam4); 3635 try (RegionScannerImpl is = region.getScanner(scan)) { 3636 assertEquals(1, is.storeHeap.getHeap().size()); 3637 } 3638 3639 scan = new Scan(); 3640 try (RegionScannerImpl is = region.getScanner(scan)) { 3641 assertEquals(families.length - 1, is.storeHeap.getHeap().size()); 3642 } 3643 } 3644 3645 /** 3646 * This method tests https://issues.apache.org/jira/browse/HBASE-2516. 3647 */ 3648 @Test 3649 public void testGetScanner_WithRegionClosed() throws IOException { 3650 byte[] fam1 = Bytes.toBytes("fam1"); 3651 byte[] fam2 = Bytes.toBytes("fam2"); 3652 3653 byte[][] families = { fam1, fam2 }; 3654 3655 // Setting up region 3656 region = initHRegion(tableName, method, CONF, families); 3657 region.closed.set(true); 3658 try { 3659 assertThrows(NotServingRegionException.class, () -> region.getScanner(null)); 3660 } finally { 3661 // so we can close the region in tearDown 3662 region.closed.set(false); 3663 } 3664 } 3665 3666 @Test 3667 public void testRegionScanner_getFilesRead() throws IOException { 3668 // Setup: init region with one family; put two rows and flush to create store files. 3669 byte[] family = Bytes.toBytes("fam1"); 3670 byte[][] families = { family }; 3671 this.region = initHRegion(tableName, method, CONF, families); 3672 Put put = new Put(Bytes.toBytes("row1")); 3673 put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); 3674 region.put(put); 3675 put = new Put(Bytes.toBytes("row2")); 3676 put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); 3677 region.put(put); 3678 region.flush(true); 3679 3680 // Collect expected store file paths from all stores (before opening the scanner). 3681 Set<Path> expectedFilePaths = new HashSet<>(); 3682 FileSystem fs = region.getFilesystem(); 3683 for (HStore store : region.getStores()) { 3684 for (HStoreFile storeFile : store.getStorefiles()) { 3685 expectedFilePaths.add(fs.makeQualified(storeFile.getPath())); 3686 } 3687 } 3688 assertTrue(expectedFilePaths.size() >= 1, "Should have at least one store file after flush"); 3689 3690 // Get region scanner; before close getFilesRead must be empty. 3691 RegionScannerImpl scanner = region.getScanner(new Scan()); 3692 3693 Set<Path> filesReadBeforeClose = scanner.getFilesRead(); 3694 assertTrue(filesReadBeforeClose.isEmpty(), "Should return empty set before closing"); 3695 3696 // Drain scanner (next up to two rows) to exercise store heap reads. 3697 List<Cell> cells = new ArrayList<>(); 3698 int count = 0; 3699 while (count < 2) { 3700 if (!scanner.next(cells)) { 3701 break; 3702 } 3703 cells.clear(); 3704 count++; 3705 } 3706 3707 // Still before close: set must remain empty until scanner is closed. 3708 filesReadBeforeClose = scanner.getFilesRead(); 3709 assertTrue(filesReadBeforeClose.isEmpty(), 3710 "Should return empty set before closing even after scanning"); 3711 scanner.close(); 3712 3713 // After close: set must contain exactly the expected store file paths. 3714 Set<Path> filesReadAfterClose = scanner.getFilesRead(); 3715 assertEquals(expectedFilePaths.size(), filesReadAfterClose.size(), 3716 "Should have exact file count after closing"); 3717 assertEquals(expectedFilePaths, filesReadAfterClose, "Should contain all expected file paths"); 3718 } 3719 3720 @Test 3721 public void testRegionScanner_Next() throws IOException { 3722 byte[] row1 = Bytes.toBytes("row1"); 3723 byte[] row2 = Bytes.toBytes("row2"); 3724 byte[] fam1 = Bytes.toBytes("fam1"); 3725 byte[] fam2 = Bytes.toBytes("fam2"); 3726 byte[] fam3 = Bytes.toBytes("fam3"); 3727 byte[] fam4 = Bytes.toBytes("fam4"); 3728 3729 byte[][] families = { fam1, fam2, fam3, fam4 }; 3730 long ts = EnvironmentEdgeManager.currentTime(); 3731 3732 // Setting up region 3733 this.region = initHRegion(tableName, method, CONF, families); 3734 // Putting data in Region 3735 Put put = null; 3736 put = new Put(row1); 3737 put.addColumn(fam1, (byte[]) null, ts, null); 3738 put.addColumn(fam2, (byte[]) null, ts, null); 3739 put.addColumn(fam3, (byte[]) null, ts, null); 3740 put.addColumn(fam4, (byte[]) null, ts, null); 3741 region.put(put); 3742 3743 put = new Put(row2); 3744 put.addColumn(fam1, (byte[]) null, ts, null); 3745 put.addColumn(fam2, (byte[]) null, ts, null); 3746 put.addColumn(fam3, (byte[]) null, ts, null); 3747 put.addColumn(fam4, (byte[]) null, ts, null); 3748 region.put(put); 3749 3750 Scan scan = new Scan(); 3751 scan.addFamily(fam2); 3752 scan.addFamily(fam4); 3753 try (InternalScanner is = region.getScanner(scan)) { 3754 List<ExtendedCell> res = null; 3755 3756 // Result 1 3757 List<ExtendedCell> expected1 = new ArrayList<>(); 3758 expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null)); 3759 expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null)); 3760 3761 res = new ArrayList<>(); 3762 is.next(res); 3763 for (int i = 0; i < res.size(); i++) { 3764 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected1.get(i), res.get(i))); 3765 } 3766 3767 // Result 2 3768 List<ExtendedCell> expected2 = new ArrayList<>(); 3769 expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null)); 3770 expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null)); 3771 3772 res = new ArrayList<>(); 3773 is.next(res); 3774 for (int i = 0; i < res.size(); i++) { 3775 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected2.get(i), res.get(i))); 3776 } 3777 } 3778 } 3779 3780 @Test 3781 public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException { 3782 byte[] row1 = Bytes.toBytes("row1"); 3783 byte[] qf1 = Bytes.toBytes("qualifier1"); 3784 byte[] qf2 = Bytes.toBytes("qualifier2"); 3785 byte[] fam1 = Bytes.toBytes("fam1"); 3786 byte[][] families = { fam1 }; 3787 3788 long ts1 = EnvironmentEdgeManager.currentTime(); 3789 long ts2 = ts1 + 1; 3790 long ts3 = ts1 + 2; 3791 3792 // Setting up region 3793 this.region = initHRegion(tableName, method, CONF, families); 3794 // Putting data in Region 3795 Put put = null; 3796 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 3797 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 3798 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 3799 3800 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 3801 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 3802 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 3803 3804 put = new Put(row1); 3805 put.add(kv13); 3806 put.add(kv12); 3807 put.add(kv11); 3808 put.add(kv23); 3809 put.add(kv22); 3810 put.add(kv21); 3811 region.put(put); 3812 3813 // Expected 3814 List<Cell> expected = new ArrayList<>(); 3815 expected.add(kv13); 3816 expected.add(kv12); 3817 3818 Scan scan = new Scan().withStartRow(row1); 3819 scan.addColumn(fam1, qf1); 3820 scan.readVersions(MAX_VERSIONS); 3821 List<Cell> actual = new ArrayList<>(); 3822 try (InternalScanner scanner = region.getScanner(scan)) { 3823 boolean hasNext = scanner.next(actual); 3824 assertEquals(false, hasNext); 3825 3826 // Verify result 3827 for (int i = 0; i < expected.size(); i++) { 3828 assertEquals(expected.get(i), actual.get(i)); 3829 } 3830 } 3831 } 3832 3833 @Test 3834 public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException { 3835 byte[] row1 = Bytes.toBytes("row1"); 3836 byte[] qf1 = Bytes.toBytes("qualifier1"); 3837 byte[] qf2 = Bytes.toBytes("qualifier2"); 3838 byte[] fam1 = Bytes.toBytes("fam1"); 3839 byte[][] families = { fam1 }; 3840 3841 long ts1 = 1; 3842 long ts2 = ts1 + 1; 3843 long ts3 = ts1 + 2; 3844 3845 // Setting up region 3846 this.region = initHRegion(tableName, method, CONF, families); 3847 // Putting data in Region 3848 Put put = null; 3849 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 3850 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 3851 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 3852 3853 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 3854 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 3855 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 3856 3857 put = new Put(row1); 3858 put.add(kv13); 3859 put.add(kv12); 3860 put.add(kv11); 3861 put.add(kv23); 3862 put.add(kv22); 3863 put.add(kv21); 3864 region.put(put); 3865 region.flush(true); 3866 3867 // Expected 3868 List<ExtendedCell> expected = new ArrayList<>(); 3869 expected.add(kv13); 3870 expected.add(kv12); 3871 expected.add(kv23); 3872 expected.add(kv22); 3873 3874 Scan scan = new Scan().withStartRow(row1); 3875 scan.addColumn(fam1, qf1); 3876 scan.addColumn(fam1, qf2); 3877 scan.readVersions(MAX_VERSIONS); 3878 List<ExtendedCell> actual = new ArrayList<>(); 3879 try (InternalScanner scanner = region.getScanner(scan)) { 3880 boolean hasNext = scanner.next(actual); 3881 assertEquals(false, hasNext); 3882 3883 // Verify result 3884 for (int i = 0; i < expected.size(); i++) { 3885 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); 3886 } 3887 } 3888 } 3889 3890 @Test 3891 public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() 3892 throws IOException { 3893 byte[] row1 = Bytes.toBytes("row1"); 3894 byte[] fam1 = Bytes.toBytes("fam1"); 3895 byte[][] families = { fam1 }; 3896 byte[] qf1 = Bytes.toBytes("qualifier1"); 3897 byte[] qf2 = Bytes.toBytes("qualifier2"); 3898 3899 long ts1 = 1; 3900 long ts2 = ts1 + 1; 3901 long ts3 = ts1 + 2; 3902 long ts4 = ts1 + 3; 3903 3904 // Setting up region 3905 this.region = initHRegion(tableName, method, CONF, families); 3906 // Putting data in Region 3907 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null); 3908 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 3909 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 3910 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 3911 3912 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null); 3913 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 3914 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 3915 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 3916 3917 Put put = null; 3918 put = new Put(row1); 3919 put.add(kv14); 3920 put.add(kv24); 3921 region.put(put); 3922 region.flush(true); 3923 3924 put = new Put(row1); 3925 put.add(kv23); 3926 put.add(kv13); 3927 region.put(put); 3928 region.flush(true); 3929 3930 put = new Put(row1); 3931 put.add(kv22); 3932 put.add(kv12); 3933 region.put(put); 3934 region.flush(true); 3935 3936 put = new Put(row1); 3937 put.add(kv21); 3938 put.add(kv11); 3939 region.put(put); 3940 3941 // Expected 3942 List<ExtendedCell> expected = new ArrayList<>(); 3943 expected.add(kv14); 3944 expected.add(kv13); 3945 expected.add(kv12); 3946 expected.add(kv24); 3947 expected.add(kv23); 3948 expected.add(kv22); 3949 3950 Scan scan = new Scan().withStartRow(row1); 3951 scan.addColumn(fam1, qf1); 3952 scan.addColumn(fam1, qf2); 3953 int versions = 3; 3954 scan.readVersions(versions); 3955 List<ExtendedCell> actual = new ArrayList<>(); 3956 try (InternalScanner scanner = region.getScanner(scan)) { 3957 boolean hasNext = scanner.next(actual); 3958 assertEquals(false, hasNext); 3959 3960 // Verify result 3961 for (int i = 0; i < expected.size(); i++) { 3962 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); 3963 } 3964 } 3965 } 3966 3967 @Test 3968 public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException { 3969 byte[] row1 = Bytes.toBytes("row1"); 3970 byte[] qf1 = Bytes.toBytes("qualifier1"); 3971 byte[] qf2 = Bytes.toBytes("qualifier2"); 3972 byte[] fam1 = Bytes.toBytes("fam1"); 3973 byte[][] families = { fam1 }; 3974 3975 long ts1 = EnvironmentEdgeManager.currentTime(); 3976 long ts2 = ts1 + 1; 3977 long ts3 = ts1 + 2; 3978 3979 // Setting up region 3980 this.region = initHRegion(tableName, method, CONF, families); 3981 // Putting data in Region 3982 Put put = null; 3983 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 3984 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 3985 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 3986 3987 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 3988 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 3989 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 3990 3991 put = new Put(row1); 3992 put.add(kv13); 3993 put.add(kv12); 3994 put.add(kv11); 3995 put.add(kv23); 3996 put.add(kv22); 3997 put.add(kv21); 3998 region.put(put); 3999 4000 // Expected 4001 List<Cell> expected = new ArrayList<>(); 4002 expected.add(kv13); 4003 expected.add(kv12); 4004 expected.add(kv23); 4005 expected.add(kv22); 4006 4007 Scan scan = new Scan().withStartRow(row1); 4008 scan.addFamily(fam1); 4009 scan.readVersions(MAX_VERSIONS); 4010 List<Cell> actual = new ArrayList<>(); 4011 try (InternalScanner scanner = region.getScanner(scan)) { 4012 boolean hasNext = scanner.next(actual); 4013 assertEquals(false, hasNext); 4014 4015 // Verify result 4016 for (int i = 0; i < expected.size(); i++) { 4017 assertEquals(expected.get(i), actual.get(i)); 4018 } 4019 } 4020 } 4021 4022 @Test 4023 public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException { 4024 byte[] row1 = Bytes.toBytes("row1"); 4025 byte[] qf1 = Bytes.toBytes("qualifier1"); 4026 byte[] qf2 = Bytes.toBytes("qualifier2"); 4027 byte[] fam1 = Bytes.toBytes("fam1"); 4028 4029 long ts1 = 1; 4030 long ts2 = ts1 + 1; 4031 long ts3 = ts1 + 2; 4032 4033 // Setting up region 4034 this.region = initHRegion(tableName, method, CONF, fam1); 4035 // Putting data in Region 4036 Put put = null; 4037 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 4038 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 4039 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 4040 4041 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 4042 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 4043 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 4044 4045 put = new Put(row1); 4046 put.add(kv13); 4047 put.add(kv12); 4048 put.add(kv11); 4049 put.add(kv23); 4050 put.add(kv22); 4051 put.add(kv21); 4052 region.put(put); 4053 region.flush(true); 4054 4055 // Expected 4056 List<ExtendedCell> expected = new ArrayList<>(); 4057 expected.add(kv13); 4058 expected.add(kv12); 4059 expected.add(kv23); 4060 expected.add(kv22); 4061 4062 Scan scan = new Scan().withStartRow(row1); 4063 scan.addFamily(fam1); 4064 scan.readVersions(MAX_VERSIONS); 4065 List<ExtendedCell> actual = new ArrayList<>(); 4066 try (InternalScanner scanner = region.getScanner(scan)) { 4067 boolean hasNext = scanner.next(actual); 4068 assertEquals(false, hasNext); 4069 4070 // Verify result 4071 for (int i = 0; i < expected.size(); i++) { 4072 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); 4073 } 4074 } 4075 } 4076 4077 @Test 4078 public void testScanner_StopRow1542() throws IOException { 4079 byte[] family = Bytes.toBytes("testFamily"); 4080 this.region = initHRegion(tableName, method, CONF, family); 4081 byte[] row1 = Bytes.toBytes("row111"); 4082 byte[] row2 = Bytes.toBytes("row222"); 4083 byte[] row3 = Bytes.toBytes("row333"); 4084 byte[] row4 = Bytes.toBytes("row444"); 4085 byte[] row5 = Bytes.toBytes("row555"); 4086 4087 byte[] col1 = Bytes.toBytes("Pub111"); 4088 byte[] col2 = Bytes.toBytes("Pub222"); 4089 4090 Put put = new Put(row1); 4091 put.addColumn(family, col1, Bytes.toBytes(10L)); 4092 region.put(put); 4093 4094 put = new Put(row2); 4095 put.addColumn(family, col1, Bytes.toBytes(15L)); 4096 region.put(put); 4097 4098 put = new Put(row3); 4099 put.addColumn(family, col2, Bytes.toBytes(20L)); 4100 region.put(put); 4101 4102 put = new Put(row4); 4103 put.addColumn(family, col2, Bytes.toBytes(30L)); 4104 region.put(put); 4105 4106 put = new Put(row5); 4107 put.addColumn(family, col1, Bytes.toBytes(40L)); 4108 region.put(put); 4109 4110 Scan scan = new Scan().withStartRow(row3).withStopRow(row4); 4111 scan.readAllVersions(); 4112 scan.addColumn(family, col1); 4113 try (InternalScanner s = region.getScanner(scan)) { 4114 List<Cell> results = new ArrayList<>(); 4115 assertEquals(false, s.next(results)); 4116 assertEquals(0, results.size()); 4117 } 4118 } 4119 4120 @Test 4121 public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException { 4122 byte[] row1 = Bytes.toBytes("row1"); 4123 byte[] fam1 = Bytes.toBytes("fam1"); 4124 byte[] qf1 = Bytes.toBytes("qualifier1"); 4125 byte[] qf2 = Bytes.toBytes("quateslifier2"); 4126 4127 long ts1 = 1; 4128 long ts2 = ts1 + 1; 4129 long ts3 = ts1 + 2; 4130 long ts4 = ts1 + 3; 4131 4132 // Setting up region 4133 this.region = initHRegion(tableName, method, CONF, fam1); 4134 // Putting data in Region 4135 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null); 4136 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 4137 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 4138 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 4139 4140 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null); 4141 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 4142 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 4143 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 4144 4145 Put put = null; 4146 put = new Put(row1); 4147 put.add(kv14); 4148 put.add(kv24); 4149 region.put(put); 4150 region.flush(true); 4151 4152 put = new Put(row1); 4153 put.add(kv23); 4154 put.add(kv13); 4155 region.put(put); 4156 region.flush(true); 4157 4158 put = new Put(row1); 4159 put.add(kv22); 4160 put.add(kv12); 4161 region.put(put); 4162 region.flush(true); 4163 4164 put = new Put(row1); 4165 put.add(kv21); 4166 put.add(kv11); 4167 region.put(put); 4168 4169 // Expected 4170 List<KeyValue> expected = new ArrayList<>(); 4171 expected.add(kv14); 4172 expected.add(kv13); 4173 expected.add(kv12); 4174 expected.add(kv24); 4175 expected.add(kv23); 4176 expected.add(kv22); 4177 4178 Scan scan = new Scan().withStartRow(row1); 4179 int versions = 3; 4180 scan.readVersions(versions); 4181 List<ExtendedCell> actual = new ArrayList<>(); 4182 try (InternalScanner scanner = region.getScanner(scan)) { 4183 boolean hasNext = scanner.next(actual); 4184 assertEquals(false, hasNext); 4185 4186 // Verify result 4187 for (int i = 0; i < expected.size(); i++) { 4188 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); 4189 } 4190 } 4191 } 4192 4193 /** 4194 * Added for HBASE-5416 Here we test scan optimization when only subset of CFs are used in filter 4195 * conditions. 4196 */ 4197 @Test 4198 public void testScanner_JoinedScanners() throws IOException { 4199 byte[] cf_essential = Bytes.toBytes("essential"); 4200 byte[] cf_joined = Bytes.toBytes("joined"); 4201 byte[] cf_alpha = Bytes.toBytes("alpha"); 4202 this.region = initHRegion(tableName, method, CONF, cf_essential, cf_joined, cf_alpha); 4203 byte[] row1 = Bytes.toBytes("row1"); 4204 byte[] row2 = Bytes.toBytes("row2"); 4205 byte[] row3 = Bytes.toBytes("row3"); 4206 4207 byte[] col_normal = Bytes.toBytes("d"); 4208 byte[] col_alpha = Bytes.toBytes("a"); 4209 4210 byte[] filtered_val = Bytes.toBytes(3); 4211 4212 Put put = new Put(row1); 4213 put.addColumn(cf_essential, col_normal, Bytes.toBytes(1)); 4214 put.addColumn(cf_joined, col_alpha, Bytes.toBytes(1)); 4215 region.put(put); 4216 4217 put = new Put(row2); 4218 put.addColumn(cf_essential, col_alpha, Bytes.toBytes(2)); 4219 put.addColumn(cf_joined, col_normal, Bytes.toBytes(2)); 4220 put.addColumn(cf_alpha, col_alpha, Bytes.toBytes(2)); 4221 region.put(put); 4222 4223 put = new Put(row3); 4224 put.addColumn(cf_essential, col_normal, filtered_val); 4225 put.addColumn(cf_joined, col_normal, filtered_val); 4226 region.put(put); 4227 4228 // Check two things: 4229 // 1. result list contains expected values 4230 // 2. result list is sorted properly 4231 4232 Scan scan = new Scan(); 4233 Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal, 4234 CompareOperator.NOT_EQUAL, filtered_val); 4235 scan.setFilter(filter); 4236 scan.setLoadColumnFamiliesOnDemand(true); 4237 try (InternalScanner s = region.getScanner(scan)) { 4238 List<Cell> results = new ArrayList<>(); 4239 assertTrue(s.next(results)); 4240 assertEquals(1, results.size()); 4241 results.clear(); 4242 4243 assertTrue(s.next(results)); 4244 assertEquals(3, results.size()); 4245 assertTrue(CellUtil.matchingFamily(results.get(0), cf_alpha), "orderCheck"); 4246 assertTrue(CellUtil.matchingFamily(results.get(1), cf_essential), "orderCheck"); 4247 assertTrue(CellUtil.matchingFamily(results.get(2), cf_joined), "orderCheck"); 4248 results.clear(); 4249 4250 assertFalse(s.next(results)); 4251 assertEquals(0, results.size()); 4252 } 4253 } 4254 4255 /** 4256 * HBASE-5416 Test case when scan limits amount of KVs returned on each next() call. 4257 */ 4258 @Test 4259 public void testScanner_JoinedScannersWithLimits() throws IOException { 4260 final byte[] cf_first = Bytes.toBytes("first"); 4261 final byte[] cf_second = Bytes.toBytes("second"); 4262 4263 this.region = initHRegion(tableName, method, CONF, cf_first, cf_second); 4264 final byte[] col_a = Bytes.toBytes("a"); 4265 final byte[] col_b = Bytes.toBytes("b"); 4266 4267 Put put; 4268 4269 for (int i = 0; i < 10; i++) { 4270 put = new Put(Bytes.toBytes("r" + Integer.toString(i))); 4271 put.addColumn(cf_first, col_a, Bytes.toBytes(i)); 4272 if (i < 5) { 4273 put.addColumn(cf_first, col_b, Bytes.toBytes(i)); 4274 put.addColumn(cf_second, col_a, Bytes.toBytes(i)); 4275 put.addColumn(cf_second, col_b, Bytes.toBytes(i)); 4276 } 4277 region.put(put); 4278 } 4279 4280 Scan scan = new Scan(); 4281 scan.setLoadColumnFamiliesOnDemand(true); 4282 Filter bogusFilter = new FilterBase() { 4283 @Override 4284 public ReturnCode filterCell(final Cell ignored) throws IOException { 4285 return ReturnCode.INCLUDE; 4286 } 4287 4288 @Override 4289 public boolean isFamilyEssential(byte[] name) { 4290 return Bytes.equals(name, cf_first); 4291 } 4292 }; 4293 4294 scan.setFilter(bogusFilter); 4295 try (InternalScanner s = region.getScanner(scan)) { 4296 // Our data looks like this: 4297 // r0: first:a, first:b, second:a, second:b 4298 // r1: first:a, first:b, second:a, second:b 4299 // r2: first:a, first:b, second:a, second:b 4300 // r3: first:a, first:b, second:a, second:b 4301 // r4: first:a, first:b, second:a, second:b 4302 // r5: first:a 4303 // r6: first:a 4304 // r7: first:a 4305 // r8: first:a 4306 // r9: first:a 4307 4308 // But due to next's limit set to 3, we should get this: 4309 // r0: first:a, first:b, second:a 4310 // r0: second:b 4311 // r1: first:a, first:b, second:a 4312 // r1: second:b 4313 // r2: first:a, first:b, second:a 4314 // r2: second:b 4315 // r3: first:a, first:b, second:a 4316 // r3: second:b 4317 // r4: first:a, first:b, second:a 4318 // r4: second:b 4319 // r5: first:a 4320 // r6: first:a 4321 // r7: first:a 4322 // r8: first:a 4323 // r9: first:a 4324 4325 List<Cell> results = new ArrayList<>(); 4326 int index = 0; 4327 ScannerContext scannerContext = ScannerContext.newBuilder().setBatchLimit(3).build(); 4328 while (true) { 4329 boolean more = s.next(results, scannerContext); 4330 if ((index >> 1) < 5) { 4331 if (index % 2 == 0) { 4332 assertEquals(3, results.size()); 4333 } else { 4334 assertEquals(1, results.size()); 4335 } 4336 } else { 4337 assertEquals(1, results.size()); 4338 } 4339 results.clear(); 4340 index++; 4341 if (!more) { 4342 break; 4343 } 4344 } 4345 } 4346 } 4347 4348 @Test 4349 public void testScannerOperationId() throws IOException { 4350 region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES); 4351 Scan scan = new Scan(); 4352 try (RegionScanner scanner = region.getScanner(scan)) { 4353 assertNull(scanner.getOperationId()); 4354 } 4355 4356 String operationId = "test_operation_id_0101"; 4357 scan = new Scan().setId(operationId); 4358 try (RegionScanner scanner = region.getScanner(scan)) { 4359 assertEquals(operationId, scanner.getOperationId()); 4360 } 4361 4362 HBaseTestingUtil.closeRegionAndWAL(this.region); 4363 } 4364 4365 /** 4366 * Write an HFile block full with Cells whose qualifier that are identical between 0 and 4367 * Short.MAX_VALUE. See HBASE-13329. 4368 */ 4369 @Test 4370 public void testLongQualifier() throws Exception { 4371 byte[] family = Bytes.toBytes("family"); 4372 this.region = initHRegion(tableName, method, CONF, family); 4373 byte[] q = new byte[Short.MAX_VALUE + 2]; 4374 Arrays.fill(q, 0, q.length - 1, (byte) 42); 4375 for (byte i = 0; i < 10; i++) { 4376 Put p = new Put(Bytes.toBytes("row")); 4377 // qualifiers that differ past Short.MAX_VALUE 4378 q[q.length - 1] = i; 4379 p.addColumn(family, q, q); 4380 region.put(p); 4381 } 4382 region.flush(false); 4383 } 4384 4385 @Test 4386 public void testCloseAndArchiveCompactedFiles() throws IOException { 4387 byte[] CF1 = Bytes.toBytes("CF1"); 4388 byte[] CF2 = Bytes.toBytes("CF2"); 4389 this.region = initHRegion(tableName, method, CONF, CF1, CF2); 4390 for (int i = 0; i < 2; i++) { 4391 int index = i; 4392 Put put = 4393 new Put(Bytes.toBytes(index)).addColumn(CF1, Bytes.toBytes("q"), Bytes.toBytes(index)); 4394 region.put(put); 4395 region.flush(true); 4396 } 4397 4398 region.compact(true); 4399 4400 HStore store1 = region.getStore(CF1); 4401 HStore store2 = region.getStore(CF2); 4402 store1.closeAndArchiveCompactedFiles(); 4403 store2.closeAndArchiveCompactedFiles(); 4404 4405 int storefilesCount = region.getStores().stream().mapToInt(Store::getStorefilesCount).sum(); 4406 assertTrue(storefilesCount == 1); 4407 4408 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 4409 Configuration conf = region.getReadOnlyConfiguration(); 4410 RegionInfo regionInfo = region.getRegionInfo(); 4411 Path store1ArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, CF1); 4412 assertTrue(fs.exists(store1ArchiveDir)); 4413 // The archived dir of CF2 does not exist because this column family has no data at all 4414 Path store2ArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, CF2); 4415 assertFalse(fs.exists(store2ArchiveDir)); 4416 } 4417 4418 @Test 4419 public void testHolesInMeta() throws Exception { 4420 byte[] family = Bytes.toBytes("family"); 4421 this.region = 4422 initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF, false, family); 4423 byte[] rowNotServed = Bytes.toBytes("a"); 4424 Get g = new Get(rowNotServed); 4425 try { 4426 region.get(g); 4427 fail(); 4428 } catch (WrongRegionException x) { 4429 // OK 4430 } 4431 byte[] row = Bytes.toBytes("y"); 4432 g = new Get(row); 4433 region.get(g); 4434 } 4435 4436 @Test 4437 public void testIndexesScanWithOneDeletedRow() throws IOException { 4438 byte[] family = Bytes.toBytes("family"); 4439 4440 // Setting up region 4441 this.region = initHRegion(tableName, method, CONF, family); 4442 Put put = new Put(Bytes.toBytes(1L)); 4443 put.addColumn(family, qual1, 1L, Bytes.toBytes(1L)); 4444 region.put(put); 4445 4446 region.flush(true); 4447 4448 Delete delete = new Delete(Bytes.toBytes(1L), 1L); 4449 region.delete(delete); 4450 4451 put = new Put(Bytes.toBytes(2L)); 4452 put.addColumn(family, qual1, 2L, Bytes.toBytes(2L)); 4453 region.put(put); 4454 4455 Scan idxScan = new Scan(); 4456 idxScan.addFamily(family); 4457 idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, 4458 Arrays.<Filter> asList( 4459 new SingleColumnValueFilter(family, qual1, CompareOperator.GREATER_OR_EQUAL, 4460 new BinaryComparator(Bytes.toBytes(0L))), 4461 new SingleColumnValueFilter(family, qual1, CompareOperator.LESS_OR_EQUAL, 4462 new BinaryComparator(Bytes.toBytes(3L)))))); 4463 try (InternalScanner scanner = region.getScanner(idxScan)) { 4464 List<Cell> res = new ArrayList<>(); 4465 4466 while (scanner.next(res)) { 4467 // Ignore res value. 4468 } 4469 assertEquals(1L, res.size()); 4470 } 4471 } 4472 4473 // //////////////////////////////////////////////////////////////////////////// 4474 // Bloom filter test 4475 // //////////////////////////////////////////////////////////////////////////// 4476 @Test 4477 public void testBloomFilterSize() throws IOException { 4478 byte[] fam1 = Bytes.toBytes("fam1"); 4479 byte[] qf1 = Bytes.toBytes("col"); 4480 byte[] val1 = Bytes.toBytes("value1"); 4481 // Create Table 4482 TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName) 4483 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam1) 4484 .setMaxVersions(Integer.MAX_VALUE).setBloomFilterType(BloomType.ROWCOL).build()) 4485 .build(); 4486 RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 4487 this.region = TEST_UTIL.createLocalHRegion(info, tableDescriptor); 4488 int num_unique_rows = 10; 4489 int duplicate_multiplier = 2; 4490 int num_storefiles = 4; 4491 4492 int version = 0; 4493 for (int f = 0; f < num_storefiles; f++) { 4494 for (int i = 0; i < duplicate_multiplier; i++) { 4495 for (int j = 0; j < num_unique_rows; j++) { 4496 Put put = new Put(Bytes.toBytes("row" + j)); 4497 put.setDurability(Durability.SKIP_WAL); 4498 long ts = version++; 4499 put.addColumn(fam1, qf1, ts, val1); 4500 region.put(put); 4501 } 4502 } 4503 region.flush(true); 4504 } 4505 // before compaction 4506 HStore store = region.getStore(fam1); 4507 Collection<HStoreFile> storeFiles = store.getStorefiles(); 4508 for (HStoreFile storefile : storeFiles) { 4509 StoreFileReader reader = storefile.getReader(); 4510 reader.loadFileInfo(); 4511 reader.loadBloomfilter(); 4512 assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries()); 4513 assertEquals(num_unique_rows, reader.getFilterEntries()); 4514 } 4515 4516 region.compact(true); 4517 4518 // after compaction 4519 storeFiles = store.getStorefiles(); 4520 for (HStoreFile storefile : storeFiles) { 4521 StoreFileReader reader = storefile.getReader(); 4522 reader.loadFileInfo(); 4523 reader.loadBloomfilter(); 4524 assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries()); 4525 assertEquals(num_unique_rows, reader.getFilterEntries()); 4526 } 4527 } 4528 4529 @Test 4530 public void testAllColumnsWithBloomFilter() throws IOException { 4531 byte[] TABLE = Bytes.toBytes(name); 4532 byte[] FAMILY = Bytes.toBytes("family"); 4533 4534 // Create table 4535 TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(TableName.valueOf(TABLE)) 4536 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY) 4537 .setMaxVersions(Integer.MAX_VALUE).setBloomFilterType(BloomType.ROWCOL).build()) 4538 .build(); 4539 RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 4540 this.region = TEST_UTIL.createLocalHRegion(info, tableDescriptor); 4541 // For row:0, col:0: insert versions 1 through 5. 4542 byte[] row = Bytes.toBytes("row:" + 0); 4543 byte[] column = Bytes.toBytes("column:" + 0); 4544 Put put = new Put(row); 4545 put.setDurability(Durability.SKIP_WAL); 4546 for (long idx = 1; idx <= 4; idx++) { 4547 put.addColumn(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx)); 4548 } 4549 region.put(put); 4550 4551 // Flush 4552 region.flush(true); 4553 4554 // Get rows 4555 Get get = new Get(row); 4556 get.readAllVersions(); 4557 Cell[] kvs = region.get(get).rawCells(); 4558 4559 // Check if rows are correct 4560 assertEquals(4, kvs.length); 4561 checkOneCell(kvs[0], FAMILY, 0, 0, 4); 4562 checkOneCell(kvs[1], FAMILY, 0, 0, 3); 4563 checkOneCell(kvs[2], FAMILY, 0, 0, 2); 4564 checkOneCell(kvs[3], FAMILY, 0, 0, 1); 4565 } 4566 4567 /** 4568 * Testcase to cover bug-fix for HBASE-2823 Ensures correct delete when issuing delete row on 4569 * columns with bloom filter set to row+col (BloomType.ROWCOL) 4570 */ 4571 @Test 4572 public void testDeleteRowWithBloomFilter() throws IOException { 4573 byte[] familyName = Bytes.toBytes("familyName"); 4574 4575 // Create Table 4576 TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName) 4577 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(familyName) 4578 .setMaxVersions(Integer.MAX_VALUE).setBloomFilterType(BloomType.ROWCOL).build()) 4579 .build(); 4580 RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 4581 this.region = TEST_UTIL.createLocalHRegion(info, tableDescriptor); 4582 // Insert some data 4583 byte[] row = Bytes.toBytes("row1"); 4584 byte[] col = Bytes.toBytes("col1"); 4585 4586 Put put = new Put(row); 4587 put.addColumn(familyName, col, 1, Bytes.toBytes("SomeRandomValue")); 4588 region.put(put); 4589 region.flush(true); 4590 4591 Delete del = new Delete(row); 4592 region.delete(del); 4593 region.flush(true); 4594 4595 // Get remaining rows (should have none) 4596 Get get = new Get(row); 4597 get.addColumn(familyName, col); 4598 4599 Cell[] keyValues = region.get(get).rawCells(); 4600 assertEquals(0, keyValues.length); 4601 } 4602 4603 @Test 4604 public void testgetHDFSBlocksDistribution() throws Exception { 4605 HBaseTestingUtil htu = new HBaseTestingUtil(); 4606 // Why do we set the block size in this test? If we set it smaller than the kvs, then we'll 4607 // break up the file in to more pieces that can be distributed across the three nodes and we 4608 // won't be able to have the condition this test asserts; that at least one node has 4609 // a copy of all replicas -- if small block size, then blocks are spread evenly across the 4610 // the three nodes. hfilev3 with tags seems to put us over the block size. St.Ack. 4611 // final int DEFAULT_BLOCK_SIZE = 1024; 4612 // htu.getConfiguration().setLong("dfs.blocksize", DEFAULT_BLOCK_SIZE); 4613 htu.getConfiguration().setInt("dfs.replication", 2); 4614 4615 // set up a cluster with 3 nodes 4616 SingleProcessHBaseCluster cluster = null; 4617 String dataNodeHosts[] = new String[] { "host1", "host2", "host3" }; 4618 int regionServersCount = 3; 4619 4620 try { 4621 StartTestingClusterOption option = StartTestingClusterOption.builder() 4622 .numRegionServers(regionServersCount).dataNodeHosts(dataNodeHosts).build(); 4623 cluster = htu.startMiniCluster(option); 4624 byte[][] families = { fam1, fam2 }; 4625 Table ht = htu.createTable(tableName, families); 4626 4627 // Setting up region 4628 byte row[] = Bytes.toBytes("row1"); 4629 byte col[] = Bytes.toBytes("col1"); 4630 4631 Put put = new Put(row); 4632 put.addColumn(fam1, col, 1, Bytes.toBytes("test1")); 4633 put.addColumn(fam2, col, 1, Bytes.toBytes("test2")); 4634 ht.put(put); 4635 4636 HRegion firstRegion = htu.getHBaseCluster().getRegions(tableName).get(0); 4637 firstRegion.flush(true); 4638 HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution(); 4639 4640 // Given the default replication factor is 2 and we have 2 HFiles, 4641 // we will have total of 4 replica of blocks on 3 datanodes; thus there 4642 // must be at least one host that have replica for 2 HFiles. That host's 4643 // weight will be equal to the unique block weight. 4644 long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight(); 4645 StringBuilder sb = new StringBuilder(); 4646 for (String host : blocksDistribution1.getTopHosts()) { 4647 if (sb.length() > 0) sb.append(", "); 4648 sb.append(host); 4649 sb.append("="); 4650 sb.append(blocksDistribution1.getWeight(host)); 4651 } 4652 4653 String topHost = blocksDistribution1.getTopHosts().get(0); 4654 long topHostWeight = blocksDistribution1.getWeight(topHost); 4655 String msg = "uniqueBlocksWeight=" + uniqueBlocksWeight1 + ", topHostWeight=" + topHostWeight 4656 + ", topHost=" + topHost + "; " + sb.toString(); 4657 LOG.info(msg); 4658 assertTrue(uniqueBlocksWeight1 == topHostWeight, msg); 4659 4660 // use the static method to compute the value, it should be the same. 4661 // static method is used by load balancer or other components 4662 HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution( 4663 htu.getConfiguration(), firstRegion.getTableDescriptor(), firstRegion.getRegionInfo()); 4664 long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight(); 4665 4666 assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2); 4667 4668 ht.close(); 4669 } finally { 4670 if (cluster != null) { 4671 htu.shutdownMiniCluster(); 4672 } 4673 } 4674 } 4675 4676 /** 4677 * Testcase to check state of region initialization task set to ABORTED or not if any exceptions 4678 * during initialization 4679 */ 4680 @Test 4681 public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception { 4682 RegionInfo info; 4683 try { 4684 FileSystem fs = mock(FileSystem.class); 4685 when(fs.exists(any())).thenThrow(new IOException()); 4686 TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(tableName); 4687 ColumnFamilyDescriptor columnFamilyDescriptor = 4688 ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf")).build(); 4689 tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor); 4690 info = RegionInfoBuilder.newBuilder(tableName).build(); 4691 Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization"); 4692 region = HRegion.newHRegion(path, null, fs, CONF, info, tableDescriptorBuilder.build(), null); 4693 // region initialization throws IOException and set task state to ABORTED. 4694 region.initialize(); 4695 fail("Region initialization should fail due to IOException"); 4696 } catch (IOException io) { 4697 List<MonitoredTask> tasks = TaskMonitor.get().getTasks(); 4698 for (MonitoredTask monitoredTask : tasks) { 4699 if ( 4700 !(monitoredTask instanceof MonitoredRPCHandler) 4701 && monitoredTask.getDescription().contains(region.toString()) 4702 ) { 4703 assertTrue(monitoredTask.getState().equals(MonitoredTask.State.ABORTED), 4704 "Region state should be ABORTED."); 4705 break; 4706 } 4707 } 4708 } 4709 } 4710 4711 /** 4712 * Verifies that the .regioninfo file is written on region creation and that is recreated if 4713 * missing during region opening. 4714 */ 4715 @Test 4716 public void testRegionInfoFileCreation() throws IOException { 4717 Path rootDir = new Path(dir + "testRegionInfoFileCreation"); 4718 4719 TableDescriptorBuilder tableDescriptorBuilder = 4720 TableDescriptorBuilder.newBuilder(TableName.valueOf(name)); 4721 ColumnFamilyDescriptor columnFamilyDescriptor = 4722 ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf")).build(); 4723 tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor); 4724 TableDescriptor tableDescriptor = tableDescriptorBuilder.build(); 4725 4726 RegionInfo hri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 4727 4728 // Create a region and skip the initialization (like CreateTableHandler) 4729 region = HBaseTestingUtil.createRegionAndWAL(hri, rootDir, CONF, tableDescriptor, false); 4730 Path regionDir = region.getRegionFileSystem().getRegionDir(); 4731 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 4732 HBaseTestingUtil.closeRegionAndWAL(region); 4733 4734 Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE); 4735 4736 // Verify that the .regioninfo file is present 4737 assertTrue(fs.exists(regionInfoFile), 4738 HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir"); 4739 4740 // Try to open the region 4741 region = HRegion.openHRegion(rootDir, hri, tableDescriptor, null, CONF); 4742 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir()); 4743 HBaseTestingUtil.closeRegionAndWAL(region); 4744 4745 // Verify that the .regioninfo file is still there 4746 assertTrue(fs.exists(regionInfoFile), 4747 HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir"); 4748 4749 // Remove the .regioninfo file and verify is recreated on region open 4750 fs.delete(regionInfoFile, true); 4751 assertFalse(fs.exists(regionInfoFile), 4752 HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir"); 4753 4754 region = HRegion.openHRegion(rootDir, hri, tableDescriptor, null, CONF); 4755 // region = TEST_UTIL.openHRegion(hri, htd); 4756 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir()); 4757 HBaseTestingUtil.closeRegionAndWAL(region); 4758 4759 // Verify that the .regioninfo file is still there 4760 assertTrue(fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)), 4761 HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir"); 4762 4763 region = null; 4764 } 4765 4766 /** 4767 * Test case to check put function with memstore flushing for same row, same ts 4768 */ 4769 @Test 4770 public void testPutWithMemStoreFlush() throws Exception { 4771 byte[] family = Bytes.toBytes("family"); 4772 byte[] qualifier = Bytes.toBytes("qualifier"); 4773 byte[] row = Bytes.toBytes("putRow"); 4774 byte[] value = null; 4775 this.region = initHRegion(tableName, method, CONF, family); 4776 Put put = null; 4777 Get get = null; 4778 List<Cell> kvs = null; 4779 Result res = null; 4780 4781 put = new Put(row); 4782 value = Bytes.toBytes("value0"); 4783 put.addColumn(family, qualifier, 1234567L, value); 4784 region.put(put); 4785 get = new Get(row); 4786 get.addColumn(family, qualifier); 4787 get.readAllVersions(); 4788 res = this.region.get(get); 4789 kvs = res.getColumnCells(family, qualifier); 4790 assertEquals(1, kvs.size()); 4791 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0))); 4792 4793 region.flush(true); 4794 get = new Get(row); 4795 get.addColumn(family, qualifier); 4796 get.readAllVersions(); 4797 res = this.region.get(get); 4798 kvs = res.getColumnCells(family, qualifier); 4799 assertEquals(1, kvs.size()); 4800 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0))); 4801 4802 put = new Put(row); 4803 value = Bytes.toBytes("value1"); 4804 put.addColumn(family, qualifier, 1234567L, value); 4805 region.put(put); 4806 get = new Get(row); 4807 get.addColumn(family, qualifier); 4808 get.readAllVersions(); 4809 res = this.region.get(get); 4810 kvs = res.getColumnCells(family, qualifier); 4811 assertEquals(1, kvs.size()); 4812 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0))); 4813 4814 region.flush(true); 4815 get = new Get(row); 4816 get.addColumn(family, qualifier); 4817 get.readAllVersions(); 4818 res = this.region.get(get); 4819 kvs = res.getColumnCells(family, qualifier); 4820 assertEquals(1, kvs.size()); 4821 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0))); 4822 } 4823 4824 /** 4825 * For this test,the spied {@link AsyncFSWAL} can not work properly because of a Mockito defect 4826 * that can not deal with classes which have a field of an inner class. See discussions in 4827 * HBASE-15536.When we reuse the code of {@link AsyncFSWAL} for {@link FSHLog}, this test could 4828 * not work for {@link FSHLog} also. 4829 */ 4830 @Test 4831 @Disabled 4832 public void testDurability() throws Exception { 4833 // there are 5 x 5 cases: 4834 // table durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT) x mutation 4835 // durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT) 4836 4837 // expected cases for append and sync wal 4838 durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false); 4839 durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false); 4840 durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false); 4841 4842 durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false); 4843 durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false); 4844 durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false); 4845 4846 durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false); 4847 durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false); 4848 4849 durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false); 4850 durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false); 4851 4852 durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false); 4853 durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false); 4854 durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false); 4855 4856 // expected cases for async wal 4857 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false); 4858 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false); 4859 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false); 4860 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false); 4861 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false); 4862 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false); 4863 4864 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true); 4865 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true); 4866 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true); 4867 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true); 4868 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true); 4869 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true); 4870 4871 // expect skip wal cases 4872 durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false); 4873 durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false); 4874 durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false); 4875 durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false); 4876 durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false); 4877 durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false); 4878 4879 } 4880 4881 private void durabilityTest(String method, Durability tableDurability, 4882 Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync, 4883 final boolean expectSyncFromLogSyncer) throws Exception { 4884 Configuration conf = HBaseConfiguration.create(CONF); 4885 conf.setLong(AbstractFSWAL.WAL_SHUTDOWN_WAIT_TIMEOUT_MS, 60 * 60 * 1000); 4886 method = method + "_" + tableDurability.name() + "_" + mutationDurability.name(); 4887 byte[] family = Bytes.toBytes("family"); 4888 Path logDir = new Path(new Path(dir + method), "log"); 4889 final Configuration walConf = new Configuration(conf); 4890 CommonFSUtils.setRootDir(walConf, logDir); 4891 // XXX: The spied AsyncFSWAL can not work properly because of a Mockito defect that can not 4892 // deal with classes which have a field of an inner class. See discussions in HBASE-15536. 4893 walConf.set(WALFactory.WAL_PROVIDER, "filesystem"); 4894 final WALFactory wals = new WALFactory(walConf, HBaseTestingUtil.getRandomUUID().toString()); 4895 final WAL wal = spy(wals.getWAL(RegionInfoBuilder.newBuilder(tableName).build())); 4896 this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF, 4897 false, tableDurability, wal, new byte[][] { family }); 4898 4899 Put put = new Put(Bytes.toBytes("r1")); 4900 put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); 4901 put.setDurability(mutationDurability); 4902 region.put(put); 4903 4904 // verify append called or not 4905 verify(wal, expectAppend ? times(1) : never()).appendData((RegionInfo) any(), 4906 (WALKeyImpl) any(), (WALEdit) any()); 4907 4908 // verify sync called or not 4909 if (expectSync || expectSyncFromLogSyncer) { 4910 TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() { 4911 @Override 4912 public boolean evaluate() throws Exception { 4913 try { 4914 if (expectSync) { 4915 verify(wal, times(1)).sync(anyLong()); // Hregion calls this one 4916 } else if (expectSyncFromLogSyncer) { 4917 verify(wal, times(1)).sync(); // wal syncer calls this one 4918 } 4919 } catch (Throwable ignore) { 4920 } 4921 return true; 4922 } 4923 }); 4924 } else { 4925 // verify(wal, never()).sync(anyLong()); 4926 verify(wal, never()).sync(); 4927 } 4928 4929 HBaseTestingUtil.closeRegionAndWAL(this.region); 4930 wals.close(); 4931 this.region = null; 4932 } 4933 4934 @Test 4935 public void testRegionReplicaSecondary() throws IOException { 4936 // create a primary region, load some data and flush 4937 // create a secondary region, and do a get against that 4938 Path rootDir = new Path(dir + name); 4939 CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir); 4940 4941 byte[][] families = 4942 new byte[][] { Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3") }; 4943 byte[] cq = Bytes.toBytes("cq"); 4944 TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)); 4945 for (byte[] family : families) { 4946 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)); 4947 } 4948 TableDescriptor tableDescriptor = builder.build(); 4949 long time = EnvironmentEdgeManager.currentTime(); 4950 RegionInfo primaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 4951 .setRegionId(time).setReplicaId(0).build(); 4952 RegionInfo secondaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 4953 .setRegionId(time).setReplicaId(1).build(); 4954 4955 HRegion primaryRegion = null, secondaryRegion = null; 4956 4957 try { 4958 primaryRegion = HBaseTestingUtil.createRegionAndWAL(primaryHri, rootDir, 4959 TEST_UTIL.getConfiguration(), tableDescriptor); 4960 4961 // load some data 4962 putData(primaryRegion, 0, 1000, cq, families); 4963 4964 // flush region 4965 primaryRegion.flush(true); 4966 4967 // open secondary region 4968 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, tableDescriptor, null, CONF); 4969 4970 verifyData(secondaryRegion, 0, 1000, cq, families); 4971 } finally { 4972 if (primaryRegion != null) { 4973 HBaseTestingUtil.closeRegionAndWAL(primaryRegion); 4974 } 4975 if (secondaryRegion != null) { 4976 HBaseTestingUtil.closeRegionAndWAL(secondaryRegion); 4977 } 4978 } 4979 } 4980 4981 @Test 4982 public void testRegionReplicaSecondaryIsReadOnly() throws IOException { 4983 // create a primary region, load some data and flush 4984 // create a secondary region, and do a put against that 4985 Path rootDir = new Path(dir + name); 4986 CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir); 4987 4988 byte[][] families = 4989 new byte[][] { Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3") }; 4990 byte[] cq = Bytes.toBytes("cq"); 4991 TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)); 4992 for (byte[] family : families) { 4993 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)); 4994 } 4995 TableDescriptor tableDescriptor = builder.build(); 4996 long time = EnvironmentEdgeManager.currentTime(); 4997 RegionInfo primaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 4998 .setRegionId(time).setReplicaId(0).build(); 4999 RegionInfo secondaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5000 .setRegionId(time).setReplicaId(1).build(); 5001 5002 HRegion primaryRegion = null, secondaryRegion = null; 5003 5004 try { 5005 primaryRegion = HBaseTestingUtil.createRegionAndWAL(primaryHri, rootDir, 5006 TEST_UTIL.getConfiguration(), tableDescriptor); 5007 5008 // load some data 5009 putData(primaryRegion, 0, 1000, cq, families); 5010 5011 // flush region 5012 primaryRegion.flush(true); 5013 5014 // open secondary region 5015 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, tableDescriptor, null, CONF); 5016 5017 try { 5018 putData(secondaryRegion, 0, 1000, cq, families); 5019 fail("Should have thrown exception"); 5020 } catch (IOException ex) { 5021 // expected 5022 } 5023 } finally { 5024 if (primaryRegion != null) { 5025 HBaseTestingUtil.closeRegionAndWAL(primaryRegion); 5026 } 5027 if (secondaryRegion != null) { 5028 HBaseTestingUtil.closeRegionAndWAL(secondaryRegion); 5029 } 5030 } 5031 } 5032 5033 static WALFactory createWALFactory(Configuration conf, Path rootDir) throws IOException { 5034 Configuration confForWAL = new Configuration(conf); 5035 confForWAL.set(HConstants.HBASE_DIR, rootDir.toString()); 5036 return new WALFactory(confForWAL, "hregion-" + RandomStringUtils.randomNumeric(8)); 5037 } 5038 5039 @Test 5040 public void testCompactionFromPrimary() throws IOException { 5041 Path rootDir = new Path(dir + name); 5042 CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir); 5043 5044 byte[][] families = 5045 new byte[][] { Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3") }; 5046 byte[] cq = Bytes.toBytes("cq"); 5047 TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)); 5048 for (byte[] family : families) { 5049 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)); 5050 } 5051 TableDescriptor tableDescriptor = builder.build(); 5052 long time = EnvironmentEdgeManager.currentTime(); 5053 RegionInfo primaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5054 .setRegionId(time).setReplicaId(0).build(); 5055 RegionInfo secondaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5056 .setRegionId(time).setReplicaId(1).build(); 5057 5058 HRegion primaryRegion = null, secondaryRegion = null; 5059 5060 try { 5061 primaryRegion = HBaseTestingUtil.createRegionAndWAL(primaryHri, rootDir, 5062 TEST_UTIL.getConfiguration(), tableDescriptor); 5063 5064 // load some data 5065 putData(primaryRegion, 0, 1000, cq, families); 5066 5067 // flush region 5068 primaryRegion.flush(true); 5069 5070 // open secondary region 5071 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, tableDescriptor, null, CONF); 5072 5073 // move the file of the primary region to the archive, simulating a compaction 5074 Collection<HStoreFile> storeFiles = primaryRegion.getStore(families[0]).getStorefiles(); 5075 HRegionFileSystem regionFs = primaryRegion.getRegionFileSystem(); 5076 StoreFileTracker sft = StoreFileTrackerFactory.create(primaryRegion.getBaseConf(), false, 5077 StoreContext.getBuilder() 5078 .withColumnFamilyDescriptor(ColumnFamilyDescriptorBuilder.newBuilder(families[0]).build()) 5079 .withFamilyStoreDirectoryPath( 5080 new Path(regionFs.getRegionDir(), Bytes.toString(families[0]))) 5081 .withRegionFileSystem(regionFs).build()); 5082 sft.removeStoreFiles(storeFiles.stream().collect(Collectors.toList())); 5083 Collection<StoreFileInfo> storeFileInfos = sft.load(); 5084 assertTrue(storeFileInfos == null || storeFileInfos.isEmpty()); 5085 5086 verifyData(secondaryRegion, 0, 1000, cq, families); 5087 } finally { 5088 if (primaryRegion != null) { 5089 HBaseTestingUtil.closeRegionAndWAL(primaryRegion); 5090 } 5091 if (secondaryRegion != null) { 5092 HBaseTestingUtil.closeRegionAndWAL(secondaryRegion); 5093 } 5094 } 5095 } 5096 5097 private void putData(int startRow, int numRows, byte[] qf, byte[]... families) 5098 throws IOException { 5099 putData(this.region, startRow, numRows, qf, families); 5100 } 5101 5102 private void putData(HRegion region, int startRow, int numRows, byte[] qf, byte[]... families) 5103 throws IOException { 5104 putData(region, Durability.SKIP_WAL, startRow, numRows, qf, families); 5105 } 5106 5107 static void putData(HRegion region, Durability durability, int startRow, int numRows, byte[] qf, 5108 byte[]... families) throws IOException { 5109 for (int i = startRow; i < startRow + numRows; i++) { 5110 Put put = new Put(Bytes.toBytes("" + i)); 5111 put.setDurability(durability); 5112 for (byte[] family : families) { 5113 put.addColumn(family, qf, null); 5114 } 5115 region.put(put); 5116 LOG.info(put.toString()); 5117 } 5118 } 5119 5120 static void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families) 5121 throws IOException { 5122 for (int i = startRow; i < startRow + numRows; i++) { 5123 byte[] row = Bytes.toBytes("" + i); 5124 Get get = new Get(row); 5125 for (byte[] family : families) { 5126 get.addColumn(family, qf); 5127 } 5128 Result result = newReg.get(get); 5129 Cell[] raw = result.rawCells(); 5130 assertEquals(families.length, result.size()); 5131 for (int j = 0; j < families.length; j++) { 5132 assertTrue(CellUtil.matchingRows(raw[j], row)); 5133 assertTrue(CellUtil.matchingFamily(raw[j], families[j])); 5134 assertTrue(CellUtil.matchingQualifier(raw[j], qf)); 5135 } 5136 } 5137 } 5138 5139 static void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException { 5140 // Now I have k, get values out and assert they are as expected. 5141 Get get = new Get(k).addFamily(family).readAllVersions(); 5142 Cell[] results = r.get(get).rawCells(); 5143 for (int j = 0; j < results.length; j++) { 5144 byte[] tmp = CellUtil.cloneValue(results[j]); 5145 // Row should be equal to value every time. 5146 assertTrue(Bytes.equals(k, tmp)); 5147 } 5148 } 5149 5150 /* 5151 * Assert first value in the passed region is <code>firstValue</code>. 5152 */ 5153 protected void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue) 5154 throws IOException { 5155 byte[][] families = { fs }; 5156 Scan scan = new Scan(); 5157 for (int i = 0; i < families.length; i++) 5158 scan.addFamily(families[i]); 5159 try (InternalScanner s = r.getScanner(scan)) { 5160 List<Cell> curVals = new ArrayList<>(); 5161 boolean first = true; 5162 OUTER_LOOP: while (s.next(curVals)) { 5163 for (Cell kv : curVals) { 5164 byte[] val = CellUtil.cloneValue(kv); 5165 byte[] curval = val; 5166 if (first) { 5167 first = false; 5168 assertTrue(Bytes.compareTo(curval, firstValue) == 0); 5169 } else { 5170 // Not asserting anything. Might as well break. 5171 break OUTER_LOOP; 5172 } 5173 } 5174 } 5175 } 5176 } 5177 5178 /** 5179 * Test that we get the expected flush results back 5180 */ 5181 @Test 5182 public void testFlushResult() throws IOException { 5183 byte[] family = Bytes.toBytes("family"); 5184 5185 this.region = initHRegion(tableName, method, family); 5186 5187 // empty memstore, flush doesn't run 5188 HRegion.FlushResult fr = region.flush(true); 5189 assertFalse(fr.isFlushSucceeded()); 5190 assertFalse(fr.isCompactionNeeded()); 5191 5192 // Flush enough files to get up to the threshold, doesn't need compactions 5193 for (int i = 0; i < 2; i++) { 5194 Put put = new Put(tableName.toBytes()).addColumn(family, family, tableName.toBytes()); 5195 region.put(put); 5196 fr = region.flush(true); 5197 assertTrue(fr.isFlushSucceeded()); 5198 assertFalse(fr.isCompactionNeeded()); 5199 } 5200 5201 // Two flushes after the threshold, compactions are needed 5202 for (int i = 0; i < 2; i++) { 5203 Put put = new Put(tableName.toBytes()).addColumn(family, family, tableName.toBytes()); 5204 region.put(put); 5205 fr = region.flush(true); 5206 assertTrue(fr.isFlushSucceeded()); 5207 assertTrue(fr.isCompactionNeeded()); 5208 } 5209 } 5210 5211 protected Configuration initSplit() { 5212 // Always compact if there is more than one store file. 5213 CONF.setInt("hbase.hstore.compactionThreshold", 2); 5214 5215 CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000); 5216 5217 // Increase the amount of time between client retries 5218 CONF.setLong("hbase.client.pause", 15 * 1000); 5219 5220 // This size should make it so we always split using the addContent 5221 // below. After adding all data, the first region is 1.3M 5222 CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128); 5223 return CONF; 5224 } 5225 5226 /** 5227 * @return A region on which you must call {@link HBaseTestingUtil#closeRegionAndWAL(HRegion)} 5228 * when done. 5229 */ 5230 protected HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf, 5231 byte[]... families) throws IOException { 5232 return initHRegion(tableName, callingMethod, conf, false, families); 5233 } 5234 5235 /** 5236 * @return A region on which you must call {@link HBaseTestingUtil#closeRegionAndWAL(HRegion)} 5237 * when done. 5238 */ 5239 private HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf, 5240 boolean isReadOnly, byte[]... families) throws IOException { 5241 return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families); 5242 } 5243 5244 private HRegion initHRegion(TableName tableName, byte[] startKey, byte[] stopKey, 5245 String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families) 5246 throws IOException { 5247 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(callingMethod + ".log"); 5248 RegionInfo hri = 5249 RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey).setEndKey(stopKey).build(); 5250 final WAL wal = HBaseTestingUtil.createWal(conf, logDir, hri); 5251 return initHRegion(tableName, startKey, stopKey, conf, isReadOnly, Durability.SYNC_WAL, wal, 5252 families); 5253 } 5254 5255 /** 5256 * @return A region on which you must call {@link HBaseTestingUtil#closeRegionAndWAL(HRegion)} 5257 * when done. 5258 */ 5259 protected HRegion initHRegion(TableName tableName, byte[] startKey, byte[] stopKey, 5260 Configuration conf, boolean isReadOnly, Durability durability, WAL wal, byte[]... families) 5261 throws IOException { 5262 ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null, 5263 MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT); 5264 return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, conf, isReadOnly, durability, 5265 wal, families); 5266 } 5267 5268 /** 5269 * Assert that the passed in Cell has expected contents for the specified row, column & timestamp. 5270 */ 5271 private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) { 5272 String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts; 5273 assertEquals("row:" + rowIdx, Bytes.toString(CellUtil.cloneRow(kv)), 5274 "Row mismatch which checking: " + ctx); 5275 assertEquals(Bytes.toString(cf), Bytes.toString(CellUtil.cloneFamily(kv)), 5276 "ColumnFamily mismatch while checking: " + ctx); 5277 assertEquals("column:" + colIdx, Bytes.toString(CellUtil.cloneQualifier(kv)), 5278 "Column qualifier mismatch while checking: " + ctx); 5279 assertEquals(ts, kv.getTimestamp(), "Timestamp mismatch while checking: " + ctx); 5280 assertEquals("value-version-" + ts, Bytes.toString(CellUtil.cloneValue(kv)), 5281 "Value mismatch while checking: " + ctx); 5282 } 5283 5284 @Test 5285 public void testReverseScanner_FromMemStore_SingleCF_Normal() throws IOException { 5286 byte[] rowC = Bytes.toBytes("rowC"); 5287 byte[] rowA = Bytes.toBytes("rowA"); 5288 byte[] rowB = Bytes.toBytes("rowB"); 5289 byte[] cf = Bytes.toBytes("CF"); 5290 byte[][] families = { cf }; 5291 byte[] col = Bytes.toBytes("C"); 5292 long ts = 1; 5293 this.region = initHRegion(tableName, method, families); 5294 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null); 5295 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, null); 5296 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null); 5297 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null); 5298 Put put = null; 5299 put = new Put(rowC); 5300 put.add(kv1); 5301 put.add(kv11); 5302 region.put(put); 5303 put = new Put(rowA); 5304 put.add(kv2); 5305 region.put(put); 5306 put = new Put(rowB); 5307 put.add(kv3); 5308 region.put(put); 5309 5310 Scan scan = new Scan().withStartRow(rowC); 5311 scan.readVersions(5); 5312 scan.setReversed(true); 5313 try (InternalScanner scanner = region.getScanner(scan)) { 5314 List<Cell> currRow = new ArrayList<>(); 5315 boolean hasNext = scanner.next(currRow); 5316 assertEquals(2, currRow.size()); 5317 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5318 currRow.get(0).getRowLength(), rowC, 0, rowC.length)); 5319 assertTrue(hasNext); 5320 currRow.clear(); 5321 hasNext = scanner.next(currRow); 5322 assertEquals(1, currRow.size()); 5323 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5324 currRow.get(0).getRowLength(), rowB, 0, rowB.length)); 5325 assertTrue(hasNext); 5326 currRow.clear(); 5327 hasNext = scanner.next(currRow); 5328 assertEquals(1, currRow.size()); 5329 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5330 currRow.get(0).getRowLength(), rowA, 0, rowA.length)); 5331 assertFalse(hasNext); 5332 } 5333 } 5334 5335 @Test 5336 public void testReverseScanner_FromMemStore_SingleCF_LargerKey() throws IOException { 5337 byte[] rowC = Bytes.toBytes("rowC"); 5338 byte[] rowA = Bytes.toBytes("rowA"); 5339 byte[] rowB = Bytes.toBytes("rowB"); 5340 byte[] rowD = Bytes.toBytes("rowD"); 5341 byte[] cf = Bytes.toBytes("CF"); 5342 byte[][] families = { cf }; 5343 byte[] col = Bytes.toBytes("C"); 5344 long ts = 1; 5345 this.region = initHRegion(tableName, method, families); 5346 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null); 5347 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, null); 5348 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null); 5349 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null); 5350 Put put = null; 5351 put = new Put(rowC); 5352 put.add(kv1); 5353 put.add(kv11); 5354 region.put(put); 5355 put = new Put(rowA); 5356 put.add(kv2); 5357 region.put(put); 5358 put = new Put(rowB); 5359 put.add(kv3); 5360 region.put(put); 5361 5362 Scan scan = new Scan().withStartRow(rowD); 5363 List<Cell> currRow = new ArrayList<>(); 5364 scan.setReversed(true); 5365 scan.readVersions(5); 5366 try (InternalScanner scanner = region.getScanner(scan)) { 5367 boolean hasNext = scanner.next(currRow); 5368 assertEquals(2, currRow.size()); 5369 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5370 currRow.get(0).getRowLength(), rowC, 0, rowC.length)); 5371 assertTrue(hasNext); 5372 currRow.clear(); 5373 hasNext = scanner.next(currRow); 5374 assertEquals(1, currRow.size()); 5375 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5376 currRow.get(0).getRowLength(), rowB, 0, rowB.length)); 5377 assertTrue(hasNext); 5378 currRow.clear(); 5379 hasNext = scanner.next(currRow); 5380 assertEquals(1, currRow.size()); 5381 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5382 currRow.get(0).getRowLength(), rowA, 0, rowA.length)); 5383 assertFalse(hasNext); 5384 } 5385 } 5386 5387 @Test 5388 public void testReverseScanner_FromMemStore_SingleCF_FullScan() throws IOException { 5389 byte[] rowC = Bytes.toBytes("rowC"); 5390 byte[] rowA = Bytes.toBytes("rowA"); 5391 byte[] rowB = Bytes.toBytes("rowB"); 5392 byte[] cf = Bytes.toBytes("CF"); 5393 byte[][] families = { cf }; 5394 byte[] col = Bytes.toBytes("C"); 5395 long ts = 1; 5396 this.region = initHRegion(tableName, method, families); 5397 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null); 5398 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, null); 5399 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null); 5400 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null); 5401 Put put = null; 5402 put = new Put(rowC); 5403 put.add(kv1); 5404 put.add(kv11); 5405 region.put(put); 5406 put = new Put(rowA); 5407 put.add(kv2); 5408 region.put(put); 5409 put = new Put(rowB); 5410 put.add(kv3); 5411 region.put(put); 5412 Scan scan = new Scan(); 5413 List<Cell> currRow = new ArrayList<>(); 5414 scan.setReversed(true); 5415 try (InternalScanner scanner = region.getScanner(scan)) { 5416 boolean hasNext = scanner.next(currRow); 5417 assertEquals(1, currRow.size()); 5418 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5419 currRow.get(0).getRowLength(), rowC, 0, rowC.length)); 5420 assertTrue(hasNext); 5421 currRow.clear(); 5422 hasNext = scanner.next(currRow); 5423 assertEquals(1, currRow.size()); 5424 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5425 currRow.get(0).getRowLength(), rowB, 0, rowB.length)); 5426 assertTrue(hasNext); 5427 currRow.clear(); 5428 hasNext = scanner.next(currRow); 5429 assertEquals(1, currRow.size()); 5430 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5431 currRow.get(0).getRowLength(), rowA, 0, rowA.length)); 5432 assertFalse(hasNext); 5433 } 5434 } 5435 5436 @Test 5437 public void testReverseScanner_moreRowsMayExistAfter() throws IOException { 5438 // case for "INCLUDE_AND_SEEK_NEXT_ROW & SEEK_NEXT_ROW" endless loop 5439 byte[] rowA = Bytes.toBytes("rowA"); 5440 byte[] rowB = Bytes.toBytes("rowB"); 5441 byte[] rowC = Bytes.toBytes("rowC"); 5442 byte[] rowD = Bytes.toBytes("rowD"); 5443 byte[] rowE = Bytes.toBytes("rowE"); 5444 byte[] cf = Bytes.toBytes("CF"); 5445 byte[][] families = { cf }; 5446 byte[] col1 = Bytes.toBytes("col1"); 5447 byte[] col2 = Bytes.toBytes("col2"); 5448 long ts = 1; 5449 this.region = initHRegion(tableName, method, families); 5450 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null); 5451 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null); 5452 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null); 5453 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null); 5454 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null); 5455 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null); 5456 Put put = null; 5457 put = new Put(rowA); 5458 put.add(kv1); 5459 region.put(put); 5460 put = new Put(rowB); 5461 put.add(kv2); 5462 region.put(put); 5463 put = new Put(rowC); 5464 put.add(kv3); 5465 region.put(put); 5466 put = new Put(rowD); 5467 put.add(kv4_1); 5468 region.put(put); 5469 put = new Put(rowD); 5470 put.add(kv4_2); 5471 region.put(put); 5472 put = new Put(rowE); 5473 put.add(kv5); 5474 region.put(put); 5475 region.flush(true); 5476 Scan scan = new Scan().withStartRow(rowD).withStopRow(rowA); 5477 scan.addColumn(families[0], col1); 5478 scan.setReversed(true); 5479 List<Cell> currRow = new ArrayList<>(); 5480 try (InternalScanner scanner = region.getScanner(scan)) { 5481 boolean hasNext = scanner.next(currRow); 5482 assertEquals(1, currRow.size()); 5483 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5484 currRow.get(0).getRowLength(), rowD, 0, rowD.length)); 5485 assertTrue(hasNext); 5486 currRow.clear(); 5487 hasNext = scanner.next(currRow); 5488 assertEquals(1, currRow.size()); 5489 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5490 currRow.get(0).getRowLength(), rowC, 0, rowC.length)); 5491 assertTrue(hasNext); 5492 currRow.clear(); 5493 hasNext = scanner.next(currRow); 5494 assertEquals(1, currRow.size()); 5495 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5496 currRow.get(0).getRowLength(), rowB, 0, rowB.length)); 5497 assertFalse(hasNext); 5498 } 5499 5500 scan = new Scan().withStartRow(rowD).withStopRow(rowA); 5501 scan.addColumn(families[0], col2); 5502 scan.setReversed(true); 5503 currRow.clear(); 5504 try (InternalScanner scanner = region.getScanner(scan)) { 5505 boolean hasNext = scanner.next(currRow); 5506 assertEquals(1, currRow.size()); 5507 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5508 currRow.get(0).getRowLength(), rowD, 0, rowD.length)); 5509 assertTrue(hasNext); 5510 } 5511 } 5512 5513 @Test 5514 public void testReverseScanner_smaller_blocksize() throws IOException { 5515 // case to ensure no conflict with HFile index optimization 5516 byte[] rowA = Bytes.toBytes("rowA"); 5517 byte[] rowB = Bytes.toBytes("rowB"); 5518 byte[] rowC = Bytes.toBytes("rowC"); 5519 byte[] rowD = Bytes.toBytes("rowD"); 5520 byte[] rowE = Bytes.toBytes("rowE"); 5521 byte[] cf = Bytes.toBytes("CF"); 5522 byte[][] families = { cf }; 5523 byte[] col1 = Bytes.toBytes("col1"); 5524 byte[] col2 = Bytes.toBytes("col2"); 5525 long ts = 1; 5526 Configuration conf = new Configuration(CONF); 5527 conf.setInt("test.block.size", 1); 5528 this.region = initHRegion(tableName, method, conf, families); 5529 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null); 5530 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null); 5531 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null); 5532 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null); 5533 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null); 5534 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null); 5535 Put put = null; 5536 put = new Put(rowA); 5537 put.add(kv1); 5538 region.put(put); 5539 put = new Put(rowB); 5540 put.add(kv2); 5541 region.put(put); 5542 put = new Put(rowC); 5543 put.add(kv3); 5544 region.put(put); 5545 put = new Put(rowD); 5546 put.add(kv4_1); 5547 region.put(put); 5548 put = new Put(rowD); 5549 put.add(kv4_2); 5550 region.put(put); 5551 put = new Put(rowE); 5552 put.add(kv5); 5553 region.put(put); 5554 region.flush(true); 5555 Scan scan = new Scan().withStartRow(rowD).withStopRow(rowA); 5556 scan.addColumn(families[0], col1); 5557 scan.setReversed(true); 5558 List<Cell> currRow = new ArrayList<>(); 5559 try (InternalScanner scanner = region.getScanner(scan)) { 5560 boolean hasNext = scanner.next(currRow); 5561 assertEquals(1, currRow.size()); 5562 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5563 currRow.get(0).getRowLength(), rowD, 0, rowD.length)); 5564 assertTrue(hasNext); 5565 currRow.clear(); 5566 hasNext = scanner.next(currRow); 5567 assertEquals(1, currRow.size()); 5568 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5569 currRow.get(0).getRowLength(), rowC, 0, rowC.length)); 5570 assertTrue(hasNext); 5571 currRow.clear(); 5572 hasNext = scanner.next(currRow); 5573 assertEquals(1, currRow.size()); 5574 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5575 currRow.get(0).getRowLength(), rowB, 0, rowB.length)); 5576 assertFalse(hasNext); 5577 } 5578 5579 scan = new Scan().withStartRow(rowD).withStopRow(rowA); 5580 scan.addColumn(families[0], col2); 5581 scan.setReversed(true); 5582 currRow.clear(); 5583 try (InternalScanner scanner = region.getScanner(scan)) { 5584 boolean hasNext = scanner.next(currRow); 5585 assertEquals(1, currRow.size()); 5586 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5587 currRow.get(0).getRowLength(), rowD, 0, rowD.length)); 5588 assertTrue(hasNext); 5589 } 5590 } 5591 5592 @Test 5593 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1() throws IOException { 5594 byte[] row0 = Bytes.toBytes("row0"); // 1 kv 5595 byte[] row1 = Bytes.toBytes("row1"); // 2 kv 5596 byte[] row2 = Bytes.toBytes("row2"); // 4 kv 5597 byte[] row3 = Bytes.toBytes("row3"); // 2 kv 5598 byte[] row4 = Bytes.toBytes("row4"); // 5 kv 5599 byte[] row5 = Bytes.toBytes("row5"); // 2 kv 5600 byte[] cf1 = Bytes.toBytes("CF1"); 5601 byte[] cf2 = Bytes.toBytes("CF2"); 5602 byte[] cf3 = Bytes.toBytes("CF3"); 5603 byte[][] families = { cf1, cf2, cf3 }; 5604 byte[] col = Bytes.toBytes("C"); 5605 long ts = 1; 5606 Configuration conf = new Configuration(CONF); 5607 // disable compactions in this test. 5608 conf.setInt("hbase.hstore.compactionThreshold", 10000); 5609 this.region = initHRegion(tableName, method, conf, families); 5610 // kv naming style: kv(row number) totalKvCountInThisRow seq no 5611 KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put, null); 5612 KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put, null); 5613 KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1, KeyValue.Type.Put, null); 5614 KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null); 5615 KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put, null); 5616 KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put, null); 5617 KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4, KeyValue.Type.Put, null); 5618 KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put, null); 5619 KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4, KeyValue.Type.Put, null); 5620 KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put, null); 5621 KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put, null); 5622 KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5, KeyValue.Type.Put, null); 5623 KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put, null); 5624 KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3, KeyValue.Type.Put, null); 5625 KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put, null); 5626 KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put, null); 5627 // hfiles(cf1/cf2) :"row1"(1 kv) / "row2"(1 kv) / "row4"(2 kv) 5628 Put put = null; 5629 put = new Put(row1); 5630 put.add(kv1_2_1); 5631 region.put(put); 5632 put = new Put(row2); 5633 put.add(kv2_4_1); 5634 region.put(put); 5635 put = new Put(row4); 5636 put.add(kv4_5_4); 5637 put.add(kv4_5_5); 5638 region.put(put); 5639 region.flush(true); 5640 // hfiles(cf1/cf3) : "row1" (1 kvs) / "row2" (1 kv) / "row4" (2 kv) 5641 put = new Put(row4); 5642 put.add(kv4_5_1); 5643 put.add(kv4_5_3); 5644 region.put(put); 5645 put = new Put(row1); 5646 put.add(kv1_2_2); 5647 region.put(put); 5648 put = new Put(row2); 5649 put.add(kv2_4_4); 5650 region.put(put); 5651 region.flush(true); 5652 // hfiles(cf1/cf3) : "row2"(2 kv) / "row3"(1 kvs) / "row4" (1 kv) 5653 put = new Put(row4); 5654 put.add(kv4_5_2); 5655 region.put(put); 5656 put = new Put(row2); 5657 put.add(kv2_4_2); 5658 put.add(kv2_4_3); 5659 region.put(put); 5660 put = new Put(row3); 5661 put.add(kv3_2_2); 5662 region.put(put); 5663 region.flush(true); 5664 // memstore(cf1/cf2/cf3) : "row0" (1 kvs) / "row3" ( 1 kv) / "row5" (max) 5665 // ( 2 kv) 5666 put = new Put(row0); 5667 put.add(kv0_1_1); 5668 region.put(put); 5669 put = new Put(row3); 5670 put.add(kv3_2_1); 5671 region.put(put); 5672 put = new Put(row5); 5673 put.add(kv5_2_1); 5674 put.add(kv5_2_2); 5675 region.put(put); 5676 // scan range = ["row4", min), skip the max "row5" 5677 Scan scan = new Scan().withStartRow(row4); 5678 scan.readVersions(5); 5679 scan.setBatch(3); 5680 scan.setReversed(true); 5681 try (InternalScanner scanner = region.getScanner(scan)) { 5682 List<Cell> currRow = new ArrayList<>(); 5683 boolean hasNext = false; 5684 // 1. scan out "row4" (5 kvs), "row5" can't be scanned out since not 5685 // included in scan range 5686 // "row4" takes 2 next() calls since batch=3 5687 hasNext = scanner.next(currRow); 5688 assertEquals(3, currRow.size()); 5689 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5690 currRow.get(0).getRowLength(), row4, 0, row4.length)); 5691 assertTrue(hasNext); 5692 currRow.clear(); 5693 hasNext = scanner.next(currRow); 5694 assertEquals(2, currRow.size()); 5695 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5696 currRow.get(0).getRowLength(), row4, 0, row4.length)); 5697 assertTrue(hasNext); 5698 // 2. scan out "row3" (2 kv) 5699 currRow.clear(); 5700 hasNext = scanner.next(currRow); 5701 assertEquals(2, currRow.size()); 5702 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5703 currRow.get(0).getRowLength(), row3, 0, row3.length)); 5704 assertTrue(hasNext); 5705 // 3. scan out "row2" (4 kvs) 5706 // "row2" takes 2 next() calls since batch=3 5707 currRow.clear(); 5708 hasNext = scanner.next(currRow); 5709 assertEquals(3, currRow.size()); 5710 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5711 currRow.get(0).getRowLength(), row2, 0, row2.length)); 5712 assertTrue(hasNext); 5713 currRow.clear(); 5714 hasNext = scanner.next(currRow); 5715 assertEquals(1, currRow.size()); 5716 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5717 currRow.get(0).getRowLength(), row2, 0, row2.length)); 5718 assertTrue(hasNext); 5719 // 4. scan out "row1" (2 kv) 5720 currRow.clear(); 5721 hasNext = scanner.next(currRow); 5722 assertEquals(2, currRow.size()); 5723 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5724 currRow.get(0).getRowLength(), row1, 0, row1.length)); 5725 assertTrue(hasNext); 5726 // 5. scan out "row0" (1 kv) 5727 currRow.clear(); 5728 hasNext = scanner.next(currRow); 5729 assertEquals(1, currRow.size()); 5730 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5731 currRow.get(0).getRowLength(), row0, 0, row0.length)); 5732 assertFalse(hasNext); 5733 } 5734 } 5735 5736 @Test 5737 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2() throws IOException { 5738 byte[] row1 = Bytes.toBytes("row1"); 5739 byte[] row2 = Bytes.toBytes("row2"); 5740 byte[] row3 = Bytes.toBytes("row3"); 5741 byte[] row4 = Bytes.toBytes("row4"); 5742 byte[] cf1 = Bytes.toBytes("CF1"); 5743 byte[] cf2 = Bytes.toBytes("CF2"); 5744 byte[] cf3 = Bytes.toBytes("CF3"); 5745 byte[] cf4 = Bytes.toBytes("CF4"); 5746 byte[][] families = { cf1, cf2, cf3, cf4 }; 5747 byte[] col = Bytes.toBytes("C"); 5748 long ts = 1; 5749 Configuration conf = new Configuration(CONF); 5750 // disable compactions in this test. 5751 conf.setInt("hbase.hstore.compactionThreshold", 10000); 5752 this.region = initHRegion(tableName, method, conf, families); 5753 KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null); 5754 KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null); 5755 KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null); 5756 KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null); 5757 // storefile1 5758 Put put = new Put(row1); 5759 put.add(kv1); 5760 region.put(put); 5761 region.flush(true); 5762 // storefile2 5763 put = new Put(row2); 5764 put.add(kv2); 5765 region.put(put); 5766 region.flush(true); 5767 // storefile3 5768 put = new Put(row3); 5769 put.add(kv3); 5770 region.put(put); 5771 region.flush(true); 5772 // memstore 5773 put = new Put(row4); 5774 put.add(kv4); 5775 region.put(put); 5776 // scan range = ["row4", min) 5777 Scan scan = new Scan().withStartRow(row4); 5778 scan.setReversed(true); 5779 scan.setBatch(10); 5780 try (InternalScanner scanner = region.getScanner(scan)) { 5781 List<Cell> currRow = new ArrayList<>(); 5782 boolean hasNext = scanner.next(currRow); 5783 assertEquals(1, currRow.size()); 5784 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5785 currRow.get(0).getRowLength(), row4, 0, row4.length)); 5786 assertTrue(hasNext); 5787 currRow.clear(); 5788 hasNext = scanner.next(currRow); 5789 assertEquals(1, currRow.size()); 5790 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5791 currRow.get(0).getRowLength(), row3, 0, row3.length)); 5792 assertTrue(hasNext); 5793 currRow.clear(); 5794 hasNext = scanner.next(currRow); 5795 assertEquals(1, currRow.size()); 5796 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5797 currRow.get(0).getRowLength(), row2, 0, row2.length)); 5798 assertTrue(hasNext); 5799 currRow.clear(); 5800 hasNext = scanner.next(currRow); 5801 assertEquals(1, currRow.size()); 5802 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5803 currRow.get(0).getRowLength(), row1, 0, row1.length)); 5804 assertFalse(hasNext); 5805 } 5806 } 5807 5808 /** 5809 * Test for HBASE-14497: Reverse Scan threw StackOverflow caused by readPt checking 5810 */ 5811 @Test 5812 public void testReverseScanner_StackOverflow() throws IOException { 5813 byte[] cf1 = Bytes.toBytes("CF1"); 5814 byte[][] families = { cf1 }; 5815 byte[] col = Bytes.toBytes("C"); 5816 Configuration conf = new Configuration(CONF); 5817 this.region = initHRegion(tableName, method, conf, families); 5818 // setup with one storefile and one memstore, to create scanner and get an earlier readPt 5819 Put put = new Put(Bytes.toBytes("19998")); 5820 put.addColumn(cf1, col, Bytes.toBytes("val")); 5821 region.put(put); 5822 region.flushcache(true, true, FlushLifeCycleTracker.DUMMY); 5823 Put put2 = new Put(Bytes.toBytes("19997")); 5824 put2.addColumn(cf1, col, Bytes.toBytes("val")); 5825 region.put(put2); 5826 5827 Scan scan = new Scan().withStartRow(Bytes.toBytes("19998")); 5828 scan.setReversed(true); 5829 try (InternalScanner scanner = region.getScanner(scan)) { 5830 // create one storefile contains many rows will be skipped 5831 // to check StoreFileScanner.seekToPreviousRow 5832 for (int i = 10000; i < 20000; i++) { 5833 Put p = new Put(Bytes.toBytes("" + i)); 5834 p.addColumn(cf1, col, Bytes.toBytes("" + i)); 5835 region.put(p); 5836 } 5837 region.flushcache(true, true, FlushLifeCycleTracker.DUMMY); 5838 5839 // create one memstore contains many rows will be skipped 5840 // to check MemStoreScanner.seekToPreviousRow 5841 for (int i = 10000; i < 20000; i++) { 5842 Put p = new Put(Bytes.toBytes("" + i)); 5843 p.addColumn(cf1, col, Bytes.toBytes("" + i)); 5844 region.put(p); 5845 } 5846 5847 List<Cell> currRow = new ArrayList<>(); 5848 boolean hasNext; 5849 do { 5850 hasNext = scanner.next(currRow); 5851 } while (hasNext); 5852 assertEquals(2, currRow.size()); 5853 assertEquals("19998", Bytes.toString(currRow.get(0).getRowArray(), 5854 currRow.get(0).getRowOffset(), currRow.get(0).getRowLength())); 5855 assertEquals("19997", Bytes.toString(currRow.get(1).getRowArray(), 5856 currRow.get(1).getRowOffset(), currRow.get(1).getRowLength())); 5857 } 5858 } 5859 5860 @Test 5861 public void testReverseScanShouldNotScanMemstoreIfReadPtLesser() throws Exception { 5862 byte[] cf1 = Bytes.toBytes("CF1"); 5863 byte[][] families = { cf1 }; 5864 byte[] col = Bytes.toBytes("C"); 5865 this.region = initHRegion(tableName, method, CONF, families); 5866 // setup with one storefile and one memstore, to create scanner and get an earlier readPt 5867 Put put = new Put(Bytes.toBytes("19996")); 5868 put.addColumn(cf1, col, Bytes.toBytes("val")); 5869 region.put(put); 5870 Put put2 = new Put(Bytes.toBytes("19995")); 5871 put2.addColumn(cf1, col, Bytes.toBytes("val")); 5872 region.put(put2); 5873 // create a reverse scan 5874 Scan scan = new Scan().withStartRow(Bytes.toBytes("19996")); 5875 scan.setReversed(true); 5876 try (RegionScannerImpl scanner = region.getScanner(scan)) { 5877 // flush the cache. This will reset the store scanner 5878 region.flushcache(true, true, FlushLifeCycleTracker.DUMMY); 5879 5880 // create one memstore contains many rows will be skipped 5881 // to check MemStoreScanner.seekToPreviousRow 5882 for (int i = 10000; i < 20000; i++) { 5883 Put p = new Put(Bytes.toBytes("" + i)); 5884 p.addColumn(cf1, col, Bytes.toBytes("" + i)); 5885 region.put(p); 5886 } 5887 List<Cell> currRow = new ArrayList<>(); 5888 boolean hasNext; 5889 boolean assertDone = false; 5890 do { 5891 hasNext = scanner.next(currRow); 5892 // With HBASE-15871, after the scanner is reset the memstore scanner should not be 5893 // added here 5894 if (!assertDone) { 5895 StoreScanner current = (StoreScanner) (scanner.storeHeap).getCurrentForTesting(); 5896 List<KeyValueScanner> scanners = current.getAllScannersForTesting(); 5897 assertEquals(1, scanners.size(), 5898 "There should be only one scanner the store file scanner"); 5899 assertDone = true; 5900 } 5901 } while (hasNext); 5902 assertEquals(2, currRow.size()); 5903 assertEquals("19996", Bytes.toString(currRow.get(0).getRowArray(), 5904 currRow.get(0).getRowOffset(), currRow.get(0).getRowLength())); 5905 assertEquals("19995", Bytes.toString(currRow.get(1).getRowArray(), 5906 currRow.get(1).getRowOffset(), currRow.get(1).getRowLength())); 5907 } 5908 } 5909 5910 @Test 5911 public void testReverseScanWhenPutCellsAfterOpenReverseScan() throws Exception { 5912 byte[] cf1 = Bytes.toBytes("CF1"); 5913 byte[][] families = { cf1 }; 5914 byte[] col = Bytes.toBytes("C"); 5915 5916 this.region = initHRegion(tableName, method, CONF, families); 5917 5918 Put put = new Put(Bytes.toBytes("199996")); 5919 put.addColumn(cf1, col, Bytes.toBytes("val")); 5920 region.put(put); 5921 Put put2 = new Put(Bytes.toBytes("199995")); 5922 put2.addColumn(cf1, col, Bytes.toBytes("val")); 5923 region.put(put2); 5924 5925 // Create a reverse scan 5926 Scan scan = new Scan().withStartRow(Bytes.toBytes("199996")); 5927 scan.setReversed(true); 5928 try (RegionScannerImpl scanner = region.getScanner(scan)) { 5929 // Put a lot of cells that have sequenceIDs grater than the readPt of the reverse scan 5930 for (int i = 100000; i < 200000; i++) { 5931 Put p = new Put(Bytes.toBytes("" + i)); 5932 p.addColumn(cf1, col, Bytes.toBytes("" + i)); 5933 region.put(p); 5934 } 5935 List<Cell> currRow = new ArrayList<>(); 5936 boolean hasNext; 5937 do { 5938 hasNext = scanner.next(currRow); 5939 } while (hasNext); 5940 5941 assertEquals(2, currRow.size()); 5942 assertEquals("199996", Bytes.toString(currRow.get(0).getRowArray(), 5943 currRow.get(0).getRowOffset(), currRow.get(0).getRowLength())); 5944 assertEquals("199995", Bytes.toString(currRow.get(1).getRowArray(), 5945 currRow.get(1).getRowOffset(), currRow.get(1).getRowLength())); 5946 } 5947 } 5948 5949 @Test 5950 public void testWriteRequestsCounter() throws IOException { 5951 byte[] fam = Bytes.toBytes("info"); 5952 byte[][] families = { fam }; 5953 this.region = initHRegion(tableName, method, CONF, families); 5954 5955 assertEquals(0L, region.getWriteRequestsCount()); 5956 5957 Put put = new Put(row); 5958 put.addColumn(fam, fam, fam); 5959 5960 assertEquals(0L, region.getWriteRequestsCount()); 5961 region.put(put); 5962 assertEquals(1L, region.getWriteRequestsCount()); 5963 region.put(put); 5964 assertEquals(2L, region.getWriteRequestsCount()); 5965 region.put(put); 5966 assertEquals(3L, region.getWriteRequestsCount()); 5967 5968 region.delete(new Delete(row)); 5969 assertEquals(4L, region.getWriteRequestsCount()); 5970 } 5971 5972 @Test 5973 public void testOpenRegionWrittenToWAL() throws Exception { 5974 final ServerName serverName = ServerName.valueOf(name, 100, 42); 5975 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName)); 5976 5977 TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)) 5978 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)) 5979 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam2)).build(); 5980 RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build(); 5981 5982 // open the region w/o rss and wal and flush some files 5983 region = HBaseTestingUtil.createRegionAndWAL(hri, TEST_UTIL.getDataTestDir(), 5984 TEST_UTIL.getConfiguration(), htd); 5985 assertNotNull(region); 5986 5987 // create a file in fam1 for the region before opening in OpenRegionHandler 5988 region.put(new Put(Bytes.toBytes("a")).addColumn(fam1, fam1, fam1)); 5989 region.flush(true); 5990 HBaseTestingUtil.closeRegionAndWAL(region); 5991 5992 ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class); 5993 5994 // capture append() calls 5995 WAL wal = mockWAL(); 5996 when(rss.getWAL(any(RegionInfo.class))).thenReturn(wal); 5997 5998 region = 5999 HRegion.openHRegion(hri, htd, rss.getWAL(hri), TEST_UTIL.getConfiguration(), rss, null); 6000 6001 verify(wal, times(1)).appendMarker(any(RegionInfo.class), any(WALKeyImpl.class), 6002 editCaptor.capture()); 6003 6004 WALEdit edit = editCaptor.getValue(); 6005 assertNotNull(edit); 6006 assertNotNull(edit.getCells()); 6007 assertEquals(1, edit.getCells().size()); 6008 RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0)); 6009 assertNotNull(desc); 6010 6011 LOG.info("RegionEventDescriptor from WAL: " + desc); 6012 6013 assertEquals(RegionEventDescriptor.EventType.REGION_OPEN, desc.getEventType()); 6014 assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getTableName().toBytes())); 6015 assertTrue( 6016 Bytes.equals(desc.getEncodedRegionName().toByteArray(), hri.getEncodedNameAsBytes())); 6017 assertTrue(desc.getLogSequenceNumber() > 0); 6018 assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer())); 6019 assertEquals(2, desc.getStoresCount()); 6020 6021 StoreDescriptor store = desc.getStores(0); 6022 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1)); 6023 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1)); 6024 assertEquals(1, store.getStoreFileCount()); // 1store file 6025 assertFalse(store.getStoreFile(0).contains("/")); // ensure path is relative 6026 6027 store = desc.getStores(1); 6028 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2)); 6029 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2)); 6030 assertEquals(0, store.getStoreFileCount()); // no store files 6031 } 6032 6033 // Helper for test testOpenRegionWrittenToWALForLogReplay 6034 static class HRegionWithSeqId extends HRegion { 6035 public HRegionWithSeqId(final Path tableDir, final WAL wal, final FileSystem fs, 6036 final Configuration confParam, final RegionInfo regionInfo, final TableDescriptor htd, 6037 final RegionServerServices rsServices) { 6038 super(tableDir, wal, fs, confParam, regionInfo, htd, rsServices); 6039 } 6040 6041 @Override 6042 protected long getNextSequenceId(WAL wal) throws IOException { 6043 return 42; 6044 } 6045 } 6046 6047 @Test 6048 public void testFlushedFileWithNoTags() throws Exception { 6049 final TableName tableName = TableName.valueOf(name); 6050 TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName) 6051 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)).build(); 6052 RegionInfo info = RegionInfoBuilder.newBuilder(tableName).build(); 6053 Path path = TEST_UTIL.getDataTestDir(getClass().getSimpleName()); 6054 region = HBaseTestingUtil.createRegionAndWAL(info, path, TEST_UTIL.getConfiguration(), 6055 tableDescriptor); 6056 Put put = new Put(Bytes.toBytes("a-b-0-0")); 6057 put.addColumn(fam1, qual1, Bytes.toBytes("c1-value")); 6058 region.put(put); 6059 region.flush(true); 6060 HStore store = region.getStore(fam1); 6061 Collection<HStoreFile> storefiles = store.getStorefiles(); 6062 for (HStoreFile sf : storefiles) { 6063 assertFalse(sf.getReader().getHFileReader().getFileContext().isIncludesTags(), 6064 "Tags should not be present "); 6065 } 6066 } 6067 6068 /** 6069 * Utility method to setup a WAL mock. 6070 * <p/> 6071 * Needs to do the bit where we close latch on the WALKeyImpl on append else test hangs. 6072 * @return a mock WAL 6073 */ 6074 private WAL mockWAL() throws IOException { 6075 WAL wal = mock(WAL.class); 6076 when(wal.appendData(any(RegionInfo.class), any(WALKeyImpl.class), any(WALEdit.class))) 6077 .thenAnswer(new Answer<Long>() { 6078 @Override 6079 public Long answer(InvocationOnMock invocation) throws Throwable { 6080 WALKeyImpl key = invocation.getArgument(1); 6081 MultiVersionConcurrencyControl.WriteEntry we = key.getMvcc().begin(); 6082 key.setWriteEntry(we); 6083 return 1L; 6084 } 6085 }); 6086 when(wal.appendMarker(any(RegionInfo.class), any(WALKeyImpl.class), any(WALEdit.class))) 6087 .thenAnswer(new Answer<Long>() { 6088 @Override 6089 public Long answer(InvocationOnMock invocation) throws Throwable { 6090 WALKeyImpl key = invocation.getArgument(1); 6091 MultiVersionConcurrencyControl.WriteEntry we = key.getMvcc().begin(); 6092 key.setWriteEntry(we); 6093 return 1L; 6094 } 6095 }); 6096 return wal; 6097 } 6098 6099 @Test 6100 public void testCloseRegionWrittenToWAL() throws Exception { 6101 Path rootDir = new Path(dir + name); 6102 CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir); 6103 6104 final ServerName serverName = ServerName.valueOf("testCloseRegionWrittenToWAL", 100, 42); 6105 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName)); 6106 6107 TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)) 6108 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)) 6109 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam2)).build(); 6110 RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build(); 6111 6112 ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class); 6113 6114 // capture append() calls 6115 WAL wal = mockWAL(); 6116 when(rss.getWAL(any(RegionInfo.class))).thenReturn(wal); 6117 6118 // create the region 6119 region = HBaseTestingUtil.createRegionAndWAL(hri, rootDir, CONF, htd); 6120 HBaseTestingUtil.closeRegionAndWAL(region); 6121 region = null; 6122 // open the region first and then close it 6123 HRegion.openHRegion(hri, htd, rss.getWAL(hri), TEST_UTIL.getConfiguration(), rss, null).close(); 6124 6125 // 2 times, one for region open, the other close region 6126 verify(wal, times(2)).appendMarker(any(RegionInfo.class), (WALKeyImpl) any(WALKeyImpl.class), 6127 editCaptor.capture()); 6128 6129 WALEdit edit = editCaptor.getAllValues().get(1); 6130 assertNotNull(edit); 6131 assertNotNull(edit.getCells()); 6132 assertEquals(1, edit.getCells().size()); 6133 RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0)); 6134 assertNotNull(desc); 6135 6136 LOG.info("RegionEventDescriptor from WAL: " + desc); 6137 6138 assertEquals(RegionEventDescriptor.EventType.REGION_CLOSE, desc.getEventType()); 6139 assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getTableName().toBytes())); 6140 assertTrue( 6141 Bytes.equals(desc.getEncodedRegionName().toByteArray(), hri.getEncodedNameAsBytes())); 6142 assertTrue(desc.getLogSequenceNumber() > 0); 6143 assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer())); 6144 assertEquals(2, desc.getStoresCount()); 6145 6146 StoreDescriptor store = desc.getStores(0); 6147 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1)); 6148 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1)); 6149 assertEquals(0, store.getStoreFileCount()); // no store files 6150 6151 store = desc.getStores(1); 6152 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2)); 6153 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2)); 6154 assertEquals(0, store.getStoreFileCount()); // no store files 6155 } 6156 6157 /** 6158 * Test RegionTooBusyException thrown when region is busy 6159 */ 6160 @Test 6161 public void testRegionTooBusy() throws IOException { 6162 byte[] family = Bytes.toBytes("family"); 6163 long defaultBusyWaitDuration = 6164 CONF.getLong("hbase.busy.wait.duration", HRegion.DEFAULT_BUSY_WAIT_DURATION); 6165 CONF.setLong("hbase.busy.wait.duration", 1000); 6166 region = initHRegion(tableName, method, CONF, family); 6167 final AtomicBoolean stopped = new AtomicBoolean(true); 6168 Thread t = new Thread(new Runnable() { 6169 @Override 6170 public void run() { 6171 try { 6172 region.lock.writeLock().lock(); 6173 stopped.set(false); 6174 while (!stopped.get()) { 6175 Thread.sleep(100); 6176 } 6177 } catch (InterruptedException ie) { 6178 } finally { 6179 region.lock.writeLock().unlock(); 6180 } 6181 } 6182 }); 6183 t.start(); 6184 Get get = new Get(row); 6185 try { 6186 while (stopped.get()) { 6187 Thread.sleep(100); 6188 } 6189 region.get(get); 6190 fail("Should throw RegionTooBusyException"); 6191 } catch (InterruptedException ie) { 6192 fail("test interrupted"); 6193 } catch (RegionTooBusyException e) { 6194 // Good, expected 6195 } finally { 6196 stopped.set(true); 6197 try { 6198 t.join(); 6199 } catch (Throwable e) { 6200 } 6201 6202 HBaseTestingUtil.closeRegionAndWAL(region); 6203 region = null; 6204 CONF.setLong("hbase.busy.wait.duration", defaultBusyWaitDuration); 6205 } 6206 } 6207 6208 @Test 6209 public void testCellTTLs() throws IOException { 6210 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge(); 6211 EnvironmentEdgeManager.injectEdge(edge); 6212 6213 final byte[] row = Bytes.toBytes("testRow"); 6214 final byte[] q1 = Bytes.toBytes("q1"); 6215 final byte[] q2 = Bytes.toBytes("q2"); 6216 final byte[] q3 = Bytes.toBytes("q3"); 6217 final byte[] q4 = Bytes.toBytes("q4"); 6218 6219 // 10 seconds 6220 TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)) 6221 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam1).setTimeToLive(10).build()) 6222 .build(); 6223 6224 Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); 6225 conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS); 6226 6227 region = HBaseTestingUtil.createRegionAndWAL( 6228 RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(), 6229 TEST_UTIL.getDataTestDir(), conf, tableDescriptor); 6230 assertNotNull(region); 6231 long now = EnvironmentEdgeManager.currentTime(); 6232 // Add a cell that will expire in 5 seconds via cell TTL 6233 region.put(new Put(row).add(new KeyValue(row, fam1, q1, now, HConstants.EMPTY_BYTE_ARRAY, 6234 new ArrayBackedTag[] { 6235 // TTL tags specify ts in milliseconds 6236 new ArrayBackedTag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) }))); 6237 // Add a cell that will expire after 10 seconds via family setting 6238 region.put(new Put(row).addColumn(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY)); 6239 // Add a cell that will expire in 15 seconds via cell TTL 6240 region.put(new Put(row).add(new KeyValue(row, fam1, q3, now + 10000 - 1, 6241 HConstants.EMPTY_BYTE_ARRAY, new ArrayBackedTag[] { 6242 // TTL tags specify ts in milliseconds 6243 new ArrayBackedTag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) }))); 6244 // Add a cell that will expire in 20 seconds via family setting 6245 region.put(new Put(row).addColumn(fam1, q4, now + 10000 - 1, HConstants.EMPTY_BYTE_ARRAY)); 6246 6247 // Flush so we are sure store scanning gets this right 6248 region.flush(true); 6249 6250 // A query at time T+0 should return all cells 6251 Result r = region.get(new Get(row)); 6252 assertNotNull(r.getValue(fam1, q1)); 6253 assertNotNull(r.getValue(fam1, q2)); 6254 assertNotNull(r.getValue(fam1, q3)); 6255 assertNotNull(r.getValue(fam1, q4)); 6256 6257 // Increment time to T+5 seconds 6258 edge.incrementTime(5000); 6259 6260 r = region.get(new Get(row)); 6261 assertNull(r.getValue(fam1, q1)); 6262 assertNotNull(r.getValue(fam1, q2)); 6263 assertNotNull(r.getValue(fam1, q3)); 6264 assertNotNull(r.getValue(fam1, q4)); 6265 6266 // Increment time to T+10 seconds 6267 edge.incrementTime(5000); 6268 6269 r = region.get(new Get(row)); 6270 assertNull(r.getValue(fam1, q1)); 6271 assertNull(r.getValue(fam1, q2)); 6272 assertNotNull(r.getValue(fam1, q3)); 6273 assertNotNull(r.getValue(fam1, q4)); 6274 6275 // Increment time to T+15 seconds 6276 edge.incrementTime(5000); 6277 6278 r = region.get(new Get(row)); 6279 assertNull(r.getValue(fam1, q1)); 6280 assertNull(r.getValue(fam1, q2)); 6281 assertNull(r.getValue(fam1, q3)); 6282 assertNotNull(r.getValue(fam1, q4)); 6283 6284 // Increment time to T+20 seconds 6285 edge.incrementTime(10000); 6286 6287 r = region.get(new Get(row)); 6288 assertNull(r.getValue(fam1, q1)); 6289 assertNull(r.getValue(fam1, q2)); 6290 assertNull(r.getValue(fam1, q3)); 6291 assertNull(r.getValue(fam1, q4)); 6292 6293 // Fun with disappearing increments 6294 6295 // Start at 1 6296 region.put(new Put(row).addColumn(fam1, q1, Bytes.toBytes(1L))); 6297 r = region.get(new Get(row)); 6298 byte[] val = r.getValue(fam1, q1); 6299 assertNotNull(val); 6300 assertEquals(1L, Bytes.toLong(val)); 6301 6302 // Increment with a TTL of 5 seconds 6303 Increment incr = new Increment(row).addColumn(fam1, q1, 1L); 6304 incr.setTTL(5000); 6305 region.increment(incr); // 2 6306 6307 // New value should be 2 6308 r = region.get(new Get(row)); 6309 val = r.getValue(fam1, q1); 6310 assertNotNull(val); 6311 assertEquals(2L, Bytes.toLong(val)); 6312 6313 // Increment time to T+25 seconds 6314 edge.incrementTime(5000); 6315 6316 // Value should be back to 1 6317 r = region.get(new Get(row)); 6318 val = r.getValue(fam1, q1); 6319 assertNotNull(val); 6320 assertEquals(1L, Bytes.toLong(val)); 6321 6322 // Increment time to T+30 seconds 6323 edge.incrementTime(5000); 6324 6325 // Original value written at T+20 should be gone now via family TTL 6326 r = region.get(new Get(row)); 6327 assertNull(r.getValue(fam1, q1)); 6328 } 6329 6330 @Test 6331 public void testTTLsUsingSmallHeartBeatCells() throws IOException { 6332 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge(); 6333 EnvironmentEdgeManager.injectEdge(edge); 6334 6335 final byte[] row = Bytes.toBytes("testRow"); 6336 final byte[] q1 = Bytes.toBytes("q1"); 6337 final byte[] q2 = Bytes.toBytes("q2"); 6338 final byte[] q3 = Bytes.toBytes("q3"); 6339 final byte[] q4 = Bytes.toBytes("q4"); 6340 final byte[] q5 = Bytes.toBytes("q5"); 6341 final byte[] q6 = Bytes.toBytes("q6"); 6342 final byte[] q7 = Bytes.toBytes("q7"); 6343 final byte[] q8 = Bytes.toBytes("q8"); 6344 6345 // 10 seconds 6346 int ttlSecs = 10; 6347 TableDescriptor tableDescriptor = 6348 TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).setColumnFamily( 6349 ColumnFamilyDescriptorBuilder.newBuilder(fam1).setTimeToLive(ttlSecs).build()).build(); 6350 6351 Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); 6352 conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS); 6353 // using small heart beat cells 6354 conf.setLong(StoreScanner.HBASE_CELLS_SCANNED_PER_HEARTBEAT_CHECK, 2); 6355 6356 region = HBaseTestingUtil.createRegionAndWAL( 6357 RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(), 6358 TEST_UTIL.getDataTestDir(), conf, tableDescriptor); 6359 assertNotNull(region); 6360 long now = EnvironmentEdgeManager.currentTime(); 6361 // Add a cell that will expire in 5 seconds via cell TTL 6362 region.put(new Put(row).addColumn(fam1, q1, now, HConstants.EMPTY_BYTE_ARRAY)); 6363 region.put(new Put(row).addColumn(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY)); 6364 region.put(new Put(row).addColumn(fam1, q3, now, HConstants.EMPTY_BYTE_ARRAY)); 6365 // Add a cell that will expire after 10 seconds via family setting 6366 region 6367 .put(new Put(row).addColumn(fam1, q4, now + ttlSecs * 1000 + 1, HConstants.EMPTY_BYTE_ARRAY)); 6368 region 6369 .put(new Put(row).addColumn(fam1, q5, now + ttlSecs * 1000 + 1, HConstants.EMPTY_BYTE_ARRAY)); 6370 6371 region.put(new Put(row).addColumn(fam1, q6, now, HConstants.EMPTY_BYTE_ARRAY)); 6372 region.put(new Put(row).addColumn(fam1, q7, now, HConstants.EMPTY_BYTE_ARRAY)); 6373 region 6374 .put(new Put(row).addColumn(fam1, q8, now + ttlSecs * 1000 + 1, HConstants.EMPTY_BYTE_ARRAY)); 6375 6376 // Flush so we are sure store scanning gets this right 6377 region.flush(true); 6378 6379 // A query at time T+0 should return all cells 6380 checkScan(8); 6381 region.delete(new Delete(row).addColumn(fam1, q8)); 6382 6383 // Increment time to T+ttlSecs seconds 6384 edge.incrementTime(ttlSecs * 1000); 6385 checkScan(2); 6386 } 6387 6388 private void checkScan(int expectCellSize) throws IOException { 6389 Scan s = new Scan().withStartRow(row); 6390 ScannerContext.Builder contextBuilder = ScannerContext.newBuilder(true); 6391 ScannerContext scannerContext = contextBuilder.build(); 6392 try (RegionScanner scanner = region.getScanner(s)) { 6393 List<Cell> kvs = new ArrayList<>(); 6394 scanner.next(kvs, scannerContext); 6395 assertEquals(expectCellSize, kvs.size()); 6396 } 6397 } 6398 6399 @Test 6400 public void testIncrementTimestampsAreMonotonic() throws IOException { 6401 region = initHRegion(tableName, method, CONF, fam1); 6402 ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); 6403 EnvironmentEdgeManager.injectEdge(edge); 6404 6405 edge.setValue(10); 6406 Increment inc = new Increment(row); 6407 inc.setDurability(Durability.SKIP_WAL); 6408 inc.addColumn(fam1, qual1, 1L); 6409 region.increment(inc); 6410 6411 Result result = region.get(new Get(row)); 6412 Cell c = result.getColumnLatestCell(fam1, qual1); 6413 assertNotNull(c); 6414 assertEquals(10L, c.getTimestamp()); 6415 6416 edge.setValue(1); // clock goes back 6417 region.increment(inc); 6418 result = region.get(new Get(row)); 6419 c = result.getColumnLatestCell(fam1, qual1); 6420 assertEquals(11L, c.getTimestamp()); 6421 assertEquals(2L, Bytes.toLong(c.getValueArray(), c.getValueOffset(), c.getValueLength())); 6422 } 6423 6424 @Test 6425 public void testAppendTimestampsAreMonotonic() throws IOException { 6426 region = initHRegion(tableName, method, CONF, fam1); 6427 ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); 6428 EnvironmentEdgeManager.injectEdge(edge); 6429 6430 edge.setValue(10); 6431 Append a = new Append(row); 6432 a.setDurability(Durability.SKIP_WAL); 6433 a.addColumn(fam1, qual1, qual1); 6434 region.append(a); 6435 6436 Result result = region.get(new Get(row)); 6437 Cell c = result.getColumnLatestCell(fam1, qual1); 6438 assertNotNull(c); 6439 assertEquals(10L, c.getTimestamp()); 6440 6441 edge.setValue(1); // clock goes back 6442 region.append(a); 6443 result = region.get(new Get(row)); 6444 c = result.getColumnLatestCell(fam1, qual1); 6445 assertEquals(11L, c.getTimestamp()); 6446 6447 byte[] expected = new byte[qual1.length * 2]; 6448 System.arraycopy(qual1, 0, expected, 0, qual1.length); 6449 System.arraycopy(qual1, 0, expected, qual1.length, qual1.length); 6450 6451 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(), expected, 0, 6452 expected.length)); 6453 } 6454 6455 @Test 6456 public void testCheckAndMutateTimestampsAreMonotonic() throws IOException { 6457 region = initHRegion(tableName, method, CONF, fam1); 6458 ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); 6459 EnvironmentEdgeManager.injectEdge(edge); 6460 6461 edge.setValue(10); 6462 Put p = new Put(row); 6463 p.setDurability(Durability.SKIP_WAL); 6464 p.addColumn(fam1, qual1, qual1); 6465 region.put(p); 6466 6467 Result result = region.get(new Get(row)); 6468 Cell c = result.getColumnLatestCell(fam1, qual1); 6469 assertNotNull(c); 6470 assertEquals(10L, c.getTimestamp()); 6471 6472 edge.setValue(1); // clock goes back 6473 p = new Put(row); 6474 p.setDurability(Durability.SKIP_WAL); 6475 p.addColumn(fam1, qual1, qual2); 6476 region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(qual1), p); 6477 result = region.get(new Get(row)); 6478 c = result.getColumnLatestCell(fam1, qual1); 6479 assertEquals(10L, c.getTimestamp()); 6480 6481 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(), qual2, 0, 6482 qual2.length)); 6483 } 6484 6485 @Test 6486 public void testBatchMutateWithWrongRegionException() throws Exception { 6487 final byte[] a = Bytes.toBytes("a"); 6488 final byte[] b = Bytes.toBytes("b"); 6489 final byte[] c = Bytes.toBytes("c"); // exclusive 6490 6491 int prevLockTimeout = CONF.getInt("hbase.rowlock.wait.duration", 30000); 6492 CONF.setInt("hbase.rowlock.wait.duration", 1000); 6493 region = initHRegion(tableName, a, c, method, CONF, false, fam1); 6494 6495 Mutation[] mutations = new Mutation[] { 6496 new Put(a).add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a) 6497 .setFamily(fam1).setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), 6498 // this is outside the region boundary 6499 new Put(c).add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(c) 6500 .setFamily(fam1).setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Type.Put).build()), 6501 new Put(b) 6502 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1) 6503 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()) }; 6504 6505 OperationStatus[] status = region.batchMutate(mutations); 6506 assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode()); 6507 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, status[1].getOperationStatusCode()); 6508 assertEquals(OperationStatusCode.SUCCESS, status[2].getOperationStatusCode()); 6509 6510 // test with a row lock held for a long time 6511 final CountDownLatch obtainedRowLock = new CountDownLatch(1); 6512 ExecutorService exec = Executors.newFixedThreadPool(2); 6513 Future<Void> f1 = exec.submit(new Callable<Void>() { 6514 @Override 6515 public Void call() throws Exception { 6516 LOG.info("Acquiring row lock"); 6517 RowLock rl = region.getRowLock(b); 6518 obtainedRowLock.countDown(); 6519 LOG.info("Waiting for 5 seconds before releasing lock"); 6520 Threads.sleep(5000); 6521 LOG.info("Releasing row lock"); 6522 rl.release(); 6523 return null; 6524 } 6525 }); 6526 obtainedRowLock.await(30, TimeUnit.SECONDS); 6527 6528 Future<Void> f2 = exec.submit(new Callable<Void>() { 6529 @Override 6530 public Void call() throws Exception { 6531 Mutation[] mutations = new Mutation[] { 6532 new Put(a) 6533 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a).setFamily(fam1) 6534 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), 6535 new Put(b) 6536 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1) 6537 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), }; 6538 6539 // this will wait for the row lock, and it will eventually succeed 6540 OperationStatus[] status = region.batchMutate(mutations); 6541 assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode()); 6542 assertEquals(OperationStatusCode.SUCCESS, status[1].getOperationStatusCode()); 6543 return null; 6544 } 6545 }); 6546 6547 f1.get(); 6548 f2.get(); 6549 6550 CONF.setInt("hbase.rowlock.wait.duration", prevLockTimeout); 6551 } 6552 6553 @Test 6554 public void testBatchMutateWithZeroRowLockWait() throws Exception { 6555 final byte[] a = Bytes.toBytes("a"); 6556 final byte[] b = Bytes.toBytes("b"); 6557 final byte[] c = Bytes.toBytes("c"); // exclusive 6558 6559 Configuration conf = new Configuration(CONF); 6560 conf.setInt("hbase.rowlock.wait.duration", 0); 6561 final RegionInfo hri = 6562 RegionInfoBuilder.newBuilder(tableName).setStartKey(a).setEndKey(c).build(); 6563 final TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName) 6564 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)).build(); 6565 region = HRegion.createHRegion(hri, TEST_UTIL.getDataTestDir(), conf, htd, 6566 HBaseTestingUtil.createWal(conf, TEST_UTIL.getDataTestDirOnTestFS(method + ".log"), hri)); 6567 6568 Mutation[] mutations = new Mutation[] { 6569 new Put(a).add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a) 6570 .setFamily(fam1).setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), 6571 new Put(b) 6572 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1) 6573 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()) }; 6574 6575 OperationStatus[] status = region.batchMutate(mutations); 6576 assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode()); 6577 assertEquals(OperationStatusCode.SUCCESS, status[1].getOperationStatusCode()); 6578 6579 // test with a row lock held for a long time 6580 final CountDownLatch obtainedRowLock = new CountDownLatch(1); 6581 ExecutorService exec = Executors.newFixedThreadPool(2); 6582 Future<Void> f1 = exec.submit(new Callable<Void>() { 6583 @Override 6584 public Void call() throws Exception { 6585 LOG.info("Acquiring row lock"); 6586 RowLock rl = region.getRowLock(b); 6587 obtainedRowLock.countDown(); 6588 LOG.info("Waiting for 5 seconds before releasing lock"); 6589 Threads.sleep(5000); 6590 LOG.info("Releasing row lock"); 6591 rl.release(); 6592 return null; 6593 } 6594 }); 6595 obtainedRowLock.await(30, TimeUnit.SECONDS); 6596 6597 Future<Void> f2 = exec.submit(new Callable<Void>() { 6598 @Override 6599 public Void call() throws Exception { 6600 Mutation[] mutations = new Mutation[] { 6601 new Put(a) 6602 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a).setFamily(fam1) 6603 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), 6604 new Put(b) 6605 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1) 6606 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), }; 6607 // when handling row b we are going to spin on the failure to get the row lock 6608 // until the lock above is released, but we will still succeed so long as that 6609 // takes less time then the test time out. 6610 OperationStatus[] status = region.batchMutate(mutations); 6611 assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode()); 6612 assertEquals(OperationStatusCode.SUCCESS, status[1].getOperationStatusCode()); 6613 return null; 6614 } 6615 }); 6616 6617 f1.get(); 6618 f2.get(); 6619 } 6620 6621 @Test 6622 public void testCheckAndRowMutateTimestampsAreMonotonic() throws IOException { 6623 region = initHRegion(tableName, method, CONF, fam1); 6624 ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); 6625 EnvironmentEdgeManager.injectEdge(edge); 6626 6627 edge.setValue(10); 6628 Put p = new Put(row); 6629 p.setDurability(Durability.SKIP_WAL); 6630 p.addColumn(fam1, qual1, qual1); 6631 region.put(p); 6632 6633 Result result = region.get(new Get(row)); 6634 Cell c = result.getColumnLatestCell(fam1, qual1); 6635 assertNotNull(c); 6636 assertEquals(10L, c.getTimestamp()); 6637 6638 edge.setValue(1); // clock goes back 6639 p = new Put(row); 6640 p.setDurability(Durability.SKIP_WAL); 6641 p.addColumn(fam1, qual1, qual2); 6642 RowMutations rm = new RowMutations(row); 6643 rm.add(p); 6644 assertTrue(region.checkAndRowMutate(row, fam1, qual1, CompareOperator.EQUAL, 6645 new BinaryComparator(qual1), rm)); 6646 result = region.get(new Get(row)); 6647 c = result.getColumnLatestCell(fam1, qual1); 6648 assertEquals(10L, c.getTimestamp()); 6649 LOG.info( 6650 "c value " + Bytes.toStringBinary(c.getValueArray(), c.getValueOffset(), c.getValueLength())); 6651 6652 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(), qual2, 0, 6653 qual2.length)); 6654 } 6655 6656 private HRegion initHRegion(TableName tableName, String callingMethod, byte[]... families) 6657 throws IOException { 6658 return initHRegion(tableName, callingMethod, HBaseConfiguration.create(), families); 6659 } 6660 6661 /** 6662 * HBASE-16429 Make sure no stuck if roll writer when ring buffer is filled with appends 6663 * @throws IOException if IO error occurred during test 6664 */ 6665 @Test 6666 public void testWritesWhileRollWriter() throws IOException { 6667 int testCount = 10; 6668 int numRows = 1024; 6669 int numFamilies = 2; 6670 int numQualifiers = 2; 6671 final byte[][] families = new byte[numFamilies][]; 6672 for (int i = 0; i < numFamilies; i++) { 6673 families[i] = Bytes.toBytes("family" + i); 6674 } 6675 final byte[][] qualifiers = new byte[numQualifiers][]; 6676 for (int i = 0; i < numQualifiers; i++) { 6677 qualifiers[i] = Bytes.toBytes("qual" + i); 6678 } 6679 6680 CONF.setInt("hbase.regionserver.wal.disruptor.event.count", 2); 6681 this.region = initHRegion(tableName, method, CONF, families); 6682 try { 6683 List<Thread> threads = new ArrayList<>(); 6684 for (int i = 0; i < numRows; i++) { 6685 final int count = i; 6686 Thread t = new Thread(new Runnable() { 6687 6688 @Override 6689 public void run() { 6690 byte[] row = Bytes.toBytes("row" + count); 6691 Put put = new Put(row); 6692 put.setDurability(Durability.SYNC_WAL); 6693 byte[] value = Bytes.toBytes(String.valueOf(count)); 6694 for (byte[] family : families) { 6695 for (byte[] qualifier : qualifiers) { 6696 put.addColumn(family, qualifier, count, value); 6697 } 6698 } 6699 try { 6700 region.put(put); 6701 } catch (IOException e) { 6702 throw new RuntimeException(e); 6703 } 6704 } 6705 }); 6706 threads.add(t); 6707 } 6708 for (Thread t : threads) { 6709 t.start(); 6710 } 6711 6712 for (int i = 0; i < testCount; i++) { 6713 region.getWAL().rollWriter(); 6714 Thread.yield(); 6715 } 6716 } finally { 6717 try { 6718 HBaseTestingUtil.closeRegionAndWAL(this.region); 6719 CONF.setInt("hbase.regionserver.wal.disruptor.event.count", 16 * 1024); 6720 } catch (DroppedSnapshotException dse) { 6721 // We could get this on way out because we interrupt the background flusher and it could 6722 // fail anywhere causing a DSE over in the background flusher... only it is not properly 6723 // dealt with so could still be memory hanging out when we get to here -- memory we can't 6724 // flush because the accounting is 'off' since original DSE. 6725 } 6726 this.region = null; 6727 } 6728 } 6729 6730 @Test 6731 public void testMutateRow() throws Exception { 6732 final byte[] row = Bytes.toBytes("row"); 6733 final byte[] q1 = Bytes.toBytes("q1"); 6734 final byte[] q2 = Bytes.toBytes("q2"); 6735 final byte[] q3 = Bytes.toBytes("q3"); 6736 final byte[] q4 = Bytes.toBytes("q4"); 6737 final String v1 = "v1"; 6738 6739 region = initHRegion(tableName, method, CONF, fam1); 6740 6741 // Initial values 6742 region 6743 .batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q2, Bytes.toBytes("toBeDeleted")), 6744 new Put(row).addColumn(fam1, q3, Bytes.toBytes(5L)), 6745 new Put(row).addColumn(fam1, q4, Bytes.toBytes("a")), }); 6746 6747 // Do mutateRow 6748 Result result = region.mutateRow( 6749 new RowMutations(row).add(Arrays.asList(new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1)), 6750 new Delete(row).addColumns(fam1, q2), new Increment(row).addColumn(fam1, q3, 1), 6751 new Append(row).addColumn(fam1, q4, Bytes.toBytes("b"))))); 6752 6753 assertNotNull(result); 6754 assertEquals(6L, Bytes.toLong(result.getValue(fam1, q3))); 6755 assertEquals("ab", Bytes.toString(result.getValue(fam1, q4))); 6756 6757 // Verify the value 6758 result = region.get(new Get(row)); 6759 assertEquals(v1, Bytes.toString(result.getValue(fam1, q1))); 6760 assertNull(result.getValue(fam1, q2)); 6761 assertEquals(6L, Bytes.toLong(result.getValue(fam1, q3))); 6762 assertEquals("ab", Bytes.toString(result.getValue(fam1, q4))); 6763 } 6764 6765 @Test 6766 public void testMutateRow_WriteRequestCount() throws Exception { 6767 byte[] row1 = Bytes.toBytes("row1"); 6768 byte[] fam1 = Bytes.toBytes("fam1"); 6769 byte[] qf1 = Bytes.toBytes("qualifier"); 6770 byte[] val1 = Bytes.toBytes("value1"); 6771 6772 RowMutations rm = new RowMutations(row1); 6773 Put put = new Put(row1); 6774 put.addColumn(fam1, qf1, val1); 6775 rm.add(put); 6776 6777 this.region = initHRegion(tableName, method, CONF, fam1); 6778 long wrcBeforeMutate = this.region.writeRequestsCount.longValue(); 6779 this.region.mutateRow(rm); 6780 long wrcAfterMutate = this.region.writeRequestsCount.longValue(); 6781 assertEquals(wrcBeforeMutate + rm.getMutations().size(), wrcAfterMutate); 6782 } 6783 6784 @Test 6785 public void testBulkLoadReplicationEnabled() throws IOException { 6786 TEST_UTIL.getConfiguration().setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true); 6787 final ServerName serverName = ServerName.valueOf(name, 100, 42); 6788 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName)); 6789 6790 TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)) 6791 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)).build(); 6792 RegionInfo hri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 6793 TEST_UTIL.createRegionDir(hri); 6794 region = HRegion.openHRegion(hri, tableDescriptor, rss.getWAL(hri), 6795 TEST_UTIL.getConfiguration(), rss, null); 6796 6797 assertTrue(region.conf.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, false)); 6798 String plugins = region.conf.get(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, ""); 6799 String replicationCoprocessorClass = ReplicationObserver.class.getCanonicalName(); 6800 assertTrue(plugins.contains(replicationCoprocessorClass)); 6801 assertTrue(region.getCoprocessorHost().getCoprocessors() 6802 .contains(ReplicationObserver.class.getSimpleName())); 6803 } 6804 6805 /** 6806 * The same as HRegion class, the only difference is that instantiateHStore will create a 6807 * different HStore - HStoreForTesting. [HBASE-8518] 6808 */ 6809 public static class HRegionForTesting extends HRegion { 6810 6811 public HRegionForTesting(final Path tableDir, final WAL wal, final FileSystem fs, 6812 final Configuration confParam, final RegionInfo regionInfo, final TableDescriptor htd, 6813 final RegionServerServices rsServices) { 6814 this(new HRegionFileSystem(confParam, fs, tableDir, regionInfo), wal, confParam, htd, 6815 rsServices); 6816 } 6817 6818 public HRegionForTesting(HRegionFileSystem fs, WAL wal, Configuration confParam, 6819 TableDescriptor htd, RegionServerServices rsServices) { 6820 super(fs, wal, confParam, htd, rsServices); 6821 } 6822 6823 /** 6824 * Create HStore instance. 6825 * @return If Mob is enabled, return HMobStore, otherwise return HStoreForTesting. 6826 */ 6827 @Override 6828 protected HStore instantiateHStore(final ColumnFamilyDescriptor family, boolean warmup) 6829 throws IOException { 6830 if (family.isMobEnabled()) { 6831 if (HFile.getFormatVersion(this.conf) < HFile.MIN_FORMAT_VERSION_WITH_TAGS) { 6832 throw new IOException("A minimum HFile version of " + HFile.MIN_FORMAT_VERSION_WITH_TAGS 6833 + " is required for MOB feature. Consider setting " + HFile.FORMAT_VERSION_KEY 6834 + " accordingly."); 6835 } 6836 return new HMobStore(this, family, this.conf, warmup); 6837 } 6838 return new HStoreForTesting(this, family, this.conf, warmup); 6839 } 6840 } 6841 6842 /** 6843 * HStoreForTesting is merely the same as HStore, the difference is in the doCompaction method of 6844 * HStoreForTesting there is a checkpoint "hbase.hstore.compaction.complete" which doesn't let 6845 * hstore compaction complete. In the former edition, this config is set in HStore class inside 6846 * compact method, though this is just for testing, otherwise it doesn't do any help. In 6847 * HBASE-8518, we try to get rid of all "hbase.hstore.compaction.complete" config (except for 6848 * testing code). 6849 */ 6850 public static class HStoreForTesting extends HStore { 6851 6852 protected HStoreForTesting(final HRegion region, final ColumnFamilyDescriptor family, 6853 final Configuration confParam, boolean warmup) throws IOException { 6854 super(region, family, confParam, warmup); 6855 } 6856 6857 @Override 6858 protected List<HStoreFile> doCompaction(CompactionRequestImpl cr, 6859 Collection<HStoreFile> filesToCompact, User user, long compactionStartTime, 6860 List<Path> newFiles) throws IOException { 6861 // let compaction incomplete. 6862 if (!this.conf.getBoolean("hbase.hstore.compaction.complete", true)) { 6863 LOG.warn("hbase.hstore.compaction.complete is set to false"); 6864 List<HStoreFile> sfs = new ArrayList<>(newFiles.size()); 6865 final boolean evictOnClose = 6866 getCacheConfig() != null ? getCacheConfig().shouldEvictOnClose() : true; 6867 for (Path newFile : newFiles) { 6868 // Create storefile around what we wrote with a reader on it. 6869 HStoreFile sf = storeEngine.createStoreFileAndReader(newFile); 6870 sf.closeStoreFile(evictOnClose); 6871 sfs.add(sf); 6872 } 6873 return sfs; 6874 } 6875 return super.doCompaction(cr, filesToCompact, user, compactionStartTime, newFiles); 6876 } 6877 } 6878 6879 @Test 6880 public void testCloseNoInterrupt() throws Exception { 6881 byte[] cf1 = Bytes.toBytes("CF1"); 6882 byte[][] families = { cf1 }; 6883 final int SLEEP_TIME = 10 * 1000; 6884 6885 Configuration conf = new Configuration(CONF); 6886 // Disable close thread interrupt and server abort behavior 6887 conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, false); 6888 conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, 1000); 6889 region = initHRegion(tableName, method, conf, families); 6890 6891 final CountDownLatch latch = new CountDownLatch(1); 6892 final AtomicBoolean holderInterrupted = new AtomicBoolean(); 6893 Thread holder = new Thread(new Runnable() { 6894 @Override 6895 public void run() { 6896 try { 6897 LOG.info("Starting region operation holder"); 6898 region.startRegionOperation(Operation.SCAN); 6899 latch.countDown(); 6900 try { 6901 Thread.sleep(SLEEP_TIME); 6902 } catch (InterruptedException e) { 6903 LOG.info("Interrupted"); 6904 holderInterrupted.set(true); 6905 } 6906 } catch (Exception e) { 6907 throw new RuntimeException(e); 6908 } finally { 6909 try { 6910 region.closeRegionOperation(); 6911 } catch (IOException e) { 6912 } 6913 LOG.info("Stopped region operation holder"); 6914 } 6915 } 6916 }); 6917 6918 holder.start(); 6919 latch.await(); 6920 HBaseTestingUtil.closeRegionAndWAL(region); 6921 region = null; 6922 holder.join(); 6923 6924 assertFalse(holderInterrupted.get(), "Region lock holder should not have been interrupted"); 6925 } 6926 6927 @Test 6928 public void testCloseInterrupt() throws Exception { 6929 byte[] cf1 = Bytes.toBytes("CF1"); 6930 byte[][] families = { cf1 }; 6931 final int SLEEP_TIME = 10 * 1000; 6932 6933 Configuration conf = new Configuration(CONF); 6934 // Enable close thread interrupt and server abort behavior 6935 conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, true); 6936 // Speed up the unit test, no need to wait default 10 seconds. 6937 conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, 1000); 6938 region = initHRegion(tableName, method, conf, families); 6939 6940 final CountDownLatch latch = new CountDownLatch(1); 6941 final AtomicBoolean holderInterrupted = new AtomicBoolean(); 6942 Thread holder = new Thread(new Runnable() { 6943 @Override 6944 public void run() { 6945 try { 6946 LOG.info("Starting region operation holder"); 6947 region.startRegionOperation(Operation.SCAN); 6948 latch.countDown(); 6949 try { 6950 Thread.sleep(SLEEP_TIME); 6951 } catch (InterruptedException e) { 6952 LOG.info("Interrupted"); 6953 holderInterrupted.set(true); 6954 } 6955 } catch (Exception e) { 6956 throw new RuntimeException(e); 6957 } finally { 6958 try { 6959 region.closeRegionOperation(); 6960 } catch (IOException e) { 6961 } 6962 LOG.info("Stopped region operation holder"); 6963 } 6964 } 6965 }); 6966 6967 holder.start(); 6968 latch.await(); 6969 region.close(); 6970 region = null; 6971 holder.join(); 6972 6973 assertTrue(holderInterrupted.get(), "Region lock holder was not interrupted"); 6974 } 6975 6976 @Test 6977 public void testCloseAbort() throws Exception { 6978 byte[] cf1 = Bytes.toBytes("CF1"); 6979 byte[][] families = { cf1 }; 6980 final int SLEEP_TIME = 10 * 1000; 6981 6982 Configuration conf = new Configuration(CONF); 6983 // Enable close thread interrupt and server abort behavior. 6984 conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, true); 6985 // Set the abort interval to a fraction of sleep time so we are guaranteed to be aborted. 6986 conf.setInt(HRegion.CLOSE_WAIT_TIME, SLEEP_TIME / 2); 6987 // Set the wait interval to a fraction of sleep time so we are guaranteed to be interrupted. 6988 conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, SLEEP_TIME / 4); 6989 region = initHRegion(tableName, method, conf, families); 6990 RegionServerServices rsServices = mock(RegionServerServices.class); 6991 when(rsServices.getServerName()).thenReturn(ServerName.valueOf("localhost", 1000, 1000)); 6992 region.rsServices = rsServices; 6993 6994 final CountDownLatch latch = new CountDownLatch(1); 6995 Thread holder = new Thread(new Runnable() { 6996 @Override 6997 public void run() { 6998 try { 6999 LOG.info("Starting region operation holder"); 7000 region.startRegionOperation(Operation.SCAN); 7001 latch.countDown(); 7002 // Hold the lock for SLEEP_TIME seconds no matter how many times we are interrupted. 7003 int timeRemaining = SLEEP_TIME; 7004 while (timeRemaining > 0) { 7005 long start = EnvironmentEdgeManager.currentTime(); 7006 try { 7007 Thread.sleep(timeRemaining); 7008 } catch (InterruptedException e) { 7009 LOG.info("Interrupted"); 7010 } 7011 long end = EnvironmentEdgeManager.currentTime(); 7012 timeRemaining -= end - start; 7013 if (timeRemaining < 0) { 7014 timeRemaining = 0; 7015 } 7016 if (timeRemaining > 0) { 7017 LOG.info("Sleeping again, remaining time " + timeRemaining + " ms"); 7018 } 7019 } 7020 } catch (Exception e) { 7021 throw new RuntimeException(e); 7022 } finally { 7023 try { 7024 region.closeRegionOperation(); 7025 } catch (IOException e) { 7026 } 7027 LOG.info("Stopped region operation holder"); 7028 } 7029 } 7030 }); 7031 7032 holder.start(); 7033 latch.await(); 7034 assertThrows(IOException.class, () -> region.close()); 7035 holder.join(); 7036 7037 // Verify the region tried to abort the server 7038 verify(rsServices, atLeast(1)).abort(anyString(), any()); 7039 } 7040 7041 @Test 7042 public void testInterruptProtection() throws Exception { 7043 byte[] cf1 = Bytes.toBytes("CF1"); 7044 byte[][] families = { cf1 }; 7045 final int SLEEP_TIME = 10 * 1000; 7046 7047 Configuration conf = new Configuration(CONF); 7048 // Enable close thread interrupt and server abort behavior. 7049 conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, true); 7050 conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, 1000); 7051 region = initHRegion(tableName, method, conf, families); 7052 7053 final CountDownLatch latch = new CountDownLatch(1); 7054 final AtomicBoolean holderInterrupted = new AtomicBoolean(); 7055 Thread holder = new Thread(new Runnable() { 7056 @Override 7057 public void run() { 7058 try { 7059 LOG.info("Starting region operation holder"); 7060 region.startRegionOperation(Operation.SCAN); 7061 LOG.info("Protecting against interrupts"); 7062 region.disableInterrupts(); 7063 try { 7064 latch.countDown(); 7065 try { 7066 Thread.sleep(SLEEP_TIME); 7067 } catch (InterruptedException e) { 7068 LOG.info("Interrupted"); 7069 holderInterrupted.set(true); 7070 } 7071 } finally { 7072 region.enableInterrupts(); 7073 } 7074 } catch (Exception e) { 7075 throw new RuntimeException(e); 7076 } finally { 7077 try { 7078 region.closeRegionOperation(); 7079 } catch (IOException e) { 7080 } 7081 LOG.info("Stopped region operation holder"); 7082 } 7083 } 7084 }); 7085 7086 holder.start(); 7087 latch.await(); 7088 region.close(); 7089 region = null; 7090 holder.join(); 7091 7092 assertFalse(holderInterrupted.get(), "Region lock holder should not have been interrupted"); 7093 } 7094 7095 @Test 7096 public void testRegionOnCoprocessorsChange() throws IOException { 7097 byte[] cf1 = Bytes.toBytes("CF1"); 7098 byte[][] families = { cf1 }; 7099 7100 Configuration conf = new Configuration(CONF); 7101 region = initHRegion(tableName, method, conf, families); 7102 assertNull(region.getCoprocessorHost()); 7103 7104 // set and verify the system coprocessors for region and user region 7105 Configuration newConf = new Configuration(conf); 7106 newConf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, MetaTableMetrics.class.getName()); 7107 newConf.set(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, 7108 NoOpRegionCoprocessor.class.getName()); 7109 // trigger configuration change 7110 region.onConfigurationChange(newConf); 7111 assertNotNull(region.getCoprocessorHost()); 7112 Set<String> coprocessors = region.getCoprocessorHost().getCoprocessors(); 7113 assertEquals(2, coprocessors.size()); 7114 assertTrue(region.getCoprocessorHost().getCoprocessors() 7115 .contains(MetaTableMetrics.class.getSimpleName())); 7116 assertTrue(region.getCoprocessorHost().getCoprocessors() 7117 .contains(NoOpRegionCoprocessor.class.getSimpleName())); 7118 7119 // remove region coprocessor and keep only user region coprocessor 7120 newConf.unset(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY); 7121 region.onConfigurationChange(newConf); 7122 assertNotNull(region.getCoprocessorHost()); 7123 coprocessors = region.getCoprocessorHost().getCoprocessors(); 7124 assertEquals(1, coprocessors.size()); 7125 assertTrue(region.getCoprocessorHost().getCoprocessors() 7126 .contains(NoOpRegionCoprocessor.class.getSimpleName())); 7127 } 7128 7129 @Test 7130 public void testRegionOnCoprocessorsWithoutChange() throws IOException { 7131 byte[] cf1 = Bytes.toBytes("CF1"); 7132 byte[][] families = { cf1 }; 7133 7134 Configuration conf = new Configuration(CONF); 7135 conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, 7136 MetaTableMetrics.class.getCanonicalName()); 7137 region = initHRegion(tableName, method, conf, families); 7138 // region service is null in unit test, we need to load the coprocessor once 7139 region.setCoprocessorHost(new RegionCoprocessorHost(region, null, conf)); 7140 RegionCoprocessor regionCoprocessor = 7141 region.getCoprocessorHost().findCoprocessor(MetaTableMetrics.class.getName()); 7142 7143 // simulate when other configuration may have changed and onConfigurationChange execute once 7144 region.onConfigurationChange(conf); 7145 RegionCoprocessor regionCoprocessorAfterOnConfigurationChange = 7146 region.getCoprocessorHost().findCoprocessor(MetaTableMetrics.class.getName()); 7147 assertEquals(regionCoprocessor, regionCoprocessorAfterOnConfigurationChange); 7148 } 7149 7150 public static class NoOpRegionCoprocessor implements RegionCoprocessor, RegionObserver { 7151 // a empty region coprocessor class 7152 } 7153 7154 /** 7155 * Test for HBASE-29662: HRegion.initialize() should fail when trying to recreate .regioninfo file 7156 * after the region directory has been deleted. This validates that .regioninfo file creation does 7157 * not create parent directories recursively. 7158 */ 7159 @Test 7160 public void testHRegionInitializeFailsWithDeletedRegionDir() throws Exception { 7161 LOG.info("Testing HRegion initialize failure with deleted region directory"); 7162 7163 TEST_UTIL = new HBaseTestingUtil(); 7164 Configuration conf = TEST_UTIL.getConfiguration(); 7165 Path testDir = TEST_UTIL.getDataTestDir("testHRegionInitFailure"); 7166 FileSystem fs = testDir.getFileSystem(conf); 7167 7168 // Create table descriptor 7169 TableName tableName = TableName.valueOf("TestHRegionInitWithDeletedDir"); 7170 byte[] family = Bytes.toBytes("info"); 7171 TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName) 7172 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build(); 7173 7174 // Create region info 7175 RegionInfo regionInfo = 7176 RegionInfoBuilder.newBuilder(tableName).setStartKey(null).setEndKey(null).build(); 7177 7178 Path tableDir = CommonFSUtils.getTableDir(testDir, tableName); 7179 7180 // Create WAL for the region 7181 WAL wal = HBaseTestingUtil.createWal(conf, testDir, regionInfo); 7182 7183 try { 7184 // Create region normally (this should succeed and create region directory) 7185 LOG.info("Creating region normally - should succeed"); 7186 HRegion region = HRegion.createHRegion(regionInfo, testDir, conf, htd, wal, true); 7187 7188 // Verify region directory exists 7189 Path regionDir = new Path(tableDir, regionInfo.getEncodedName()); 7190 assertTrue(fs.exists(regionDir), "Region directory should exist after creation"); 7191 7192 Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE); 7193 assertTrue(fs.exists(regionInfoFile), "Region info file should exist after creation"); 7194 7195 // Delete the region directory (simulating external deletion or corruption) 7196 assertTrue(fs.delete(regionDir, true)); 7197 assertFalse(fs.exists(regionDir), "Region directory should not exist after deletion"); 7198 7199 // Try to open/initialize the region again - this should fail 7200 LOG.info("Attempting to re-initialize region with deleted directory - should fail"); 7201 7202 // Create a new region instance (simulating region server restart or reopen) 7203 HRegion newRegion = HRegion.newHRegion(tableDir, wal, fs, conf, regionInfo, htd, null); 7204 // Try to initialize - this should fail because the regionDir doesn't exist 7205 IOException regionInitializeException = null; 7206 try { 7207 newRegion.initialize(null); 7208 } catch (IOException e) { 7209 regionInitializeException = e; 7210 } 7211 7212 // Verify the exception is related to missing parent directory 7213 assertNotNull(regionInitializeException, "Exception should be thrown"); 7214 String exceptionMessage = regionInitializeException.getMessage().toLowerCase(); 7215 assertTrue(exceptionMessage.contains("region directory does not exist")); 7216 assertFalse(fs.exists(regionDir), 7217 "Region directory should still not exist after failed initialization"); 7218 7219 } finally { 7220 if (wal != null) { 7221 wal.close(); 7222 } 7223 TEST_UTIL.cleanupTestDir(); 7224 } 7225 } 7226 7227 /** 7228 * A region can miss read-only coprocessors after repeated toggles if 7229 * {@code maybeUpdateCoprocessors} syncs to the server configuration instead of the region's 7230 * {@link org.apache.hadoop.hbase.CompoundConfiguration}. 7231 */ 7232 @Test 7233 public void testRegionOnConfigurationChangeLoadsReadOnlyCoprocessorsAfterRepeatedToggle() 7234 throws IOException { 7235 byte[] cf1 = Bytes.toBytes("CF1"); 7236 Configuration serverConf = new Configuration(CONF); 7237 serverConf.setBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY, false); 7238 7239 region = initHRegion(tableName, method, serverConf, new byte[][] { cf1 }); 7240 region.setCoprocessorHost(new RegionCoprocessorHost(region, null, region.getConfiguration())); 7241 7242 String regionCpKey = CoprocessorHost.REGION_COPROCESSOR_CONF_KEY; 7243 boolean[] toggleSequence = new boolean[] { true, false, true, true, false, false }; 7244 7245 for (boolean readOnlyEnabled : toggleSequence) { 7246 Configuration newConf = new Configuration(serverConf); 7247 newConf.setBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY, readOnlyEnabled); 7248 region.onConfigurationChange(newConf); 7249 7250 if (readOnlyEnabled) { 7251 assertNotNull( 7252 region.getCoprocessorHost().findCoprocessor(RegionReadOnlyController.class.getName())); 7253 } else { 7254 assertNull( 7255 region.getCoprocessorHost().findCoprocessor(RegionReadOnlyController.class.getName())); 7256 } 7257 assertEquals(readOnlyEnabled, CoprocessorConfigurationUtil 7258 .areReadOnlyCoprocessorsLoaded(region.getConfiguration(), regionCpKey)); 7259 } 7260 } 7261 7262 /** 7263 * Verifies the new RegionCoprocessorHost object created in HRegion.onConfigurationChange() uses a 7264 * CompoundConfiguration in its constructor rather than a Configuration object. 7265 */ 7266 @Test 7267 public void testRegionCoprocessorHostUsesCompoundConfigurationAfterConfigChange() 7268 throws Exception { 7269 byte[] cf1 = Bytes.toBytes("CF1"); 7270 Configuration serverConf = new Configuration(CONF); 7271 serverConf.setBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY, false); 7272 7273 region = initHRegion(tableName, method, serverConf, new byte[][] { cf1 }); 7274 region.setCoprocessorHost(new RegionCoprocessorHost(region, null, region.getConfiguration())); 7275 7276 // Enable read-only mode to trigger coprocessor reload and new RegionCoprocessorHost creation 7277 Configuration newConf = new Configuration(serverConf); 7278 newConf.setBoolean(HConstants.HBASE_GLOBAL_READONLY_ENABLED_KEY, true); 7279 region.onConfigurationChange(newConf); 7280 7281 // Use reflection to access the protected conf field on CoprocessorHost, which holds the 7282 // Configuration passed to the RegionCoprocessorHost constructor 7283 Field confField = CoprocessorHost.class.getDeclaredField("conf"); 7284 confField.setAccessible(true); 7285 Configuration hostConf = (Configuration) confField.get(region.getCoprocessorHost()); 7286 assertInstanceOf(CompoundConfiguration.class, hostConf); 7287 } 7288}