001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.apache.hadoop.hbase.HBaseTestingUtil.COLUMNS; 021import static org.apache.hadoop.hbase.HBaseTestingUtil.fam1; 022import static org.apache.hadoop.hbase.HBaseTestingUtil.fam2; 023import static org.apache.hadoop.hbase.HBaseTestingUtil.fam3; 024import static org.junit.Assert.assertArrayEquals; 025import static org.junit.Assert.assertEquals; 026import static org.junit.Assert.assertFalse; 027import static org.junit.Assert.assertNotNull; 028import static org.junit.Assert.assertNull; 029import static org.junit.Assert.assertThrows; 030import static org.junit.Assert.assertTrue; 031import static org.junit.Assert.fail; 032import static org.mockito.ArgumentMatchers.any; 033import static org.mockito.ArgumentMatchers.anyLong; 034import static org.mockito.ArgumentMatchers.anyString; 035import static org.mockito.ArgumentMatchers.isA; 036import static org.mockito.Mockito.atLeast; 037import static org.mockito.Mockito.doAnswer; 038import static org.mockito.Mockito.doThrow; 039import static org.mockito.Mockito.mock; 040import static org.mockito.Mockito.never; 041import static org.mockito.Mockito.spy; 042import static org.mockito.Mockito.times; 043import static org.mockito.Mockito.verify; 044import static org.mockito.Mockito.when; 045 046import java.io.IOException; 047import java.io.InterruptedIOException; 048import java.math.BigDecimal; 049import java.security.PrivilegedExceptionAction; 050import java.util.ArrayList; 051import java.util.Arrays; 052import java.util.Collection; 053import java.util.List; 054import java.util.Map; 055import java.util.NavigableMap; 056import java.util.Objects; 057import java.util.Set; 058import java.util.TreeMap; 059import java.util.concurrent.Callable; 060import java.util.concurrent.CountDownLatch; 061import java.util.concurrent.ExecutorService; 062import java.util.concurrent.Executors; 063import java.util.concurrent.Future; 064import java.util.concurrent.TimeUnit; 065import java.util.concurrent.atomic.AtomicBoolean; 066import java.util.concurrent.atomic.AtomicInteger; 067import java.util.concurrent.atomic.AtomicLong; 068import java.util.concurrent.atomic.AtomicReference; 069import java.util.stream.Collectors; 070import org.apache.commons.lang3.RandomStringUtils; 071import org.apache.hadoop.conf.Configuration; 072import org.apache.hadoop.fs.FSDataOutputStream; 073import org.apache.hadoop.fs.FileStatus; 074import org.apache.hadoop.fs.FileSystem; 075import org.apache.hadoop.fs.Path; 076import org.apache.hadoop.hbase.ArrayBackedTag; 077import org.apache.hadoop.hbase.Cell; 078import org.apache.hadoop.hbase.Cell.Type; 079import org.apache.hadoop.hbase.CellBuilderFactory; 080import org.apache.hadoop.hbase.CellBuilderType; 081import org.apache.hadoop.hbase.CellUtil; 082import org.apache.hadoop.hbase.CompareOperator; 083import org.apache.hadoop.hbase.CompatibilitySingletonFactory; 084import org.apache.hadoop.hbase.DoNotRetryIOException; 085import org.apache.hadoop.hbase.DroppedSnapshotException; 086import org.apache.hadoop.hbase.ExtendedCell; 087import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; 088import org.apache.hadoop.hbase.HBaseClassTestRule; 089import org.apache.hadoop.hbase.HBaseConfiguration; 090import org.apache.hadoop.hbase.HBaseTestingUtil; 091import org.apache.hadoop.hbase.HConstants; 092import org.apache.hadoop.hbase.HConstants.OperationStatusCode; 093import org.apache.hadoop.hbase.HDFSBlocksDistribution; 094import org.apache.hadoop.hbase.KeyValue; 095import org.apache.hadoop.hbase.MultithreadedTestUtil; 096import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread; 097import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread; 098import org.apache.hadoop.hbase.NotServingRegionException; 099import org.apache.hadoop.hbase.PrivateCellUtil; 100import org.apache.hadoop.hbase.RegionTooBusyException; 101import org.apache.hadoop.hbase.ServerName; 102import org.apache.hadoop.hbase.SingleProcessHBaseCluster; 103import org.apache.hadoop.hbase.StartTestingClusterOption; 104import org.apache.hadoop.hbase.TableName; 105import org.apache.hadoop.hbase.TagType; 106import org.apache.hadoop.hbase.Waiter; 107import org.apache.hadoop.hbase.client.Append; 108import org.apache.hadoop.hbase.client.CheckAndMutate; 109import org.apache.hadoop.hbase.client.CheckAndMutateResult; 110import org.apache.hadoop.hbase.client.ClientInternalHelper; 111import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 112import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 113import org.apache.hadoop.hbase.client.Delete; 114import org.apache.hadoop.hbase.client.Durability; 115import org.apache.hadoop.hbase.client.Get; 116import org.apache.hadoop.hbase.client.Increment; 117import org.apache.hadoop.hbase.client.Mutation; 118import org.apache.hadoop.hbase.client.Put; 119import org.apache.hadoop.hbase.client.RegionInfo; 120import org.apache.hadoop.hbase.client.RegionInfoBuilder; 121import org.apache.hadoop.hbase.client.Result; 122import org.apache.hadoop.hbase.client.RowMutations; 123import org.apache.hadoop.hbase.client.Scan; 124import org.apache.hadoop.hbase.client.Table; 125import org.apache.hadoop.hbase.client.TableDescriptor; 126import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 127import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; 128import org.apache.hadoop.hbase.coprocessor.MetaTableMetrics; 129import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor; 130import org.apache.hadoop.hbase.coprocessor.RegionObserver; 131import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException; 132import org.apache.hadoop.hbase.filter.BigDecimalComparator; 133import org.apache.hadoop.hbase.filter.BinaryComparator; 134import org.apache.hadoop.hbase.filter.ColumnCountGetFilter; 135import org.apache.hadoop.hbase.filter.Filter; 136import org.apache.hadoop.hbase.filter.FilterBase; 137import org.apache.hadoop.hbase.filter.FilterList; 138import org.apache.hadoop.hbase.filter.NullComparator; 139import org.apache.hadoop.hbase.filter.PrefixFilter; 140import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter; 141import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; 142import org.apache.hadoop.hbase.filter.SubstringComparator; 143import org.apache.hadoop.hbase.filter.ValueFilter; 144import org.apache.hadoop.hbase.io.TimeRange; 145import org.apache.hadoop.hbase.io.hfile.HFile; 146import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler; 147import org.apache.hadoop.hbase.monitoring.MonitoredTask; 148import org.apache.hadoop.hbase.monitoring.TaskMonitor; 149import org.apache.hadoop.hbase.regionserver.Region.Operation; 150import org.apache.hadoop.hbase.regionserver.Region.RowLock; 151import org.apache.hadoop.hbase.regionserver.TestHStore.FaultyFileSystem; 152import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl; 153import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; 154import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; 155import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL; 156import org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL; 157import org.apache.hadoop.hbase.regionserver.wal.FSHLog; 158import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource; 159import org.apache.hadoop.hbase.regionserver.wal.WALUtil; 160import org.apache.hadoop.hbase.replication.regionserver.ReplicationObserver; 161import org.apache.hadoop.hbase.security.User; 162import org.apache.hadoop.hbase.test.MetricsAssertHelper; 163import org.apache.hadoop.hbase.testclassification.LargeTests; 164import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; 165import org.apache.hadoop.hbase.util.Bytes; 166import org.apache.hadoop.hbase.util.CommonFSUtils; 167import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 168import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; 169import org.apache.hadoop.hbase.util.HFileArchiveUtil; 170import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge; 171import org.apache.hadoop.hbase.util.ManualEnvironmentEdge; 172import org.apache.hadoop.hbase.util.Threads; 173import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; 174import org.apache.hadoop.hbase.wal.FaultyFSLog; 175import org.apache.hadoop.hbase.wal.NettyAsyncFSWALConfigHelper; 176import org.apache.hadoop.hbase.wal.WAL; 177import org.apache.hadoop.hbase.wal.WALEdit; 178import org.apache.hadoop.hbase.wal.WALEditInternalHelper; 179import org.apache.hadoop.hbase.wal.WALFactory; 180import org.apache.hadoop.hbase.wal.WALKeyImpl; 181import org.apache.hadoop.hbase.wal.WALProvider; 182import org.apache.hadoop.hbase.wal.WALProvider.Writer; 183import org.apache.hadoop.hbase.wal.WALSplitUtil; 184import org.apache.hadoop.hbase.wal.WALStreamReader; 185import org.junit.After; 186import org.junit.Assert; 187import org.junit.Before; 188import org.junit.ClassRule; 189import org.junit.Ignore; 190import org.junit.Rule; 191import org.junit.Test; 192import org.junit.experimental.categories.Category; 193import org.junit.rules.ExpectedException; 194import org.junit.rules.TestName; 195import org.mockito.ArgumentCaptor; 196import org.mockito.ArgumentMatcher; 197import org.mockito.invocation.InvocationOnMock; 198import org.mockito.stubbing.Answer; 199import org.slf4j.Logger; 200import org.slf4j.LoggerFactory; 201 202import org.apache.hbase.thirdparty.com.google.common.collect.Lists; 203import org.apache.hbase.thirdparty.com.google.protobuf.ByteString; 204import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup; 205import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup; 206import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel; 207 208import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 209import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor; 210import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor; 211import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.FlushAction; 212import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor; 213import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor; 214import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.StoreDescriptor; 215 216/** 217 * Basic stand-alone testing of HRegion. No clusters! A lot of the meta information for an HRegion 218 * now lives inside other HRegions or in the HBaseMaster, so only basic testing is possible. 219 */ 220@Category({ VerySlowRegionServerTests.class, LargeTests.class }) 221@SuppressWarnings("deprecation") 222public class TestHRegion { 223 224 @ClassRule 225 public static final HBaseClassTestRule CLASS_RULE = 226 HBaseClassTestRule.forClass(TestHRegion.class); 227 228 // Do not spin up clusters in here. If you need to spin up a cluster, do it 229 // over in TestHRegionOnCluster. 230 private static final Logger LOG = LoggerFactory.getLogger(TestHRegion.class); 231 @Rule 232 public TestName name = new TestName(); 233 @Rule 234 public final ExpectedException thrown = ExpectedException.none(); 235 236 private static final String COLUMN_FAMILY = "MyCF"; 237 private static final byte[] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY); 238 private static final EventLoopGroup GROUP = new NioEventLoopGroup(); 239 240 HRegion region = null; 241 // Do not run unit tests in parallel (? Why not? It don't work? Why not? St.Ack) 242 protected static HBaseTestingUtil TEST_UTIL; 243 public static Configuration CONF; 244 private String dir; 245 private final int MAX_VERSIONS = 2; 246 247 // Test names 248 protected TableName tableName; 249 protected String method; 250 protected final byte[] qual = Bytes.toBytes("qual"); 251 protected final byte[] qual1 = Bytes.toBytes("qual1"); 252 protected final byte[] qual2 = Bytes.toBytes("qual2"); 253 protected final byte[] qual3 = Bytes.toBytes("qual3"); 254 protected final byte[] value = Bytes.toBytes("value"); 255 protected final byte[] value1 = Bytes.toBytes("value1"); 256 protected final byte[] value2 = Bytes.toBytes("value2"); 257 protected final byte[] row = Bytes.toBytes("rowA"); 258 protected final byte[] row2 = Bytes.toBytes("rowB"); 259 260 protected final MetricsAssertHelper metricsAssertHelper = 261 CompatibilitySingletonFactory.getInstance(MetricsAssertHelper.class); 262 263 @Before 264 public void setup() throws IOException { 265 TEST_UTIL = new HBaseTestingUtil(); 266 CONF = TEST_UTIL.getConfiguration(); 267 NettyAsyncFSWALConfigHelper.setEventLoopConfig(CONF, GROUP, NioSocketChannel.class); 268 dir = TEST_UTIL.getDataTestDir("TestHRegion").toString(); 269 method = name.getMethodName(); 270 tableName = TableName.valueOf(method); 271 CONF.set(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, String.valueOf(0.09)); 272 CONF.setLong(AbstractFSWAL.WAL_SYNC_TIMEOUT_MS, 10000); 273 } 274 275 @After 276 public void tearDown() throws IOException { 277 // Region may have been closed, but it is still no harm if we close it again here using HTU. 278 HBaseTestingUtil.closeRegionAndWAL(region); 279 EnvironmentEdgeManagerTestHelper.reset(); 280 LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir()); 281 TEST_UTIL.cleanupTestDir(); 282 } 283 284 /** 285 * Test that I can use the max flushed sequence id after the close. 286 */ 287 @Test 288 public void testSequenceId() throws IOException { 289 region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES); 290 assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId()); 291 // Weird. This returns 0 if no store files or no edits. Afraid to change it. 292 assertEquals(0, (long) region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES)); 293 HBaseTestingUtil.closeRegionAndWAL(this.region); 294 assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId()); 295 assertEquals(0, (long) region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES)); 296 HRegion oldRegion = region; 297 try { 298 // Open region again. 299 region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES); 300 byte[] value = Bytes.toBytes(method); 301 // Make a random put against our cf. 302 Put put = new Put(value); 303 put.addColumn(COLUMN_FAMILY_BYTES, null, value); 304 region.put(put); 305 // No flush yet so init numbers should still be in place. 306 assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId()); 307 assertEquals(0, (long) region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES)); 308 region.flush(true); 309 long max = region.getMaxFlushedSeqId(); 310 HBaseTestingUtil.closeRegionAndWAL(this.region); 311 assertEquals(max, region.getMaxFlushedSeqId()); 312 this.region = null; 313 } finally { 314 HBaseTestingUtil.closeRegionAndWAL(oldRegion); 315 } 316 } 317 318 /** 319 * Test for Bug 2 of HBASE-10466. "Bug 2: Conditions for the first flush of region close 320 * (so-called pre-flush) If memstoreSize is smaller than a certain value, or when region close 321 * starts a flush is ongoing, the first flush is skipped and only the second flush takes place. 322 * However, two flushes are required in case previous flush fails and leaves some data in 323 * snapshot. The bug could cause loss of data in current memstore. The fix is removing all 324 * conditions except abort check so we ensure 2 flushes for region close." 325 */ 326 @Test 327 public void testCloseCarryingSnapshot() throws IOException { 328 region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES); 329 HStore store = region.getStore(COLUMN_FAMILY_BYTES); 330 // Get some random bytes. 331 byte[] value = Bytes.toBytes(method); 332 // Make a random put against our cf. 333 Put put = new Put(value); 334 put.addColumn(COLUMN_FAMILY_BYTES, null, value); 335 // First put something in current memstore, which will be in snapshot after flusher.prepare() 336 region.put(put); 337 StoreFlushContext storeFlushCtx = store.createFlushContext(12345, FlushLifeCycleTracker.DUMMY); 338 storeFlushCtx.prepare(); 339 // Second put something in current memstore 340 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value); 341 region.put(put); 342 // Close with something in memstore and something in the snapshot. Make sure all is cleared. 343 HBaseTestingUtil.closeRegionAndWAL(region); 344 assertEquals(0, region.getMemStoreDataSize()); 345 region = null; 346 } 347 348 /* 349 * This test is for verifying memstore snapshot size is correctly updated in case of rollback See 350 * HBASE-10845 351 */ 352 @Test 353 public void testMemstoreSnapshotSize() throws IOException { 354 class MyFaultyFSLog extends FaultyFSLog { 355 StoreFlushContext storeFlushCtx; 356 357 public MyFaultyFSLog(FileSystem fs, Path rootDir, String logName, Configuration conf) 358 throws IOException { 359 super(fs, rootDir, logName, conf); 360 } 361 362 void setStoreFlushCtx(StoreFlushContext storeFlushCtx) { 363 this.storeFlushCtx = storeFlushCtx; 364 } 365 366 @Override 367 protected void doSync(long txid, boolean forceSync) throws IOException { 368 storeFlushCtx.prepare(); 369 super.doSync(txid, forceSync); 370 } 371 } 372 373 FileSystem fs = FileSystem.get(CONF); 374 Path rootDir = new Path(dir + "testMemstoreSnapshotSize"); 375 MyFaultyFSLog faultyLog = new MyFaultyFSLog(fs, rootDir, "testMemstoreSnapshotSize", CONF); 376 faultyLog.init(); 377 region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, faultyLog, 378 COLUMN_FAMILY_BYTES); 379 380 HStore store = region.getStore(COLUMN_FAMILY_BYTES); 381 // Get some random bytes. 382 byte[] value = Bytes.toBytes(method); 383 faultyLog.setStoreFlushCtx(store.createFlushContext(12345, FlushLifeCycleTracker.DUMMY)); 384 385 Put put = new Put(value); 386 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value); 387 faultyLog.setFailureType(FaultyFSLog.FailureType.SYNC); 388 boolean threwIOE = false; 389 try { 390 region.put(put); 391 } catch (IOException ioe) { 392 threwIOE = true; 393 } finally { 394 assertTrue("The regionserver should have thrown an exception", threwIOE); 395 } 396 MemStoreSize mss = store.getFlushableSize(); 397 assertTrue("flushable size should be zero, but it is " + mss, mss.getDataSize() == 0); 398 } 399 400 /** 401 * Create a WAL outside of the usual helper in 402 * {@link HBaseTestingUtil#createWal(Configuration, Path, RegionInfo)} because that method doesn't 403 * play nicely with FaultyFileSystem. Call this method before overriding {@code fs.file.impl}. 404 * @param callingMethod a unique component for the path, probably the name of the test method. 405 */ 406 private static WAL createWALCompatibleWithFaultyFileSystem(String callingMethod, 407 Configuration conf, TableName tableName) throws IOException { 408 final Path logDir = TEST_UTIL.getDataTestDirOnTestFS(callingMethod + ".log"); 409 final Configuration walConf = new Configuration(conf); 410 CommonFSUtils.setRootDir(walConf, logDir); 411 return new WALFactory(walConf, callingMethod) 412 .getWAL(RegionInfoBuilder.newBuilder(tableName).build()); 413 } 414 415 @Test 416 public void testMemstoreSizeAccountingWithFailedPostBatchMutate() throws IOException { 417 String testName = "testMemstoreSizeAccountingWithFailedPostBatchMutate"; 418 FileSystem fs = FileSystem.get(CONF); 419 Path rootDir = new Path(dir + testName); 420 FSHLog hLog = new FSHLog(fs, rootDir, testName, CONF); 421 hLog.init(); 422 region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, hLog, 423 COLUMN_FAMILY_BYTES); 424 HStore store = region.getStore(COLUMN_FAMILY_BYTES); 425 assertEquals(0, region.getMemStoreDataSize()); 426 427 // Put one value 428 byte[] value = Bytes.toBytes(method); 429 Put put = new Put(value); 430 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value); 431 region.put(put); 432 long onePutSize = region.getMemStoreDataSize(); 433 assertTrue(onePutSize > 0); 434 435 RegionCoprocessorHost mockedCPHost = mock(RegionCoprocessorHost.class); 436 doThrow(new IOException()).when(mockedCPHost).postBatchMutate(any()); 437 region.setCoprocessorHost(mockedCPHost); 438 439 put = new Put(value); 440 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("dfg"), value); 441 try { 442 region.put(put); 443 fail("Should have failed with IOException"); 444 } catch (IOException expected) { 445 } 446 long expectedSize = onePutSize * 2; 447 assertEquals("memstoreSize should be incremented", expectedSize, region.getMemStoreDataSize()); 448 assertEquals("flushable size should be incremented", expectedSize, 449 store.getFlushableSize().getDataSize()); 450 451 region.setCoprocessorHost(null); 452 } 453 454 /** 455 * A test case of HBASE-21041 456 */ 457 @Test 458 public void testFlushAndMemstoreSizeCounting() throws Exception { 459 byte[] family = Bytes.toBytes("family"); 460 this.region = initHRegion(tableName, method, CONF, family); 461 for (byte[] row : HBaseTestingUtil.ROWS) { 462 Put put = new Put(row); 463 put.addColumn(family, family, row); 464 region.put(put); 465 } 466 region.flush(true); 467 // After flush, data size should be zero 468 assertEquals(0, region.getMemStoreDataSize()); 469 // After flush, a new active mutable segment is created, so the heap size 470 // should equal to MutableSegment.DEEP_OVERHEAD 471 assertEquals(MutableSegment.DEEP_OVERHEAD, region.getMemStoreHeapSize()); 472 // After flush, offheap should be zero 473 assertEquals(0, region.getMemStoreOffHeapSize()); 474 } 475 476 /** 477 * Test we do not lose data if we fail a flush and then close. Part of HBase-10466. Tests the 478 * following from the issue description: "Bug 1: Wrong calculation of HRegion.memstoreSize: When a 479 * flush fails, data to be flushed is kept in each MemStore's snapshot and wait for next flush 480 * attempt to continue on it. But when the next flush succeeds, the counter of total memstore size 481 * in HRegion is always deduced by the sum of current memstore sizes instead of snapshots left 482 * from previous failed flush. This calculation is problematic that almost every time there is 483 * failed flush, HRegion.memstoreSize gets reduced by a wrong value. If region flush could not 484 * proceed for a couple cycles, the size in current memstore could be much larger than the 485 * snapshot. It's likely to drift memstoreSize much smaller than expected. In extreme case, if the 486 * error accumulates to even bigger than HRegion's memstore size limit, any further flush is 487 * skipped because flush does not do anything if memstoreSize is not larger than 0." 488 */ 489 @Test 490 public void testFlushSizeAccounting() throws Exception { 491 final Configuration conf = HBaseConfiguration.create(CONF); 492 final WAL wal = createWALCompatibleWithFaultyFileSystem(method, conf, tableName); 493 // Only retry once. 494 conf.setInt("hbase.hstore.flush.retries.number", 1); 495 final User user = User.createUserForTesting(conf, method, new String[] { "foo" }); 496 // Inject our faulty LocalFileSystem 497 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class); 498 user.runAs(new PrivilegedExceptionAction<Object>() { 499 @Override 500 public Object run() throws Exception { 501 // Make sure it worked (above is sensitive to caching details in hadoop core) 502 FileSystem fs = FileSystem.get(conf); 503 Assert.assertEquals(FaultyFileSystem.class, fs.getClass()); 504 FaultyFileSystem ffs = (FaultyFileSystem) fs; 505 HRegion region = null; 506 try { 507 // Initialize region 508 region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, wal, 509 COLUMN_FAMILY_BYTES); 510 long size = region.getMemStoreDataSize(); 511 Assert.assertEquals(0, size); 512 // Put one item into memstore. Measure the size of one item in memstore. 513 Put p1 = new Put(row); 514 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[]) null)); 515 region.put(p1); 516 final long sizeOfOnePut = region.getMemStoreDataSize(); 517 // Fail a flush which means the current memstore will hang out as memstore 'snapshot'. 518 try { 519 LOG.info("Flushing"); 520 region.flush(true); 521 Assert.fail("Didn't bubble up IOE!"); 522 } catch (DroppedSnapshotException dse) { 523 // What we are expecting 524 region.closing.set(false); // this is needed for the rest of the test to work 525 } 526 // Make it so all writes succeed from here on out 527 ffs.fault.set(false); 528 // Check sizes. Should still be the one entry. 529 Assert.assertEquals(sizeOfOnePut, region.getMemStoreDataSize()); 530 // Now add two entries so that on this next flush that fails, we can see if we 531 // subtract the right amount, the snapshot size only. 532 Put p2 = new Put(row); 533 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[]) null)); 534 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[]) null)); 535 region.put(p2); 536 long expectedSize = sizeOfOnePut * 3; 537 Assert.assertEquals(expectedSize, region.getMemStoreDataSize()); 538 // Do a successful flush. It will clear the snapshot only. Thats how flushes work. 539 // If already a snapshot, we clear it else we move the memstore to be snapshot and flush 540 // it 541 region.flush(true); 542 // Make sure our memory accounting is right. 543 Assert.assertEquals(sizeOfOnePut * 2, region.getMemStoreDataSize()); 544 } finally { 545 HBaseTestingUtil.closeRegionAndWAL(region); 546 } 547 return null; 548 } 549 }); 550 FileSystem.closeAllForUGI(user.getUGI()); 551 } 552 553 @Test 554 public void testCloseWithFailingFlush() throws Exception { 555 final Configuration conf = HBaseConfiguration.create(CONF); 556 final WAL wal = createWALCompatibleWithFaultyFileSystem(method, conf, tableName); 557 // Only retry once. 558 conf.setInt("hbase.hstore.flush.retries.number", 1); 559 final User user = User.createUserForTesting(conf, this.method, new String[] { "foo" }); 560 // Inject our faulty LocalFileSystem 561 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class); 562 user.runAs(new PrivilegedExceptionAction<Object>() { 563 @Override 564 public Object run() throws Exception { 565 // Make sure it worked (above is sensitive to caching details in hadoop core) 566 FileSystem fs = FileSystem.get(conf); 567 Assert.assertEquals(FaultyFileSystem.class, fs.getClass()); 568 FaultyFileSystem ffs = (FaultyFileSystem) fs; 569 HRegion region = null; 570 try { 571 // Initialize region 572 region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, wal, 573 COLUMN_FAMILY_BYTES); 574 long size = region.getMemStoreDataSize(); 575 Assert.assertEquals(0, size); 576 // Put one item into memstore. Measure the size of one item in memstore. 577 Put p1 = new Put(row); 578 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[]) null)); 579 region.put(p1); 580 // Manufacture an outstanding snapshot -- fake a failed flush by doing prepare step only. 581 HStore store = region.getStore(COLUMN_FAMILY_BYTES); 582 StoreFlushContext storeFlushCtx = 583 store.createFlushContext(12345, FlushLifeCycleTracker.DUMMY); 584 storeFlushCtx.prepare(); 585 // Now add two entries to the foreground memstore. 586 Put p2 = new Put(row); 587 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[]) null)); 588 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[]) null)); 589 region.put(p2); 590 // Now try close on top of a failing flush. 591 HBaseTestingUtil.closeRegionAndWAL(region); 592 region = null; 593 fail(); 594 } catch (DroppedSnapshotException dse) { 595 // Expected 596 LOG.info("Expected DroppedSnapshotException"); 597 } finally { 598 // Make it so all writes succeed from here on out so can close clean 599 ffs.fault.set(false); 600 HBaseTestingUtil.closeRegionAndWAL(region); 601 } 602 return null; 603 } 604 }); 605 FileSystem.closeAllForUGI(user.getUGI()); 606 } 607 608 @Test 609 public void testCompactionAffectedByScanners() throws Exception { 610 byte[] family = Bytes.toBytes("family"); 611 this.region = initHRegion(tableName, method, CONF, family); 612 613 Put put = new Put(Bytes.toBytes("r1")); 614 put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); 615 region.put(put); 616 region.flush(true); 617 618 Scan scan = new Scan(); 619 scan.readVersions(3); 620 // open the first scanner 621 try (RegionScanner scanner1 = region.getScanner(scan)) { 622 Delete delete = new Delete(Bytes.toBytes("r1")); 623 region.delete(delete); 624 region.flush(true); 625 // open the second scanner 626 try (RegionScanner scanner2 = region.getScanner(scan)) { 627 List<Cell> results = new ArrayList<>(); 628 629 LOG.info("Smallest read point:" + region.getSmallestReadPoint()); 630 631 // make a major compaction 632 region.compact(true); 633 634 // open the third scanner 635 try (RegionScanner scanner3 = region.getScanner(scan)) { 636 // get data from scanner 1, 2, 3 after major compaction 637 scanner1.next(results); 638 LOG.info(results.toString()); 639 assertEquals(1, results.size()); 640 641 results.clear(); 642 scanner2.next(results); 643 LOG.info(results.toString()); 644 assertEquals(0, results.size()); 645 646 results.clear(); 647 scanner3.next(results); 648 LOG.info(results.toString()); 649 assertEquals(0, results.size()); 650 } 651 } 652 } 653 } 654 655 @Test 656 public void testToShowNPEOnRegionScannerReseek() throws Exception { 657 byte[] family = Bytes.toBytes("family"); 658 this.region = initHRegion(tableName, method, CONF, family); 659 660 Put put = new Put(Bytes.toBytes("r1")); 661 put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); 662 region.put(put); 663 put = new Put(Bytes.toBytes("r2")); 664 put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); 665 region.put(put); 666 region.flush(true); 667 668 Scan scan = new Scan(); 669 scan.readVersions(3); 670 // open the first scanner 671 try (RegionScanner scanner1 = region.getScanner(scan)) { 672 LOG.info("Smallest read point:" + region.getSmallestReadPoint()); 673 674 region.compact(true); 675 676 scanner1.reseek(Bytes.toBytes("r2")); 677 List<Cell> results = new ArrayList<>(); 678 scanner1.next(results); 679 Cell keyValue = results.get(0); 680 assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0); 681 scanner1.close(); 682 } 683 } 684 685 @Test 686 public void testArchiveRecoveredEditsReplay() throws Exception { 687 byte[] family = Bytes.toBytes("family"); 688 this.region = initHRegion(tableName, method, CONF, family); 689 final WALFactory wals = new WALFactory(CONF, method); 690 try { 691 Path regiondir = region.getRegionFileSystem().getRegionDir(); 692 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 693 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 694 695 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 696 697 long maxSeqId = 1050; 698 long minSeqId = 1000; 699 700 for (long i = minSeqId; i <= maxSeqId; i += 10) { 701 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); 702 fs.create(recoveredEdits); 703 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 704 705 long time = System.nanoTime(); 706 WALEdit edit = new WALEdit(); 707 WALEditInternalHelper.addExtendedCell(edit, 708 new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i))); 709 writer.append(new WAL.Entry( 710 new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); 711 712 writer.close(); 713 } 714 MonitoredTask status = TaskMonitor.get().createStatus(method); 715 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 716 for (HStore store : region.getStores()) { 717 maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId - 1); 718 } 719 CONF.set("hbase.region.archive.recovered.edits", "true"); 720 CONF.set(CommonFSUtils.HBASE_WAL_DIR, "/custom_wal_dir"); 721 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status); 722 assertEquals(maxSeqId, seqId); 723 region.getMVCC().advanceTo(seqId); 724 String fakeFamilyName = recoveredEditsDir.getName(); 725 Path rootDir = new Path(CONF.get(HConstants.HBASE_DIR)); 726 Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePathForRootDir(rootDir, 727 region.getRegionInfo(), Bytes.toBytes(fakeFamilyName)); 728 FileStatus[] list = TEST_UTIL.getTestFileSystem().listStatus(storeArchiveDir); 729 assertEquals(6, list.length); 730 } finally { 731 CONF.set("hbase.region.archive.recovered.edits", "false"); 732 CONF.set(CommonFSUtils.HBASE_WAL_DIR, ""); 733 HBaseTestingUtil.closeRegionAndWAL(this.region); 734 this.region = null; 735 wals.close(); 736 } 737 } 738 739 @Test 740 public void testSkipRecoveredEditsReplay() throws Exception { 741 byte[] family = Bytes.toBytes("family"); 742 this.region = initHRegion(tableName, method, CONF, family); 743 final WALFactory wals = new WALFactory(CONF, method); 744 try { 745 Path regiondir = region.getRegionFileSystem().getRegionDir(); 746 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 747 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 748 749 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 750 751 long maxSeqId = 1050; 752 long minSeqId = 1000; 753 754 for (long i = minSeqId; i <= maxSeqId; i += 10) { 755 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); 756 fs.create(recoveredEdits); 757 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 758 759 long time = System.nanoTime(); 760 WALEdit edit = new WALEdit(); 761 WALEditInternalHelper.addExtendedCell(edit, 762 new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i))); 763 writer.append(new WAL.Entry( 764 new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); 765 766 writer.close(); 767 } 768 MonitoredTask status = TaskMonitor.get().createStatus(method); 769 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 770 for (HStore store : region.getStores()) { 771 maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId - 1); 772 } 773 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status); 774 assertEquals(maxSeqId, seqId); 775 region.getMVCC().advanceTo(seqId); 776 Get get = new Get(row); 777 Result result = region.get(get); 778 for (long i = minSeqId; i <= maxSeqId; i += 10) { 779 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i)); 780 assertEquals(1, kvs.size()); 781 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0))); 782 } 783 } finally { 784 HBaseTestingUtil.closeRegionAndWAL(this.region); 785 this.region = null; 786 wals.close(); 787 } 788 } 789 790 @Test 791 public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception { 792 byte[] family = Bytes.toBytes("family"); 793 this.region = initHRegion(tableName, method, CONF, family); 794 final WALFactory wals = new WALFactory(CONF, method); 795 try { 796 Path regiondir = region.getRegionFileSystem().getRegionDir(); 797 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 798 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 799 800 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 801 802 long maxSeqId = 1050; 803 long minSeqId = 1000; 804 805 for (long i = minSeqId; i <= maxSeqId; i += 10) { 806 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); 807 fs.create(recoveredEdits); 808 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 809 810 long time = System.nanoTime(); 811 WALEdit edit = new WALEdit(); 812 WALEditInternalHelper.addExtendedCell(edit, 813 new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i))); 814 writer.append(new WAL.Entry( 815 new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); 816 817 writer.close(); 818 } 819 long recoverSeqId = 1030; 820 MonitoredTask status = TaskMonitor.get().createStatus(method); 821 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 822 for (HStore store : region.getStores()) { 823 maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), recoverSeqId - 1); 824 } 825 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status); 826 assertEquals(maxSeqId, seqId); 827 region.getMVCC().advanceTo(seqId); 828 Get get = new Get(row); 829 Result result = region.get(get); 830 for (long i = minSeqId; i <= maxSeqId; i += 10) { 831 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i)); 832 if (i < recoverSeqId) { 833 assertEquals(0, kvs.size()); 834 } else { 835 assertEquals(1, kvs.size()); 836 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0))); 837 } 838 } 839 } finally { 840 HBaseTestingUtil.closeRegionAndWAL(this.region); 841 this.region = null; 842 wals.close(); 843 } 844 } 845 846 @Test 847 public void testSkipRecoveredEditsReplayAllIgnored() throws Exception { 848 byte[] family = Bytes.toBytes("family"); 849 this.region = initHRegion(tableName, method, CONF, family); 850 Path regiondir = region.getRegionFileSystem().getRegionDir(); 851 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 852 853 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 854 for (int i = 1000; i < 1050; i += 10) { 855 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); 856 FSDataOutputStream dos = fs.create(recoveredEdits); 857 dos.writeInt(i); 858 dos.close(); 859 } 860 long minSeqId = 2000; 861 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1)); 862 FSDataOutputStream dos = fs.create(recoveredEdits); 863 dos.close(); 864 865 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 866 for (HStore store : region.getStores()) { 867 maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId); 868 } 869 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, null); 870 assertEquals(minSeqId, seqId); 871 } 872 873 @Test 874 public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception { 875 byte[] family = Bytes.toBytes("family"); 876 this.region = initHRegion(tableName, method, CONF, family); 877 final WALFactory wals = new WALFactory(CONF, method); 878 try { 879 Path regiondir = region.getRegionFileSystem().getRegionDir(); 880 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 881 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 882 byte[][] columns = region.getTableDescriptor().getColumnFamilyNames().toArray(new byte[0][]); 883 884 assertEquals(0, region.getStoreFileList(columns).size()); 885 886 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 887 888 long maxSeqId = 1050; 889 long minSeqId = 1000; 890 891 for (long i = minSeqId; i <= maxSeqId; i += 10) { 892 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); 893 fs.create(recoveredEdits); 894 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 895 896 long time = System.nanoTime(); 897 WALEdit edit = null; 898 if (i == maxSeqId) { 899 edit = WALEdit.createCompaction(region.getRegionInfo(), 900 CompactionDescriptor.newBuilder().setTableName(ByteString.copyFrom(tableName.getName())) 901 .setFamilyName(ByteString.copyFrom(regionName)) 902 .setEncodedRegionName(ByteString.copyFrom(regionName)) 903 .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString()))) 904 .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName())).build()); 905 } else { 906 edit = new WALEdit(); 907 WALEditInternalHelper.addExtendedCell(edit, 908 new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i))); 909 } 910 writer.append(new WAL.Entry( 911 new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); 912 writer.close(); 913 } 914 915 long recoverSeqId = 1030; 916 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 917 MonitoredTask status = TaskMonitor.get().createStatus(method); 918 for (HStore store : region.getStores()) { 919 maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), recoverSeqId - 1); 920 } 921 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status); 922 assertEquals(maxSeqId, seqId); 923 924 // assert that the files are flushed 925 assertEquals(1, region.getStoreFileList(columns).size()); 926 927 } finally { 928 HBaseTestingUtil.closeRegionAndWAL(this.region); 929 this.region = null; 930 wals.close(); 931 } 932 } 933 934 @Test 935 public void testRecoveredEditsReplayCompaction() throws Exception { 936 testRecoveredEditsReplayCompaction(false); 937 testRecoveredEditsReplayCompaction(true); 938 } 939 940 public void testRecoveredEditsReplayCompaction(boolean mismatchedRegionName) throws Exception { 941 CONF.setClass(HConstants.REGION_IMPL, HRegionForTesting.class, Region.class); 942 byte[] family = Bytes.toBytes("family"); 943 this.region = initHRegion(tableName, method, CONF, family); 944 final WALFactory wals = new WALFactory(CONF, method); 945 try { 946 Path regiondir = region.getRegionFileSystem().getRegionDir(); 947 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 948 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 949 950 long maxSeqId = 3; 951 long minSeqId = 0; 952 953 for (long i = minSeqId; i < maxSeqId; i++) { 954 Put put = new Put(Bytes.toBytes(i)); 955 put.addColumn(family, Bytes.toBytes(i), Bytes.toBytes(i)); 956 region.put(put); 957 region.flush(true); 958 } 959 960 // this will create a region with 3 files 961 assertEquals(3, region.getStore(family).getStorefilesCount()); 962 List<Path> storeFiles = new ArrayList<>(3); 963 for (HStoreFile sf : region.getStore(family).getStorefiles()) { 964 storeFiles.add(sf.getPath()); 965 } 966 967 // disable compaction completion 968 CONF.setBoolean("hbase.hstore.compaction.complete", false); 969 region.compactStores(); 970 971 // ensure that nothing changed 972 assertEquals(3, region.getStore(family).getStorefilesCount()); 973 974 // now find the compacted file, and manually add it to the recovered edits 975 Path tmpDir = new Path(region.getRegionFileSystem().getTempDir(), Bytes.toString(family)); 976 FileStatus[] files = CommonFSUtils.listStatus(fs, tmpDir); 977 String errorMsg = "Expected to find 1 file in the region temp directory " 978 + "from the compaction, could not find any"; 979 assertNotNull(errorMsg, files); 980 assertEquals(errorMsg, 1, files.length); 981 // move the file inside region dir 982 Path newFile = 983 region.getRegionFileSystem().commitStoreFile(Bytes.toString(family), files[0].getPath()); 984 985 byte[] encodedNameAsBytes = this.region.getRegionInfo().getEncodedNameAsBytes(); 986 byte[] fakeEncodedNameAsBytes = new byte[encodedNameAsBytes.length]; 987 for (int i = 0; i < encodedNameAsBytes.length; i++) { 988 // Mix the byte array to have a new encodedName 989 fakeEncodedNameAsBytes[i] = (byte) (encodedNameAsBytes[i] + 1); 990 } 991 992 CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor( 993 this.region.getRegionInfo(), mismatchedRegionName ? fakeEncodedNameAsBytes : null, family, 994 storeFiles, Lists.newArrayList(newFile), 995 region.getRegionFileSystem().getStoreDir(Bytes.toString(family))); 996 997 WALUtil.writeCompactionMarker(region.getWAL(), this.region.getReplicationScope(), 998 this.region.getRegionInfo(), compactionDescriptor, region.getMVCC(), null); 999 1000 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 1001 1002 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000)); 1003 fs.create(recoveredEdits); 1004 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 1005 1006 long time = System.nanoTime(); 1007 1008 writer.append(new WAL.Entry( 1009 new WALKeyImpl(regionName, tableName, 10, time, HConstants.DEFAULT_CLUSTER_ID), 1010 WALEdit.createCompaction(region.getRegionInfo(), compactionDescriptor))); 1011 writer.close(); 1012 1013 // close the region now, and reopen again 1014 region.getTableDescriptor(); 1015 region.getRegionInfo(); 1016 HBaseTestingUtil.closeRegionAndWAL(this.region); 1017 try { 1018 region = HRegion.openHRegion(region, null); 1019 } catch (WrongRegionException wre) { 1020 fail("Matching encoded region name should not have produced WrongRegionException"); 1021 } 1022 1023 // now check whether we have only one store file, the compacted one 1024 Collection<HStoreFile> sfs = region.getStore(family).getStorefiles(); 1025 for (HStoreFile sf : sfs) { 1026 LOG.info(Objects.toString(sf.getPath())); 1027 } 1028 if (!mismatchedRegionName) { 1029 assertEquals(1, region.getStore(family).getStorefilesCount()); 1030 } 1031 files = CommonFSUtils.listStatus(fs, tmpDir); 1032 assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0); 1033 1034 for (long i = minSeqId; i < maxSeqId; i++) { 1035 Get get = new Get(Bytes.toBytes(i)); 1036 Result result = region.get(get); 1037 byte[] value = result.getValue(family, Bytes.toBytes(i)); 1038 assertArrayEquals(Bytes.toBytes(i), value); 1039 } 1040 } finally { 1041 HBaseTestingUtil.closeRegionAndWAL(this.region); 1042 this.region = null; 1043 wals.close(); 1044 CONF.setClass(HConstants.REGION_IMPL, HRegion.class, Region.class); 1045 } 1046 } 1047 1048 @Test 1049 public void testFlushMarkers() throws Exception { 1050 // tests that flush markers are written to WAL and handled at recovered edits 1051 byte[] family = Bytes.toBytes("family"); 1052 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + ".log"); 1053 final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration()); 1054 CommonFSUtils.setRootDir(walConf, logDir); 1055 final WALFactory wals = new WALFactory(walConf, method); 1056 final WAL wal = wals.getWAL(RegionInfoBuilder.newBuilder(tableName).build()); 1057 1058 this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF, 1059 false, Durability.USE_DEFAULT, wal, family); 1060 try { 1061 Path regiondir = region.getRegionFileSystem().getRegionDir(); 1062 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 1063 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 1064 1065 long maxSeqId = 3; 1066 long minSeqId = 0; 1067 1068 for (long i = minSeqId; i < maxSeqId; i++) { 1069 Put put = new Put(Bytes.toBytes(i)); 1070 put.addColumn(family, Bytes.toBytes(i), Bytes.toBytes(i)); 1071 region.put(put); 1072 region.flush(true); 1073 } 1074 1075 // this will create a region with 3 files from flush 1076 assertEquals(3, region.getStore(family).getStorefilesCount()); 1077 List<String> storeFiles = new ArrayList<>(3); 1078 for (HStoreFile sf : region.getStore(family).getStorefiles()) { 1079 storeFiles.add(sf.getPath().getName()); 1080 } 1081 1082 // now verify that the flush markers are written 1083 wal.shutdown(); 1084 WALStreamReader reader = WALFactory.createStreamReader(fs, 1085 AbstractFSWALProvider.getCurrentFileName(wal), TEST_UTIL.getConfiguration()); 1086 try { 1087 List<WAL.Entry> flushDescriptors = new ArrayList<>(); 1088 long lastFlushSeqId = -1; 1089 while (true) { 1090 WAL.Entry entry = reader.next(); 1091 if (entry == null) { 1092 break; 1093 } 1094 Cell cell = entry.getEdit().getCells().get(0); 1095 if (WALEdit.isMetaEditFamily(cell)) { 1096 FlushDescriptor flushDesc = WALEdit.getFlushDescriptor(cell); 1097 assertNotNull(flushDesc); 1098 assertArrayEquals(tableName.getName(), flushDesc.getTableName().toByteArray()); 1099 if (flushDesc.getAction() == FlushAction.START_FLUSH) { 1100 assertTrue(flushDesc.getFlushSequenceNumber() > lastFlushSeqId); 1101 } else if (flushDesc.getAction() == FlushAction.COMMIT_FLUSH) { 1102 assertTrue(flushDesc.getFlushSequenceNumber() == lastFlushSeqId); 1103 } 1104 lastFlushSeqId = flushDesc.getFlushSequenceNumber(); 1105 assertArrayEquals(regionName, flushDesc.getEncodedRegionName().toByteArray()); 1106 assertEquals(1, flushDesc.getStoreFlushesCount()); // only one store 1107 StoreFlushDescriptor storeFlushDesc = flushDesc.getStoreFlushes(0); 1108 assertArrayEquals(family, storeFlushDesc.getFamilyName().toByteArray()); 1109 assertEquals("family", storeFlushDesc.getStoreHomeDir()); 1110 if (flushDesc.getAction() == FlushAction.START_FLUSH) { 1111 assertEquals(0, storeFlushDesc.getFlushOutputCount()); 1112 } else { 1113 assertEquals(1, storeFlushDesc.getFlushOutputCount()); // only one file from flush 1114 assertTrue(storeFiles.contains(storeFlushDesc.getFlushOutput(0))); 1115 } 1116 1117 flushDescriptors.add(entry); 1118 } 1119 } 1120 1121 assertEquals(3 * 2, flushDescriptors.size()); // START_FLUSH and COMMIT_FLUSH per flush 1122 1123 // now write those markers to the recovered edits again. 1124 1125 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 1126 1127 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000)); 1128 fs.create(recoveredEdits); 1129 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 1130 1131 for (WAL.Entry entry : flushDescriptors) { 1132 writer.append(entry); 1133 } 1134 writer.close(); 1135 } finally { 1136 reader.close(); 1137 } 1138 1139 // close the region now, and reopen again 1140 HBaseTestingUtil.closeRegionAndWAL(this.region); 1141 region = HRegion.openHRegion(region, null); 1142 1143 // now check whether we have can read back the data from region 1144 for (long i = minSeqId; i < maxSeqId; i++) { 1145 Get get = new Get(Bytes.toBytes(i)); 1146 Result result = region.get(get); 1147 byte[] value = result.getValue(family, Bytes.toBytes(i)); 1148 assertArrayEquals(Bytes.toBytes(i), value); 1149 } 1150 } finally { 1151 HBaseTestingUtil.closeRegionAndWAL(this.region); 1152 this.region = null; 1153 wals.close(); 1154 } 1155 } 1156 1157 static class IsFlushWALMarker implements ArgumentMatcher<WALEdit> { 1158 volatile FlushAction[] actions; 1159 1160 public IsFlushWALMarker(FlushAction... actions) { 1161 this.actions = actions; 1162 } 1163 1164 @Override 1165 public boolean matches(WALEdit edit) { 1166 List<Cell> cells = edit.getCells(); 1167 if (cells.isEmpty()) { 1168 return false; 1169 } 1170 if (WALEdit.isMetaEditFamily(cells.get(0))) { 1171 FlushDescriptor desc; 1172 try { 1173 desc = WALEdit.getFlushDescriptor(cells.get(0)); 1174 } catch (IOException e) { 1175 LOG.warn(e.toString(), e); 1176 return false; 1177 } 1178 if (desc != null) { 1179 for (FlushAction action : actions) { 1180 if (desc.getAction() == action) { 1181 return true; 1182 } 1183 } 1184 } 1185 } 1186 return false; 1187 } 1188 1189 public IsFlushWALMarker set(FlushAction... actions) { 1190 this.actions = actions; 1191 return this; 1192 } 1193 } 1194 1195 @Test 1196 public void testFlushMarkersWALFail() throws Exception { 1197 // test the cases where the WAL append for flush markers fail. 1198 byte[] family = Bytes.toBytes("family"); 1199 1200 // spy an actual WAL implementation to throw exception (was not able to mock) 1201 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + "log"); 1202 1203 final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration()); 1204 CommonFSUtils.setRootDir(walConf, logDir); 1205 // Make up a WAL that we can manipulate at append time. 1206 class FailAppendFlushMarkerWAL extends FSHLog { 1207 volatile FlushAction[] flushActions = null; 1208 1209 public FailAppendFlushMarkerWAL(FileSystem fs, Path root, String logDir, Configuration conf) 1210 throws IOException { 1211 super(fs, root, logDir, conf); 1212 } 1213 1214 @Override 1215 protected Writer createWriterInstance(FileSystem fs, Path path) throws IOException { 1216 final Writer w = super.createWriterInstance(fs, path); 1217 return new Writer() { 1218 @Override 1219 public void close() throws IOException { 1220 w.close(); 1221 } 1222 1223 @Override 1224 public void sync(boolean forceSync) throws IOException { 1225 w.sync(forceSync); 1226 } 1227 1228 @Override 1229 public void append(Entry entry) throws IOException { 1230 List<Cell> cells = entry.getEdit().getCells(); 1231 if (WALEdit.isMetaEditFamily(cells.get(0))) { 1232 FlushDescriptor desc = WALEdit.getFlushDescriptor(cells.get(0)); 1233 if (desc != null) { 1234 for (FlushAction flushAction : flushActions) { 1235 if (desc.getAction().equals(flushAction)) { 1236 throw new IOException("Failed to append flush marker! " + flushAction); 1237 } 1238 } 1239 } 1240 } 1241 w.append(entry); 1242 } 1243 1244 @Override 1245 public long getLength() { 1246 return w.getLength(); 1247 } 1248 1249 @Override 1250 public long getSyncedLength() { 1251 return w.getSyncedLength(); 1252 } 1253 }; 1254 } 1255 } 1256 FailAppendFlushMarkerWAL wal = new FailAppendFlushMarkerWAL(FileSystem.get(walConf), 1257 CommonFSUtils.getRootDir(walConf), method, walConf); 1258 wal.init(); 1259 this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF, 1260 false, Durability.USE_DEFAULT, wal, family); 1261 int i = 0; 1262 Put put = new Put(Bytes.toBytes(i)); 1263 put.setDurability(Durability.SKIP_WAL); // have to skip mocked wal 1264 put.addColumn(family, Bytes.toBytes(i), Bytes.toBytes(i)); 1265 region.put(put); 1266 1267 // 1. Test case where START_FLUSH throws exception 1268 wal.flushActions = new FlushAction[] { FlushAction.START_FLUSH }; 1269 1270 // start cache flush will throw exception 1271 try { 1272 region.flush(true); 1273 fail("This should have thrown exception"); 1274 } catch (DroppedSnapshotException unexpected) { 1275 // this should not be a dropped snapshot exception. Meaning that RS will not abort 1276 throw unexpected; 1277 } catch (IOException expected) { 1278 // expected 1279 } 1280 // The WAL is hosed now. It has two edits appended. We cannot roll the log without it 1281 // throwing a DroppedSnapshotException to force an abort. Just clean up the mess. 1282 region.close(true); 1283 wal.close(); 1284 // release the snapshot and active segment, so netty will not report memory leak 1285 for (HStore store : region.getStores()) { 1286 AbstractMemStore memstore = (AbstractMemStore) store.memstore; 1287 memstore.doClearSnapShot(); 1288 memstore.close(); 1289 } 1290 1291 // 2. Test case where START_FLUSH succeeds but COMMIT_FLUSH will throw exception 1292 wal.flushActions = new FlushAction[] { FlushAction.COMMIT_FLUSH }; 1293 wal = new FailAppendFlushMarkerWAL(FileSystem.get(walConf), CommonFSUtils.getRootDir(walConf), 1294 method, walConf); 1295 wal.init(); 1296 this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF, 1297 false, Durability.USE_DEFAULT, wal, family); 1298 region.put(put); 1299 // 3. Test case where ABORT_FLUSH will throw exception. 1300 // Even if ABORT_FLUSH throws exception, we should not fail with IOE, but continue with 1301 // DroppedSnapshotException. Below COMMIT_FLUSH will cause flush to abort 1302 wal.flushActions = new FlushAction[] { FlushAction.COMMIT_FLUSH, FlushAction.ABORT_FLUSH }; 1303 1304 // we expect this exception, since we were able to write the snapshot, but failed to 1305 // write the flush marker to WAL 1306 assertThrows(DroppedSnapshotException.class, () -> region.flush(true)); 1307 1308 region.close(true); 1309 // release the snapshot and active segment, so netty will not report memory leak 1310 for (HStore store : region.getStores()) { 1311 AbstractMemStore memstore = (AbstractMemStore) store.memstore; 1312 memstore.doClearSnapShot(); 1313 memstore.close(); 1314 } 1315 region = null; 1316 } 1317 1318 @Test 1319 public void testGetWhileRegionClose() throws IOException { 1320 Configuration hc = initSplit(); 1321 int numRows = 100; 1322 byte[][] families = { fam1, fam2, fam3 }; 1323 1324 // Setting up region 1325 this.region = initHRegion(tableName, method, hc, families); 1326 // Put data in region 1327 final int startRow = 100; 1328 putData(startRow, numRows, qual1, families); 1329 putData(startRow, numRows, qual2, families); 1330 putData(startRow, numRows, qual3, families); 1331 final AtomicBoolean done = new AtomicBoolean(false); 1332 final AtomicInteger gets = new AtomicInteger(0); 1333 GetTillDoneOrException[] threads = new GetTillDoneOrException[10]; 1334 try { 1335 // Set ten threads running concurrently getting from the region. 1336 for (int i = 0; i < threads.length / 2; i++) { 1337 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets); 1338 threads[i].setDaemon(true); 1339 threads[i].start(); 1340 } 1341 // Artificially make the condition by setting closing flag explicitly. 1342 // I can't make the issue happen with a call to region.close(). 1343 this.region.closing.set(true); 1344 for (int i = threads.length / 2; i < threads.length; i++) { 1345 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets); 1346 threads[i].setDaemon(true); 1347 threads[i].start(); 1348 } 1349 } finally { 1350 if (this.region != null) { 1351 HBaseTestingUtil.closeRegionAndWAL(this.region); 1352 this.region = null; 1353 } 1354 } 1355 done.set(true); 1356 for (GetTillDoneOrException t : threads) { 1357 try { 1358 t.join(); 1359 } catch (InterruptedException e) { 1360 e.printStackTrace(); 1361 } 1362 if (t.e != null) { 1363 LOG.info("Exception=" + t.e); 1364 assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException); 1365 } 1366 } 1367 } 1368 1369 /* 1370 * Thread that does get on single row until 'done' flag is flipped. If an exception causes us to 1371 * fail, it records it. 1372 */ 1373 class GetTillDoneOrException extends Thread { 1374 private final Get g; 1375 private final AtomicBoolean done; 1376 private final AtomicInteger count; 1377 private Exception e; 1378 1379 GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, 1380 final AtomicInteger c) { 1381 super("getter." + i); 1382 this.g = new Get(r); 1383 this.done = d; 1384 this.count = c; 1385 } 1386 1387 @Override 1388 public void run() { 1389 while (!this.done.get()) { 1390 try { 1391 assertTrue(region.get(g).size() > 0); 1392 this.count.incrementAndGet(); 1393 } catch (Exception e) { 1394 this.e = e; 1395 break; 1396 } 1397 } 1398 } 1399 } 1400 1401 /* 1402 * An involved filter test. Has multiple column families and deletes in mix. 1403 */ 1404 @Test 1405 public void testWeirdCacheBehaviour() throws Exception { 1406 final TableName tableName = TableName.valueOf(name.getMethodName()); 1407 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"), 1408 Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") }; 1409 this.region = initHRegion(tableName, method, CONF, FAMILIES); 1410 String value = "this is the value"; 1411 String value2 = "this is some other value"; 1412 String keyPrefix1 = "prefix1"; 1413 String keyPrefix2 = "prefix2"; 1414 String keyPrefix3 = "prefix3"; 1415 putRows(this.region, 3, value, keyPrefix1); 1416 putRows(this.region, 3, value, keyPrefix2); 1417 putRows(this.region, 3, value, keyPrefix3); 1418 putRows(this.region, 3, value2, keyPrefix1); 1419 putRows(this.region, 3, value2, keyPrefix2); 1420 putRows(this.region, 3, value2, keyPrefix3); 1421 System.out.println("Checking values for key: " + keyPrefix1); 1422 assertEquals("Got back incorrect number of rows from scan", 3, 1423 getNumberOfRows(keyPrefix1, value2, this.region)); 1424 System.out.println("Checking values for key: " + keyPrefix2); 1425 assertEquals("Got back incorrect number of rows from scan", 3, 1426 getNumberOfRows(keyPrefix2, value2, this.region)); 1427 System.out.println("Checking values for key: " + keyPrefix3); 1428 assertEquals("Got back incorrect number of rows from scan", 3, 1429 getNumberOfRows(keyPrefix3, value2, this.region)); 1430 deleteColumns(this.region, value2, keyPrefix1); 1431 deleteColumns(this.region, value2, keyPrefix2); 1432 deleteColumns(this.region, value2, keyPrefix3); 1433 System.out.println("Starting important checks....."); 1434 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0, 1435 getNumberOfRows(keyPrefix1, value2, this.region)); 1436 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0, 1437 getNumberOfRows(keyPrefix2, value2, this.region)); 1438 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0, 1439 getNumberOfRows(keyPrefix3, value2, this.region)); 1440 } 1441 1442 @Test 1443 public void testAppendWithReadOnlyTable() throws Exception { 1444 final TableName tableName = TableName.valueOf(name.getMethodName()); 1445 this.region = initHRegion(tableName, method, CONF, true, Bytes.toBytes("somefamily")); 1446 boolean exceptionCaught = false; 1447 Append append = new Append(Bytes.toBytes("somerow")); 1448 append.setDurability(Durability.SKIP_WAL); 1449 append.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1450 Bytes.toBytes("somevalue")); 1451 try { 1452 region.append(append); 1453 } catch (IOException e) { 1454 exceptionCaught = true; 1455 } 1456 assertTrue(exceptionCaught == true); 1457 } 1458 1459 @Test 1460 public void testIncrWithReadOnlyTable() throws Exception { 1461 final TableName tableName = TableName.valueOf(name.getMethodName()); 1462 this.region = initHRegion(tableName, method, CONF, true, Bytes.toBytes("somefamily")); 1463 boolean exceptionCaught = false; 1464 Increment inc = new Increment(Bytes.toBytes("somerow")); 1465 inc.setDurability(Durability.SKIP_WAL); 1466 inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L); 1467 try { 1468 region.increment(inc); 1469 } catch (IOException e) { 1470 exceptionCaught = true; 1471 } 1472 assertTrue(exceptionCaught == true); 1473 } 1474 1475 private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException { 1476 int count = 0; 1477 try (InternalScanner scanner = buildScanner(keyPrefix, value, r)) { 1478 boolean more = false; 1479 List<Cell> results = new ArrayList<>(); 1480 do { 1481 more = scanner.next(results); 1482 if (results != null && !results.isEmpty()) { 1483 count++; 1484 } else { 1485 break; 1486 } 1487 Delete delete = new Delete(CellUtil.cloneRow(results.get(0))); 1488 delete.addColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2")); 1489 r.delete(delete); 1490 results.clear(); 1491 } while (more); 1492 } 1493 assertEquals("Did not perform correct number of deletes", 3, count); 1494 } 1495 1496 private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception { 1497 try (InternalScanner resultScanner = buildScanner(keyPrefix, value, r)) { 1498 int numberOfResults = 0; 1499 List<Cell> results = new ArrayList<>(); 1500 boolean more = false; 1501 do { 1502 more = resultScanner.next(results); 1503 if (results != null && !results.isEmpty()) { 1504 numberOfResults++; 1505 } else { 1506 break; 1507 } 1508 for (Cell kv : results) { 1509 LOG.info("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv))); 1510 } 1511 results.clear(); 1512 } while (more); 1513 return numberOfResults; 1514 } 1515 } 1516 1517 private InternalScanner buildScanner(String keyPrefix, String value, HRegion r) 1518 throws IOException { 1519 // Defaults FilterList.Operator.MUST_PASS_ALL. 1520 FilterList allFilters = new FilterList(); 1521 allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix))); 1522 // Only return rows where this column value exists in the row. 1523 SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"), 1524 Bytes.toBytes("qual2"), CompareOperator.EQUAL, Bytes.toBytes(value)); 1525 filter.setFilterIfMissing(true); 1526 allFilters.addFilter(filter); 1527 Scan scan = new Scan(); 1528 scan.addFamily(Bytes.toBytes("trans-blob")); 1529 scan.addFamily(Bytes.toBytes("trans-type")); 1530 scan.addFamily(Bytes.toBytes("trans-date")); 1531 scan.addFamily(Bytes.toBytes("trans-tags")); 1532 scan.addFamily(Bytes.toBytes("trans-group")); 1533 scan.setFilter(allFilters); 1534 return r.getScanner(scan); 1535 } 1536 1537 private void putRows(HRegion r, int numRows, String value, String key) throws IOException { 1538 for (int i = 0; i < numRows; i++) { 1539 String row = key + "_" + i/* UUID.randomUUID().toString() */; 1540 System.out.println(String.format("Saving row: %s, with value %s", row, value)); 1541 Put put = new Put(Bytes.toBytes(row)); 1542 put.setDurability(Durability.SKIP_WAL); 1543 put.addColumn(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob")); 1544 put.addColumn(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement")); 1545 put.addColumn(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999")); 1546 put.addColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value)); 1547 put.addColumn(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId")); 1548 r.put(put); 1549 } 1550 } 1551 1552 @Test 1553 public void testFamilyWithAndWithoutColon() throws Exception { 1554 byte[] cf = Bytes.toBytes(COLUMN_FAMILY); 1555 this.region = initHRegion(tableName, method, CONF, cf); 1556 Put p = new Put(tableName.toBytes()); 1557 byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":"); 1558 p.addColumn(cfwithcolon, cfwithcolon, cfwithcolon); 1559 boolean exception = false; 1560 try { 1561 this.region.put(p); 1562 } catch (NoSuchColumnFamilyException e) { 1563 exception = true; 1564 } 1565 assertTrue(exception); 1566 } 1567 1568 @Test 1569 public void testBatchPut_whileNoRowLocksHeld() throws IOException { 1570 final Put[] puts = new Put[10]; 1571 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); 1572 long syncs = prepareRegionForBachPut(puts, source, false); 1573 1574 OperationStatus[] codes = this.region.batchMutate(puts); 1575 assertEquals(10, codes.length); 1576 for (int i = 0; i < 10; i++) { 1577 assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode()); 1578 } 1579 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source); 1580 1581 LOG.info("Next a batch put with one invalid family"); 1582 puts[5].addColumn(Bytes.toBytes("BAD_CF"), qual, value); 1583 codes = this.region.batchMutate(puts); 1584 assertEquals(10, codes.length); 1585 for (int i = 0; i < 10; i++) { 1586 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS, 1587 codes[i].getOperationStatusCode()); 1588 } 1589 1590 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source); 1591 } 1592 1593 @Test 1594 public void testBatchPut_whileMultipleRowLocksHeld() throws Exception { 1595 final Put[] puts = new Put[10]; 1596 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); 1597 long syncs = prepareRegionForBachPut(puts, source, false); 1598 1599 puts[5].addColumn(Bytes.toBytes("BAD_CF"), qual, value); 1600 1601 LOG.info("batchPut will have to break into four batches to avoid row locks"); 1602 RowLock rowLock1 = region.getRowLock(Bytes.toBytes("row_2")); 1603 RowLock rowLock2 = region.getRowLock(Bytes.toBytes("row_1")); 1604 RowLock rowLock3 = region.getRowLock(Bytes.toBytes("row_3")); 1605 RowLock rowLock4 = region.getRowLock(Bytes.toBytes("row_3"), true); 1606 1607 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF); 1608 final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<>(); 1609 final CountDownLatch startingPuts = new CountDownLatch(1); 1610 final CountDownLatch startingClose = new CountDownLatch(1); 1611 TestThread putter = new TestThread(ctx) { 1612 @Override 1613 public void doWork() throws IOException { 1614 startingPuts.countDown(); 1615 retFromThread.set(region.batchMutate(puts)); 1616 } 1617 }; 1618 LOG.info("...starting put thread while holding locks"); 1619 ctx.addThread(putter); 1620 ctx.startThreads(); 1621 1622 // Now attempt to close the region from another thread. Prior to HBASE-12565 1623 // this would cause the in-progress batchMutate operation to to fail with 1624 // exception because it use to release and re-acquire the close-guard lock 1625 // between batches. Caller then didn't get status indicating which writes succeeded. 1626 // We now expect this thread to block until the batchMutate call finishes. 1627 Thread regionCloseThread = new TestThread(ctx) { 1628 @Override 1629 public void doWork() { 1630 try { 1631 startingPuts.await(); 1632 // Give some time for the batch mutate to get in. 1633 // We don't want to race with the mutate 1634 Thread.sleep(10); 1635 startingClose.countDown(); 1636 HBaseTestingUtil.closeRegionAndWAL(region); 1637 region = null; 1638 } catch (IOException e) { 1639 throw new RuntimeException(e); 1640 } catch (InterruptedException e) { 1641 throw new RuntimeException(e); 1642 } 1643 } 1644 }; 1645 regionCloseThread.start(); 1646 1647 startingClose.await(); 1648 startingPuts.await(); 1649 Thread.sleep(100); 1650 LOG.info("...releasing row lock 1, which should let put thread continue"); 1651 rowLock1.release(); 1652 rowLock2.release(); 1653 rowLock3.release(); 1654 waitForCounter(source, "syncTimeNumOps", syncs + 1); 1655 1656 LOG.info("...joining on put thread"); 1657 ctx.stop(); 1658 regionCloseThread.join(); 1659 1660 OperationStatus[] codes = retFromThread.get(); 1661 for (int i = 0; i < codes.length; i++) { 1662 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS, 1663 codes[i].getOperationStatusCode()); 1664 } 1665 rowLock4.release(); 1666 } 1667 1668 private void waitForCounter(MetricsWALSource source, String metricName, long expectedCount) 1669 throws InterruptedException { 1670 long startWait = EnvironmentEdgeManager.currentTime(); 1671 long currentCount; 1672 while ((currentCount = metricsAssertHelper.getCounter(metricName, source)) < expectedCount) { 1673 Thread.sleep(100); 1674 if (EnvironmentEdgeManager.currentTime() - startWait > 10000) { 1675 fail(String.format("Timed out waiting for '%s' >= '%s', currentCount=%s", metricName, 1676 expectedCount, currentCount)); 1677 } 1678 } 1679 } 1680 1681 @Test 1682 public void testAtomicBatchPut() throws IOException { 1683 final Put[] puts = new Put[10]; 1684 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); 1685 long syncs = prepareRegionForBachPut(puts, source, false); 1686 1687 // 1. Straight forward case, should succeed 1688 OperationStatus[] codes = this.region.batchMutate(puts, true); 1689 assertEquals(10, codes.length); 1690 for (int i = 0; i < 10; i++) { 1691 assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode()); 1692 } 1693 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source); 1694 1695 // 2. Failed to get lock 1696 RowLock lock = region.getRowLock(Bytes.toBytes("row_" + 3)); 1697 // Method {@link HRegion#getRowLock(byte[])} is reentrant. As 'row_3' is locked in this 1698 // thread, need to run {@link HRegion#batchMutate(HRegion.BatchOperation)} in different thread 1699 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF); 1700 final AtomicReference<IOException> retFromThread = new AtomicReference<>(); 1701 final CountDownLatch finishedPuts = new CountDownLatch(1); 1702 TestThread putter = new TestThread(ctx) { 1703 @Override 1704 public void doWork() throws IOException { 1705 try { 1706 region.batchMutate(puts, true); 1707 } catch (IOException ioe) { 1708 LOG.error("test failed!", ioe); 1709 retFromThread.set(ioe); 1710 } 1711 finishedPuts.countDown(); 1712 } 1713 }; 1714 LOG.info("...starting put thread while holding locks"); 1715 ctx.addThread(putter); 1716 ctx.startThreads(); 1717 LOG.info("...waiting for batch puts while holding locks"); 1718 try { 1719 finishedPuts.await(); 1720 } catch (InterruptedException e) { 1721 LOG.error("Interrupted!", e); 1722 } finally { 1723 if (lock != null) { 1724 lock.release(); 1725 } 1726 } 1727 assertNotNull(retFromThread.get()); 1728 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source); 1729 1730 // 3. Exception thrown in validation 1731 LOG.info("Next a batch put with one invalid family"); 1732 puts[5].addColumn(Bytes.toBytes("BAD_CF"), qual, value); 1733 thrown.expect(NoSuchColumnFamilyException.class); 1734 this.region.batchMutate(puts, true); 1735 } 1736 1737 @Test 1738 public void testBatchPutWithTsSlop() throws Exception { 1739 // add data with a timestamp that is too recent for range. Ensure assert 1740 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000); 1741 final Put[] puts = new Put[10]; 1742 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); 1743 1744 long syncs = prepareRegionForBachPut(puts, source, true); 1745 1746 OperationStatus[] codes = this.region.batchMutate(puts); 1747 assertEquals(10, codes.length); 1748 for (int i = 0; i < 10; i++) { 1749 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode()); 1750 } 1751 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source); 1752 } 1753 1754 /** Returns syncs initial syncTimeNumOps */ 1755 private long prepareRegionForBachPut(final Put[] puts, final MetricsWALSource source, 1756 boolean slop) throws IOException { 1757 this.region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES); 1758 1759 LOG.info("First a batch put with all valid puts"); 1760 for (int i = 0; i < puts.length; i++) { 1761 puts[i] = slop 1762 ? new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100) 1763 : new Put(Bytes.toBytes("row_" + i)); 1764 puts[i].addColumn(COLUMN_FAMILY_BYTES, qual, value); 1765 } 1766 1767 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source); 1768 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source); 1769 return syncs; 1770 } 1771 1772 // //////////////////////////////////////////////////////////////////////////// 1773 // checkAndMutate tests 1774 // //////////////////////////////////////////////////////////////////////////// 1775 @Test 1776 @Deprecated 1777 public void testCheckAndMutate_WithEmptyRowValue() throws IOException { 1778 byte[] row1 = Bytes.toBytes("row1"); 1779 byte[] fam1 = Bytes.toBytes("fam1"); 1780 byte[] qf1 = Bytes.toBytes("qualifier"); 1781 byte[] emptyVal = new byte[] {}; 1782 byte[] val1 = Bytes.toBytes("value1"); 1783 byte[] val2 = Bytes.toBytes("value2"); 1784 1785 // Setting up region 1786 this.region = initHRegion(tableName, method, CONF, fam1); 1787 // Putting empty data in key 1788 Put put = new Put(row1); 1789 put.addColumn(fam1, qf1, emptyVal); 1790 1791 // checkAndPut with empty value 1792 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1793 new BinaryComparator(emptyVal), put); 1794 assertTrue(res); 1795 1796 // Putting data in key 1797 put = new Put(row1); 1798 put.addColumn(fam1, qf1, val1); 1799 1800 // checkAndPut with correct value 1801 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1802 new BinaryComparator(emptyVal), put); 1803 assertTrue(res); 1804 1805 // not empty anymore 1806 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1807 new BinaryComparator(emptyVal), put); 1808 assertFalse(res); 1809 1810 Delete delete = new Delete(row1); 1811 delete.addColumn(fam1, qf1); 1812 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1813 new BinaryComparator(emptyVal), delete); 1814 assertFalse(res); 1815 1816 put = new Put(row1); 1817 put.addColumn(fam1, qf1, val2); 1818 // checkAndPut with correct value 1819 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val1), 1820 put); 1821 assertTrue(res); 1822 1823 // checkAndDelete with correct value 1824 delete = new Delete(row1); 1825 delete.addColumn(fam1, qf1); 1826 delete.addColumn(fam1, qf1); 1827 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val2), 1828 delete); 1829 assertTrue(res); 1830 1831 delete = new Delete(row1); 1832 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1833 new BinaryComparator(emptyVal), delete); 1834 assertTrue(res); 1835 1836 // checkAndPut looking for a null value 1837 put = new Put(row1); 1838 put.addColumn(fam1, qf1, val1); 1839 1840 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new NullComparator(), put); 1841 assertTrue(res); 1842 } 1843 1844 @Test 1845 @Deprecated 1846 public void testCheckAndMutate_WithWrongValue() throws IOException { 1847 byte[] row1 = Bytes.toBytes("row1"); 1848 byte[] fam1 = Bytes.toBytes("fam1"); 1849 byte[] qf1 = Bytes.toBytes("qualifier"); 1850 byte[] val1 = Bytes.toBytes("value1"); 1851 byte[] val2 = Bytes.toBytes("value2"); 1852 BigDecimal bd1 = new BigDecimal(Double.MAX_VALUE); 1853 BigDecimal bd2 = new BigDecimal(Double.MIN_VALUE); 1854 1855 // Setting up region 1856 this.region = initHRegion(tableName, method, CONF, fam1); 1857 // Putting data in key 1858 Put put = new Put(row1); 1859 put.addColumn(fam1, qf1, val1); 1860 region.put(put); 1861 1862 // checkAndPut with wrong value 1863 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1864 new BinaryComparator(val2), put); 1865 assertEquals(false, res); 1866 1867 // checkAndDelete with wrong value 1868 Delete delete = new Delete(row1); 1869 delete.addFamily(fam1); 1870 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val2), 1871 put); 1872 assertEquals(false, res); 1873 1874 // Putting data in key 1875 put = new Put(row1); 1876 put.addColumn(fam1, qf1, Bytes.toBytes(bd1)); 1877 region.put(put); 1878 1879 // checkAndPut with wrong value 1880 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1881 new BigDecimalComparator(bd2), put); 1882 assertEquals(false, res); 1883 1884 // checkAndDelete with wrong value 1885 delete = new Delete(row1); 1886 delete.addFamily(fam1); 1887 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1888 new BigDecimalComparator(bd2), put); 1889 assertEquals(false, res); 1890 } 1891 1892 @Test 1893 @Deprecated 1894 public void testCheckAndMutate_WithCorrectValue() throws IOException { 1895 byte[] row1 = Bytes.toBytes("row1"); 1896 byte[] fam1 = Bytes.toBytes("fam1"); 1897 byte[] qf1 = Bytes.toBytes("qualifier"); 1898 byte[] val1 = Bytes.toBytes("value1"); 1899 BigDecimal bd1 = new BigDecimal(Double.MIN_VALUE); 1900 1901 // Setting up region 1902 this.region = initHRegion(tableName, method, CONF, fam1); 1903 // Putting data in key 1904 long now = EnvironmentEdgeManager.currentTime(); 1905 Put put = new Put(row1); 1906 put.addColumn(fam1, qf1, now, val1); 1907 region.put(put); 1908 1909 // checkAndPut with correct value 1910 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1911 new BinaryComparator(val1), put); 1912 assertEquals("First", true, res); 1913 1914 // checkAndDelete with correct value 1915 Delete delete = new Delete(row1, now + 1); 1916 delete.addColumn(fam1, qf1); 1917 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val1), 1918 delete); 1919 assertEquals("Delete", true, res); 1920 1921 // Putting data in key 1922 put = new Put(row1); 1923 put.addColumn(fam1, qf1, now + 2, Bytes.toBytes(bd1)); 1924 region.put(put); 1925 1926 // checkAndPut with correct value 1927 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1928 new BigDecimalComparator(bd1), put); 1929 assertEquals("Second put", true, res); 1930 1931 // checkAndDelete with correct value 1932 delete = new Delete(row1, now + 3); 1933 delete.addColumn(fam1, qf1); 1934 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1935 new BigDecimalComparator(bd1), delete); 1936 assertEquals("Second delete", true, res); 1937 } 1938 1939 @Test 1940 @Deprecated 1941 public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException { 1942 byte[] row1 = Bytes.toBytes("row1"); 1943 byte[] fam1 = Bytes.toBytes("fam1"); 1944 byte[] qf1 = Bytes.toBytes("qualifier"); 1945 byte[] val1 = Bytes.toBytes("value1"); 1946 byte[] val2 = Bytes.toBytes("value2"); 1947 byte[] val3 = Bytes.toBytes("value3"); 1948 byte[] val4 = Bytes.toBytes("value4"); 1949 1950 // Setting up region 1951 this.region = initHRegion(tableName, method, CONF, fam1); 1952 // Putting val3 in key 1953 Put put = new Put(row1); 1954 put.addColumn(fam1, qf1, val3); 1955 region.put(put); 1956 1957 // Test CompareOp.LESS: original = val3, compare with val3, fail 1958 boolean res = 1959 region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS, new BinaryComparator(val3), put); 1960 assertEquals(false, res); 1961 1962 // Test CompareOp.LESS: original = val3, compare with val4, fail 1963 res = 1964 region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS, new BinaryComparator(val4), put); 1965 assertEquals(false, res); 1966 1967 // Test CompareOp.LESS: original = val3, compare with val2, 1968 // succeed (now value = val2) 1969 put = new Put(row1); 1970 put.addColumn(fam1, qf1, val2); 1971 res = 1972 region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS, new BinaryComparator(val2), put); 1973 assertEquals(true, res); 1974 1975 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val3, fail 1976 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS_OR_EQUAL, 1977 new BinaryComparator(val3), put); 1978 assertEquals(false, res); 1979 1980 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val2, 1981 // succeed (value still = val2) 1982 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS_OR_EQUAL, 1983 new BinaryComparator(val2), put); 1984 assertEquals(true, res); 1985 1986 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val1, 1987 // succeed (now value = val3) 1988 put = new Put(row1); 1989 put.addColumn(fam1, qf1, val3); 1990 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS_OR_EQUAL, 1991 new BinaryComparator(val1), put); 1992 assertEquals(true, res); 1993 1994 // Test CompareOp.GREATER: original = val3, compare with val3, fail 1995 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER, 1996 new BinaryComparator(val3), put); 1997 assertEquals(false, res); 1998 1999 // Test CompareOp.GREATER: original = val3, compare with val2, fail 2000 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER, 2001 new BinaryComparator(val2), put); 2002 assertEquals(false, res); 2003 2004 // Test CompareOp.GREATER: original = val3, compare with val4, 2005 // succeed (now value = val2) 2006 put = new Put(row1); 2007 put.addColumn(fam1, qf1, val2); 2008 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER, 2009 new BinaryComparator(val4), put); 2010 assertEquals(true, res); 2011 2012 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val1, fail 2013 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER_OR_EQUAL, 2014 new BinaryComparator(val1), put); 2015 assertEquals(false, res); 2016 2017 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val2, 2018 // succeed (value still = val2) 2019 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER_OR_EQUAL, 2020 new BinaryComparator(val2), put); 2021 assertEquals(true, res); 2022 2023 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val3, succeed 2024 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER_OR_EQUAL, 2025 new BinaryComparator(val3), put); 2026 assertEquals(true, res); 2027 } 2028 2029 @Test 2030 @Deprecated 2031 public void testCheckAndPut_ThatPutWasWritten() throws IOException { 2032 byte[] row1 = Bytes.toBytes("row1"); 2033 byte[] fam1 = Bytes.toBytes("fam1"); 2034 byte[] fam2 = Bytes.toBytes("fam2"); 2035 byte[] qf1 = Bytes.toBytes("qualifier"); 2036 byte[] val1 = Bytes.toBytes("value1"); 2037 byte[] val2 = Bytes.toBytes("value2"); 2038 2039 byte[][] families = { fam1, fam2 }; 2040 2041 // Setting up region 2042 this.region = initHRegion(tableName, method, CONF, families); 2043 // Putting data in the key to check 2044 Put put = new Put(row1); 2045 put.addColumn(fam1, qf1, val1); 2046 region.put(put); 2047 2048 // Creating put to add 2049 long ts = EnvironmentEdgeManager.currentTime(); 2050 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2); 2051 put = new Put(row1); 2052 put.add(kv); 2053 2054 // checkAndPut with wrong value 2055 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 2056 new BinaryComparator(val1), put); 2057 assertEquals(true, res); 2058 2059 Get get = new Get(row1); 2060 get.addColumn(fam2, qf1); 2061 Cell[] actual = region.get(get).rawCells(); 2062 2063 Cell[] expected = { kv }; 2064 2065 assertEquals(expected.length, actual.length); 2066 for (int i = 0; i < actual.length; i++) { 2067 assertEquals(expected[i], actual[i]); 2068 } 2069 } 2070 2071 @Test 2072 @Deprecated 2073 public void testCheckAndPut_wrongRowInPut() throws IOException { 2074 this.region = initHRegion(tableName, method, CONF, COLUMNS); 2075 Put put = new Put(row2); 2076 put.addColumn(fam1, qual1, value1); 2077 try { 2078 region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(value2), 2079 put); 2080 fail(); 2081 } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) { 2082 // expected exception. 2083 } 2084 } 2085 2086 @Test 2087 @Deprecated 2088 public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException { 2089 byte[] row1 = Bytes.toBytes("row1"); 2090 byte[] fam1 = Bytes.toBytes("fam1"); 2091 byte[] fam2 = Bytes.toBytes("fam2"); 2092 byte[] qf1 = Bytes.toBytes("qualifier1"); 2093 byte[] qf2 = Bytes.toBytes("qualifier2"); 2094 byte[] qf3 = Bytes.toBytes("qualifier3"); 2095 byte[] val1 = Bytes.toBytes("value1"); 2096 byte[] val2 = Bytes.toBytes("value2"); 2097 byte[] val3 = Bytes.toBytes("value3"); 2098 byte[] emptyVal = new byte[] {}; 2099 2100 byte[][] families = { fam1, fam2 }; 2101 2102 // Setting up region 2103 this.region = initHRegion(tableName, method, CONF, families); 2104 // Put content 2105 Put put = new Put(row1); 2106 put.addColumn(fam1, qf1, val1); 2107 region.put(put); 2108 Threads.sleep(2); 2109 2110 put = new Put(row1); 2111 put.addColumn(fam1, qf1, val2); 2112 put.addColumn(fam2, qf1, val3); 2113 put.addColumn(fam2, qf2, val2); 2114 put.addColumn(fam2, qf3, val1); 2115 put.addColumn(fam1, qf3, val1); 2116 region.put(put); 2117 2118 LOG.info("get={}", region.get(new Get(row1).addColumn(fam1, qf1)).toString()); 2119 2120 // Multi-column delete 2121 Delete delete = new Delete(row1); 2122 delete.addColumn(fam1, qf1); 2123 delete.addColumn(fam2, qf1); 2124 delete.addColumn(fam1, qf3); 2125 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 2126 new BinaryComparator(val2), delete); 2127 assertEquals(true, res); 2128 2129 Get get = new Get(row1); 2130 get.addColumn(fam1, qf1); 2131 get.addColumn(fam1, qf3); 2132 get.addColumn(fam2, qf2); 2133 Result r = region.get(get); 2134 assertEquals(2, r.size()); 2135 assertArrayEquals(val1, r.getValue(fam1, qf1)); 2136 assertArrayEquals(val2, r.getValue(fam2, qf2)); 2137 2138 // Family delete 2139 delete = new Delete(row1); 2140 delete.addFamily(fam2); 2141 res = region.checkAndMutate(row1, fam2, qf1, CompareOperator.EQUAL, 2142 new BinaryComparator(emptyVal), delete); 2143 assertEquals(true, res); 2144 2145 get = new Get(row1); 2146 r = region.get(get); 2147 assertEquals(1, r.size()); 2148 assertArrayEquals(val1, r.getValue(fam1, qf1)); 2149 2150 // Row delete 2151 delete = new Delete(row1); 2152 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val1), 2153 delete); 2154 assertEquals(true, res); 2155 get = new Get(row1); 2156 r = region.get(get); 2157 assertEquals(0, r.size()); 2158 } 2159 2160 @Test 2161 @Deprecated 2162 public void testCheckAndMutate_WithFilters() throws Throwable { 2163 final byte[] FAMILY = Bytes.toBytes("fam"); 2164 2165 // Setting up region 2166 this.region = initHRegion(tableName, method, CONF, FAMILY); 2167 2168 // Put one row 2169 Put put = new Put(row); 2170 put.addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")); 2171 put.addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")); 2172 put.addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")); 2173 region.put(put); 2174 2175 // Put with success 2176 boolean ok = region.checkAndMutate(row, 2177 new FilterList( 2178 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2179 Bytes.toBytes("a")), 2180 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2181 Bytes.toBytes("b"))), 2182 new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d"))); 2183 assertTrue(ok); 2184 2185 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))); 2186 assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D")))); 2187 2188 // Put with failure 2189 ok = region.checkAndMutate(row, 2190 new FilterList( 2191 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2192 Bytes.toBytes("a")), 2193 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2194 Bytes.toBytes("c"))), 2195 new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e"))); 2196 assertFalse(ok); 2197 2198 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E"))).isEmpty()); 2199 2200 // Delete with success 2201 ok = region.checkAndMutate(row, 2202 new FilterList( 2203 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2204 Bytes.toBytes("a")), 2205 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2206 Bytes.toBytes("b"))), 2207 new Delete(row).addColumns(FAMILY, Bytes.toBytes("D"))); 2208 assertTrue(ok); 2209 2210 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))).isEmpty()); 2211 2212 // Mutate with success 2213 ok = region.checkAndRowMutate(row, 2214 new FilterList( 2215 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2216 Bytes.toBytes("a")), 2217 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2218 Bytes.toBytes("b"))), 2219 new RowMutations(row) 2220 .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e"))) 2221 .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A")))); 2222 assertTrue(ok); 2223 2224 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E"))); 2225 assertEquals("e", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("E")))); 2226 2227 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty()); 2228 } 2229 2230 @Test 2231 @Deprecated 2232 public void testCheckAndMutate_WithFiltersAndTimeRange() throws Throwable { 2233 final byte[] FAMILY = Bytes.toBytes("fam"); 2234 2235 // Setting up region 2236 this.region = initHRegion(tableName, method, CONF, FAMILY); 2237 2238 // Put with specifying the timestamp 2239 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), 100, Bytes.toBytes("a"))); 2240 2241 // Put with success 2242 boolean ok = region.checkAndMutate(row, 2243 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2244 Bytes.toBytes("a")), 2245 TimeRange.between(0, 101), 2246 new Put(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b"))); 2247 assertTrue(ok); 2248 2249 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2250 assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2251 2252 // Put with failure 2253 ok = region.checkAndMutate(row, 2254 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2255 Bytes.toBytes("a")), 2256 TimeRange.between(0, 100), 2257 new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c"))); 2258 assertFalse(ok); 2259 2260 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("C"))).isEmpty()); 2261 2262 // Mutate with success 2263 ok = region.checkAndRowMutate(row, 2264 new SingleColumnValueFilter( 2265 FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, Bytes.toBytes("a")), 2266 TimeRange.between(0, 101), 2267 new RowMutations(row) 2268 .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d"))) 2269 .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A")))); 2270 assertTrue(ok); 2271 2272 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))); 2273 assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D")))); 2274 2275 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty()); 2276 } 2277 2278 @Test 2279 @Deprecated 2280 public void testCheckAndMutate_wrongMutationType() throws Throwable { 2281 // Setting up region 2282 this.region = initHRegion(tableName, method, CONF, fam1); 2283 2284 try { 2285 region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(value1), 2286 new Increment(row).addColumn(fam1, qual1, 1)); 2287 fail("should throw DoNotRetryIOException"); 2288 } catch (DoNotRetryIOException e) { 2289 assertEquals("Unsupported mutate type: INCREMENT", e.getMessage()); 2290 } 2291 2292 try { 2293 region.checkAndMutate(row, 2294 new SingleColumnValueFilter(fam1, qual1, CompareOperator.EQUAL, value1), 2295 new Increment(row).addColumn(fam1, qual1, 1)); 2296 fail("should throw DoNotRetryIOException"); 2297 } catch (DoNotRetryIOException e) { 2298 assertEquals("Unsupported mutate type: INCREMENT", e.getMessage()); 2299 } 2300 } 2301 2302 @Test 2303 @Deprecated 2304 public void testCheckAndMutate_wrongRow() throws Throwable { 2305 final byte[] wrongRow = Bytes.toBytes("wrongRow"); 2306 2307 // Setting up region 2308 this.region = initHRegion(tableName, method, CONF, fam1); 2309 2310 try { 2311 region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(value1), 2312 new Put(wrongRow).addColumn(fam1, qual1, value1)); 2313 fail("should throw DoNotRetryIOException"); 2314 } catch (DoNotRetryIOException e) { 2315 assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>", 2316 e.getMessage()); 2317 } 2318 2319 try { 2320 region.checkAndMutate(row, 2321 new SingleColumnValueFilter(fam1, qual1, CompareOperator.EQUAL, value1), 2322 new Put(wrongRow).addColumn(fam1, qual1, value1)); 2323 fail("should throw DoNotRetryIOException"); 2324 } catch (DoNotRetryIOException e) { 2325 assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>", 2326 e.getMessage()); 2327 } 2328 2329 try { 2330 region.checkAndRowMutate(row, fam1, qual1, CompareOperator.EQUAL, 2331 new BinaryComparator(value1), 2332 new RowMutations(wrongRow).add((Mutation) new Put(wrongRow).addColumn(fam1, qual1, value1)) 2333 .add((Mutation) new Delete(wrongRow).addColumns(fam1, qual2))); 2334 fail("should throw DoNotRetryIOException"); 2335 } catch (DoNotRetryIOException e) { 2336 assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>", 2337 e.getMessage()); 2338 } 2339 2340 try { 2341 region.checkAndRowMutate(row, 2342 new SingleColumnValueFilter(fam1, qual1, CompareOperator.EQUAL, value1), 2343 new RowMutations(wrongRow).add((Mutation) new Put(wrongRow).addColumn(fam1, qual1, value1)) 2344 .add((Mutation) new Delete(wrongRow).addColumns(fam1, qual2))); 2345 fail("should throw DoNotRetryIOException"); 2346 } catch (DoNotRetryIOException e) { 2347 assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>", 2348 e.getMessage()); 2349 } 2350 } 2351 2352 @Test 2353 public void testCheckAndMutateWithEmptyRowValue() throws IOException { 2354 byte[] row1 = Bytes.toBytes("row1"); 2355 byte[] fam1 = Bytes.toBytes("fam1"); 2356 byte[] qf1 = Bytes.toBytes("qualifier"); 2357 byte[] emptyVal = new byte[] {}; 2358 byte[] val1 = Bytes.toBytes("value1"); 2359 byte[] val2 = Bytes.toBytes("value2"); 2360 2361 // Setting up region 2362 this.region = initHRegion(tableName, method, CONF, fam1); 2363 // Putting empty data in key 2364 Put put = new Put(row1); 2365 put.addColumn(fam1, qf1, emptyVal); 2366 2367 // checkAndPut with empty value 2368 CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2369 .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(put)); 2370 assertTrue(res.isSuccess()); 2371 assertNull(res.getResult()); 2372 2373 // Putting data in key 2374 put = new Put(row1); 2375 put.addColumn(fam1, qf1, val1); 2376 2377 // checkAndPut with correct value 2378 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2379 .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(put)); 2380 assertTrue(res.isSuccess()); 2381 assertNull(res.getResult()); 2382 2383 // not empty anymore 2384 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2385 .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(put)); 2386 assertFalse(res.isSuccess()); 2387 assertNull(res.getResult()); 2388 2389 Delete delete = new Delete(row1); 2390 delete.addColumn(fam1, qf1); 2391 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2392 .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(delete)); 2393 assertFalse(res.isSuccess()); 2394 assertNull(res.getResult()); 2395 2396 put = new Put(row1); 2397 put.addColumn(fam1, qf1, val2); 2398 // checkAndPut with correct value 2399 res = region.checkAndMutate( 2400 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(put)); 2401 assertTrue(res.isSuccess()); 2402 assertNull(res.getResult()); 2403 2404 // checkAndDelete with correct value 2405 delete = new Delete(row1); 2406 delete.addColumn(fam1, qf1); 2407 delete.addColumn(fam1, qf1); 2408 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2409 .ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(delete)); 2410 assertTrue(res.isSuccess()); 2411 assertNull(res.getResult()); 2412 2413 delete = new Delete(row1); 2414 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2415 .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(delete)); 2416 assertTrue(res.isSuccess()); 2417 assertNull(res.getResult()); 2418 2419 // checkAndPut looking for a null value 2420 put = new Put(row1); 2421 put.addColumn(fam1, qf1, val1); 2422 2423 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1).ifNotExists(fam1, qf1).build(put)); 2424 assertTrue(res.isSuccess()); 2425 assertNull(res.getResult()); 2426 } 2427 2428 @Test 2429 public void testCheckAndMutateWithWrongValue() throws IOException { 2430 byte[] row1 = Bytes.toBytes("row1"); 2431 byte[] fam1 = Bytes.toBytes("fam1"); 2432 byte[] qf1 = Bytes.toBytes("qualifier"); 2433 byte[] val1 = Bytes.toBytes("value1"); 2434 byte[] val2 = Bytes.toBytes("value2"); 2435 BigDecimal bd1 = new BigDecimal(Double.MAX_VALUE); 2436 BigDecimal bd2 = new BigDecimal(Double.MIN_VALUE); 2437 2438 // Setting up region 2439 this.region = initHRegion(tableName, method, CONF, fam1); 2440 // Putting data in key 2441 Put put = new Put(row1); 2442 put.addColumn(fam1, qf1, val1); 2443 region.put(put); 2444 2445 // checkAndPut with wrong value 2446 CheckAndMutateResult res = region.checkAndMutate( 2447 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(put)); 2448 assertFalse(res.isSuccess()); 2449 assertNull(res.getResult()); 2450 2451 // checkAndDelete with wrong value 2452 Delete delete = new Delete(row1); 2453 delete.addFamily(fam1); 2454 res = region.checkAndMutate( 2455 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(put)); 2456 assertFalse(res.isSuccess()); 2457 assertNull(res.getResult()); 2458 2459 // Putting data in key 2460 put = new Put(row1); 2461 put.addColumn(fam1, qf1, Bytes.toBytes(bd1)); 2462 region.put(put); 2463 2464 // checkAndPut with wrong value 2465 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2466 .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd2)).build(put)); 2467 assertFalse(res.isSuccess()); 2468 assertNull(res.getResult()); 2469 2470 // checkAndDelete with wrong value 2471 delete = new Delete(row1); 2472 delete.addFamily(fam1); 2473 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2474 .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd2)).build(delete)); 2475 assertFalse(res.isSuccess()); 2476 assertNull(res.getResult()); 2477 } 2478 2479 @Test 2480 public void testCheckAndMutateWithCorrectValue() throws IOException { 2481 byte[] row1 = Bytes.toBytes("row1"); 2482 byte[] fam1 = Bytes.toBytes("fam1"); 2483 byte[] qf1 = Bytes.toBytes("qualifier"); 2484 byte[] val1 = Bytes.toBytes("value1"); 2485 BigDecimal bd1 = new BigDecimal(Double.MIN_VALUE); 2486 2487 // Setting up region 2488 this.region = initHRegion(tableName, method, CONF, fam1); 2489 // Putting data in key 2490 long now = EnvironmentEdgeManager.currentTime(); 2491 Put put = new Put(row1); 2492 put.addColumn(fam1, qf1, now, val1); 2493 region.put(put); 2494 2495 // checkAndPut with correct value 2496 CheckAndMutateResult res = region.checkAndMutate( 2497 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(put)); 2498 assertTrue("First", res.isSuccess()); 2499 2500 // checkAndDelete with correct value 2501 Delete delete = new Delete(row1, now + 1); 2502 delete.addColumn(fam1, qf1); 2503 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2504 .ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(delete)); 2505 assertTrue("Delete", res.isSuccess()); 2506 assertNull(res.getResult()); 2507 2508 // Putting data in key 2509 put = new Put(row1); 2510 put.addColumn(fam1, qf1, now + 2, Bytes.toBytes(bd1)); 2511 region.put(put); 2512 2513 // checkAndPut with correct value 2514 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2515 .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd1)).build(put)); 2516 assertTrue("Second put", res.isSuccess()); 2517 assertNull(res.getResult()); 2518 2519 // checkAndDelete with correct value 2520 delete = new Delete(row1, now + 3); 2521 delete.addColumn(fam1, qf1); 2522 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2523 .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd1)).build(delete)); 2524 assertTrue("Second delete", res.isSuccess()); 2525 assertNull(res.getResult()); 2526 } 2527 2528 @Test 2529 public void testCheckAndMutateWithNonEqualCompareOp() throws IOException { 2530 byte[] row1 = Bytes.toBytes("row1"); 2531 byte[] fam1 = Bytes.toBytes("fam1"); 2532 byte[] qf1 = Bytes.toBytes("qualifier"); 2533 byte[] val1 = Bytes.toBytes("value1"); 2534 byte[] val2 = Bytes.toBytes("value2"); 2535 byte[] val3 = Bytes.toBytes("value3"); 2536 byte[] val4 = Bytes.toBytes("value4"); 2537 2538 // Setting up region 2539 this.region = initHRegion(tableName, method, CONF, fam1); 2540 // Putting val3 in key 2541 Put put = new Put(row1); 2542 put.addColumn(fam1, qf1, val3); 2543 region.put(put); 2544 2545 // Test CompareOp.LESS: original = val3, compare with val3, fail 2546 CheckAndMutateResult res = region.checkAndMutate( 2547 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.LESS, val3).build(put)); 2548 assertFalse(res.isSuccess()); 2549 assertNull(res.getResult()); 2550 2551 // Test CompareOp.LESS: original = val3, compare with val4, fail 2552 res = region.checkAndMutate( 2553 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.LESS, val4).build(put)); 2554 assertFalse(res.isSuccess()); 2555 assertNull(res.getResult()); 2556 2557 // Test CompareOp.LESS: original = val3, compare with val2, 2558 // succeed (now value = val2) 2559 put = new Put(row1); 2560 put.addColumn(fam1, qf1, val2); 2561 res = region.checkAndMutate( 2562 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.LESS, val2).build(put)); 2563 assertTrue(res.isSuccess()); 2564 assertNull(res.getResult()); 2565 2566 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val3, fail 2567 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2568 .ifMatches(fam1, qf1, CompareOperator.LESS_OR_EQUAL, val3).build(put)); 2569 assertFalse(res.isSuccess()); 2570 assertNull(res.getResult()); 2571 2572 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val2, 2573 // succeed (value still = val2) 2574 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2575 .ifMatches(fam1, qf1, CompareOperator.LESS_OR_EQUAL, val2).build(put)); 2576 assertTrue(res.isSuccess()); 2577 assertNull(res.getResult()); 2578 2579 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val1, 2580 // succeed (now value = val3) 2581 put = new Put(row1); 2582 put.addColumn(fam1, qf1, val3); 2583 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2584 .ifMatches(fam1, qf1, CompareOperator.LESS_OR_EQUAL, val1).build(put)); 2585 assertTrue(res.isSuccess()); 2586 assertNull(res.getResult()); 2587 2588 // Test CompareOp.GREATER: original = val3, compare with val3, fail 2589 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2590 .ifMatches(fam1, qf1, CompareOperator.GREATER, val3).build(put)); 2591 assertFalse(res.isSuccess()); 2592 assertNull(res.getResult()); 2593 2594 // Test CompareOp.GREATER: original = val3, compare with val2, fail 2595 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2596 .ifMatches(fam1, qf1, CompareOperator.GREATER, val2).build(put)); 2597 assertFalse(res.isSuccess()); 2598 assertNull(res.getResult()); 2599 2600 // Test CompareOp.GREATER: original = val3, compare with val4, 2601 // succeed (now value = val2) 2602 put = new Put(row1); 2603 put.addColumn(fam1, qf1, val2); 2604 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2605 .ifMatches(fam1, qf1, CompareOperator.GREATER, val4).build(put)); 2606 assertTrue(res.isSuccess()); 2607 assertNull(res.getResult()); 2608 2609 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val1, fail 2610 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2611 .ifMatches(fam1, qf1, CompareOperator.GREATER_OR_EQUAL, val1).build(put)); 2612 assertFalse(res.isSuccess()); 2613 assertNull(res.getResult()); 2614 2615 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val2, 2616 // succeed (value still = val2) 2617 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2618 .ifMatches(fam1, qf1, CompareOperator.GREATER_OR_EQUAL, val2).build(put)); 2619 assertTrue(res.isSuccess()); 2620 assertNull(res.getResult()); 2621 2622 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val3, succeed 2623 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2624 .ifMatches(fam1, qf1, CompareOperator.GREATER_OR_EQUAL, val3).build(put)); 2625 assertTrue(res.isSuccess()); 2626 assertNull(res.getResult()); 2627 } 2628 2629 @Test 2630 public void testCheckAndPutThatPutWasWritten() throws IOException { 2631 byte[] row1 = Bytes.toBytes("row1"); 2632 byte[] fam1 = Bytes.toBytes("fam1"); 2633 byte[] fam2 = Bytes.toBytes("fam2"); 2634 byte[] qf1 = Bytes.toBytes("qualifier"); 2635 byte[] val1 = Bytes.toBytes("value1"); 2636 byte[] val2 = Bytes.toBytes("value2"); 2637 2638 byte[][] families = { fam1, fam2 }; 2639 2640 // Setting up region 2641 this.region = initHRegion(tableName, method, CONF, families); 2642 // Putting data in the key to check 2643 Put put = new Put(row1); 2644 put.addColumn(fam1, qf1, val1); 2645 region.put(put); 2646 2647 // Creating put to add 2648 long ts = EnvironmentEdgeManager.currentTime(); 2649 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2); 2650 put = new Put(row1); 2651 put.add(kv); 2652 2653 // checkAndPut with wrong value 2654 CheckAndMutateResult res = region.checkAndMutate( 2655 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(put)); 2656 assertTrue(res.isSuccess()); 2657 assertNull(res.getResult()); 2658 2659 Get get = new Get(row1); 2660 get.addColumn(fam2, qf1); 2661 Cell[] actual = region.get(get).rawCells(); 2662 2663 Cell[] expected = { kv }; 2664 2665 assertEquals(expected.length, actual.length); 2666 for (int i = 0; i < actual.length; i++) { 2667 assertEquals(expected[i], actual[i]); 2668 } 2669 } 2670 2671 @Test 2672 public void testCheckAndDeleteThatDeleteWasWritten() throws IOException { 2673 byte[] row1 = Bytes.toBytes("row1"); 2674 byte[] fam1 = Bytes.toBytes("fam1"); 2675 byte[] fam2 = Bytes.toBytes("fam2"); 2676 byte[] qf1 = Bytes.toBytes("qualifier1"); 2677 byte[] qf2 = Bytes.toBytes("qualifier2"); 2678 byte[] qf3 = Bytes.toBytes("qualifier3"); 2679 byte[] val1 = Bytes.toBytes("value1"); 2680 byte[] val2 = Bytes.toBytes("value2"); 2681 byte[] val3 = Bytes.toBytes("value3"); 2682 byte[] emptyVal = new byte[] {}; 2683 2684 byte[][] families = { fam1, fam2 }; 2685 2686 // Setting up region 2687 this.region = initHRegion(tableName, method, CONF, families); 2688 // Put content 2689 Put put = new Put(row1); 2690 put.addColumn(fam1, qf1, val1); 2691 region.put(put); 2692 Threads.sleep(2); 2693 2694 put = new Put(row1); 2695 put.addColumn(fam1, qf1, val2); 2696 put.addColumn(fam2, qf1, val3); 2697 put.addColumn(fam2, qf2, val2); 2698 put.addColumn(fam2, qf3, val1); 2699 put.addColumn(fam1, qf3, val1); 2700 region.put(put); 2701 2702 LOG.info("get={}", region.get(new Get(row1).addColumn(fam1, qf1)).toString()); 2703 2704 // Multi-column delete 2705 Delete delete = new Delete(row1); 2706 delete.addColumn(fam1, qf1); 2707 delete.addColumn(fam2, qf1); 2708 delete.addColumn(fam1, qf3); 2709 CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2710 .ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(delete)); 2711 assertTrue(res.isSuccess()); 2712 assertNull(res.getResult()); 2713 2714 Get get = new Get(row1); 2715 get.addColumn(fam1, qf1); 2716 get.addColumn(fam1, qf3); 2717 get.addColumn(fam2, qf2); 2718 Result r = region.get(get); 2719 assertEquals(2, r.size()); 2720 assertArrayEquals(val1, r.getValue(fam1, qf1)); 2721 assertArrayEquals(val2, r.getValue(fam2, qf2)); 2722 2723 // Family delete 2724 delete = new Delete(row1); 2725 delete.addFamily(fam2); 2726 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2727 .ifMatches(fam2, qf1, CompareOperator.EQUAL, emptyVal).build(delete)); 2728 assertTrue(res.isSuccess()); 2729 assertNull(res.getResult()); 2730 2731 get = new Get(row1); 2732 r = region.get(get); 2733 assertEquals(1, r.size()); 2734 assertArrayEquals(val1, r.getValue(fam1, qf1)); 2735 2736 // Row delete 2737 delete = new Delete(row1); 2738 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2739 .ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(delete)); 2740 assertTrue(res.isSuccess()); 2741 assertNull(res.getResult()); 2742 2743 get = new Get(row1); 2744 r = region.get(get); 2745 assertEquals(0, r.size()); 2746 } 2747 2748 @Test 2749 public void testCheckAndMutateWithFilters() throws Throwable { 2750 final byte[] FAMILY = Bytes.toBytes("fam"); 2751 2752 // Setting up region 2753 this.region = initHRegion(tableName, method, CONF, FAMILY); 2754 2755 // Put one row 2756 Put put = new Put(row); 2757 put.addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")); 2758 put.addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")); 2759 put.addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")); 2760 region.put(put); 2761 2762 // Put with success 2763 CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2764 .ifMatches(new FilterList( 2765 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2766 Bytes.toBytes("a")), 2767 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2768 Bytes.toBytes("b")))) 2769 .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d")))); 2770 assertTrue(res.isSuccess()); 2771 assertNull(res.getResult()); 2772 2773 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))); 2774 assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D")))); 2775 2776 // Put with failure 2777 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2778 .ifMatches(new FilterList( 2779 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2780 Bytes.toBytes("a")), 2781 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2782 Bytes.toBytes("c")))) 2783 .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e")))); 2784 assertFalse(res.isSuccess()); 2785 assertNull(res.getResult()); 2786 2787 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E"))).isEmpty()); 2788 2789 // Delete with success 2790 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2791 .ifMatches(new FilterList( 2792 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2793 Bytes.toBytes("a")), 2794 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2795 Bytes.toBytes("b")))) 2796 .build(new Delete(row).addColumns(FAMILY, Bytes.toBytes("D")))); 2797 assertTrue(res.isSuccess()); 2798 assertNull(res.getResult()); 2799 2800 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))).isEmpty()); 2801 2802 // Mutate with success 2803 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2804 .ifMatches(new FilterList( 2805 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2806 Bytes.toBytes("a")), 2807 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2808 Bytes.toBytes("b")))) 2809 .build(new RowMutations(row) 2810 .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e"))) 2811 .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A"))))); 2812 assertTrue(res.isSuccess()); 2813 assertNull(res.getResult()); 2814 2815 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E"))); 2816 assertEquals("e", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("E")))); 2817 2818 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty()); 2819 } 2820 2821 @Test 2822 public void testCheckAndMutateWithFiltersAndTimeRange() throws Throwable { 2823 final byte[] FAMILY = Bytes.toBytes("fam"); 2824 2825 // Setting up region 2826 this.region = initHRegion(tableName, method, CONF, FAMILY); 2827 2828 // Put with specifying the timestamp 2829 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), 100, Bytes.toBytes("a"))); 2830 2831 // Put with success 2832 CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2833 .ifMatches(new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2834 Bytes.toBytes("a"))) 2835 .timeRange(TimeRange.between(0, 101)) 2836 .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")))); 2837 assertTrue(res.isSuccess()); 2838 assertNull(res.getResult()); 2839 2840 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2841 assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2842 2843 // Put with failure 2844 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2845 .ifMatches(new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2846 Bytes.toBytes("a"))) 2847 .timeRange(TimeRange.between(0, 100)) 2848 .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")))); 2849 assertFalse(res.isSuccess()); 2850 assertNull(res.getResult()); 2851 2852 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("C"))).isEmpty()); 2853 2854 // RowMutations with success 2855 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2856 .ifMatches(new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2857 Bytes.toBytes("a"))) 2858 .timeRange(TimeRange.between(0, 101)) 2859 .build(new RowMutations(row) 2860 .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d"))) 2861 .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A"))))); 2862 assertTrue(res.isSuccess()); 2863 assertNull(res.getResult()); 2864 2865 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))); 2866 assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D")))); 2867 2868 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty()); 2869 } 2870 2871 @Test 2872 public void testCheckAndIncrement() throws Throwable { 2873 final byte[] FAMILY = Bytes.toBytes("fam"); 2874 2875 // Setting up region 2876 this.region = initHRegion(tableName, method, CONF, FAMILY); 2877 2878 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a"))); 2879 2880 // CheckAndIncrement with correct value 2881 CheckAndMutateResult res = region.checkAndMutate( 2882 CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")) 2883 .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 1))); 2884 assertTrue(res.isSuccess()); 2885 assertEquals(1, Bytes.toLong(res.getResult().getValue(FAMILY, Bytes.toBytes("B")))); 2886 2887 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2888 assertEquals(1, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B")))); 2889 2890 // CheckAndIncrement with wrong value 2891 res = region.checkAndMutate( 2892 CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("b")) 2893 .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 1))); 2894 assertFalse(res.isSuccess()); 2895 assertNull(res.getResult()); 2896 2897 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2898 assertEquals(1, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B")))); 2899 2900 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c"))); 2901 2902 // CheckAndIncrement with a filter and correct value 2903 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2904 .ifMatches(new FilterList( 2905 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2906 Bytes.toBytes("a")), 2907 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL, 2908 Bytes.toBytes("c")))) 2909 .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 2))); 2910 assertTrue(res.isSuccess()); 2911 assertEquals(3, Bytes.toLong(res.getResult().getValue(FAMILY, Bytes.toBytes("B")))); 2912 2913 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2914 assertEquals(3, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B")))); 2915 2916 // CheckAndIncrement with a filter and correct value 2917 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2918 .ifMatches(new FilterList( 2919 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2920 Bytes.toBytes("b")), 2921 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL, 2922 Bytes.toBytes("d")))) 2923 .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 2))); 2924 assertFalse(res.isSuccess()); 2925 assertNull(res.getResult()); 2926 2927 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2928 assertEquals(3, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B")))); 2929 } 2930 2931 @Test 2932 public void testCheckAndAppend() throws Throwable { 2933 final byte[] FAMILY = Bytes.toBytes("fam"); 2934 2935 // Setting up region 2936 this.region = initHRegion(tableName, method, CONF, FAMILY); 2937 2938 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a"))); 2939 2940 // CheckAndAppend with correct value 2941 CheckAndMutateResult res = region.checkAndMutate( 2942 CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")) 2943 .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")))); 2944 assertTrue(res.isSuccess()); 2945 assertEquals("b", Bytes.toString(res.getResult().getValue(FAMILY, Bytes.toBytes("B")))); 2946 2947 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2948 assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2949 2950 // CheckAndAppend with wrong value 2951 res = region.checkAndMutate( 2952 CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("b")) 2953 .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")))); 2954 assertFalse(res.isSuccess()); 2955 assertNull(res.getResult()); 2956 2957 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2958 assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2959 2960 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c"))); 2961 2962 // CheckAndAppend with a filter and correct value 2963 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2964 .ifMatches(new FilterList( 2965 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2966 Bytes.toBytes("a")), 2967 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL, 2968 Bytes.toBytes("c")))) 2969 .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("bb")))); 2970 assertTrue(res.isSuccess()); 2971 assertEquals("bbb", Bytes.toString(res.getResult().getValue(FAMILY, Bytes.toBytes("B")))); 2972 2973 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2974 assertEquals("bbb", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2975 2976 // CheckAndAppend with a filter and wrong value 2977 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2978 .ifMatches(new FilterList( 2979 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2980 Bytes.toBytes("b")), 2981 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL, 2982 Bytes.toBytes("d")))) 2983 .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("bb")))); 2984 assertFalse(res.isSuccess()); 2985 assertNull(res.getResult()); 2986 2987 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2988 assertEquals("bbb", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2989 } 2990 2991 @Test 2992 public void testCheckAndIncrementAndAppend() throws Throwable { 2993 // Setting up region 2994 this.region = initHRegion(tableName, method, CONF, fam1); 2995 2996 // CheckAndMutate with Increment and Append 2997 CheckAndMutate checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, qual) 2998 .build(new RowMutations(row).add((Mutation) new Increment(row).addColumn(fam1, qual1, 1L)) 2999 .add((Mutation) new Append(row).addColumn(fam1, qual2, Bytes.toBytes("a")))); 3000 3001 CheckAndMutateResult result = region.checkAndMutate(checkAndMutate); 3002 assertTrue(result.isSuccess()); 3003 assertEquals(1L, Bytes.toLong(result.getResult().getValue(fam1, qual1))); 3004 assertEquals("a", Bytes.toString(result.getResult().getValue(fam1, qual2))); 3005 3006 Result r = region.get(new Get(row)); 3007 assertEquals(1L, Bytes.toLong(r.getValue(fam1, qual1))); 3008 assertEquals("a", Bytes.toString(r.getValue(fam1, qual2))); 3009 3010 // Set return results to false 3011 checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, qual) 3012 .build(new RowMutations(row) 3013 .add((Mutation) new Increment(row).addColumn(fam1, qual1, 1L).setReturnResults(false)) 3014 .add((Mutation) new Append(row).addColumn(fam1, qual2, Bytes.toBytes("a")) 3015 .setReturnResults(false))); 3016 3017 result = region.checkAndMutate(checkAndMutate); 3018 assertTrue(result.isSuccess()); 3019 assertNull(result.getResult().getValue(fam1, qual1)); 3020 assertNull(result.getResult().getValue(fam1, qual2)); 3021 3022 r = region.get(new Get(row)); 3023 assertEquals(2L, Bytes.toLong(r.getValue(fam1, qual1))); 3024 assertEquals("aa", Bytes.toString(r.getValue(fam1, qual2))); 3025 3026 checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, qual) 3027 .build(new RowMutations(row).add((Mutation) new Increment(row).addColumn(fam1, qual1, 1L)) 3028 .add((Mutation) new Append(row).addColumn(fam1, qual2, Bytes.toBytes("a")) 3029 .setReturnResults(false))); 3030 3031 result = region.checkAndMutate(checkAndMutate); 3032 assertTrue(result.isSuccess()); 3033 assertEquals(3L, Bytes.toLong(result.getResult().getValue(fam1, qual1))); 3034 assertNull(result.getResult().getValue(fam1, qual2)); 3035 3036 r = region.get(new Get(row)); 3037 assertEquals(3L, Bytes.toLong(r.getValue(fam1, qual1))); 3038 assertEquals("aaa", Bytes.toString(r.getValue(fam1, qual2))); 3039 } 3040 3041 @Test 3042 public void testCheckAndRowMutations() throws Throwable { 3043 final byte[] row = Bytes.toBytes("row"); 3044 final byte[] q1 = Bytes.toBytes("q1"); 3045 final byte[] q2 = Bytes.toBytes("q2"); 3046 final byte[] q3 = Bytes.toBytes("q3"); 3047 final byte[] q4 = Bytes.toBytes("q4"); 3048 final String v1 = "v1"; 3049 3050 region = initHRegion(tableName, method, CONF, fam1); 3051 3052 // Initial values 3053 region 3054 .batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q2, Bytes.toBytes("toBeDeleted")), 3055 new Put(row).addColumn(fam1, q3, Bytes.toBytes(5L)), 3056 new Put(row).addColumn(fam1, q4, Bytes.toBytes("a")), }); 3057 3058 // Do CheckAndRowMutations 3059 CheckAndMutate checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, q1).build( 3060 new RowMutations(row).add(Arrays.asList(new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1)), 3061 new Delete(row).addColumns(fam1, q2), new Increment(row).addColumn(fam1, q3, 1), 3062 new Append(row).addColumn(fam1, q4, Bytes.toBytes("b"))))); 3063 3064 CheckAndMutateResult result = region.checkAndMutate(checkAndMutate); 3065 assertTrue(result.isSuccess()); 3066 assertEquals(6L, Bytes.toLong(result.getResult().getValue(fam1, q3))); 3067 assertEquals("ab", Bytes.toString(result.getResult().getValue(fam1, q4))); 3068 3069 // Verify the value 3070 Result r = region.get(new Get(row)); 3071 assertEquals(v1, Bytes.toString(r.getValue(fam1, q1))); 3072 assertNull(r.getValue(fam1, q2)); 3073 assertEquals(6L, Bytes.toLong(r.getValue(fam1, q3))); 3074 assertEquals("ab", Bytes.toString(r.getValue(fam1, q4))); 3075 3076 // Do CheckAndRowMutations again 3077 checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, q1) 3078 .build(new RowMutations(row).add(Arrays.asList(new Delete(row).addColumns(fam1, q1), 3079 new Put(row).addColumn(fam1, q2, Bytes.toBytes(v1)), 3080 new Increment(row).addColumn(fam1, q3, 1), 3081 new Append(row).addColumn(fam1, q4, Bytes.toBytes("b"))))); 3082 3083 result = region.checkAndMutate(checkAndMutate); 3084 assertFalse(result.isSuccess()); 3085 assertNull(result.getResult()); 3086 3087 // Verify the value 3088 r = region.get(new Get(row)); 3089 assertEquals(v1, Bytes.toString(r.getValue(fam1, q1))); 3090 assertNull(r.getValue(fam1, q2)); 3091 assertEquals(6L, Bytes.toLong(r.getValue(fam1, q3))); 3092 assertEquals("ab", Bytes.toString(r.getValue(fam1, q4))); 3093 } 3094 3095 // //////////////////////////////////////////////////////////////////////////// 3096 // Delete tests 3097 // //////////////////////////////////////////////////////////////////////////// 3098 @Test 3099 public void testDelete_multiDeleteColumn() throws IOException { 3100 byte[] row1 = Bytes.toBytes("row1"); 3101 byte[] fam1 = Bytes.toBytes("fam1"); 3102 byte[] qual = Bytes.toBytes("qualifier"); 3103 byte[] value = Bytes.toBytes("value"); 3104 3105 Put put = new Put(row1); 3106 put.addColumn(fam1, qual, 1, value); 3107 put.addColumn(fam1, qual, 2, value); 3108 3109 this.region = initHRegion(tableName, method, CONF, fam1); 3110 region.put(put); 3111 3112 // We do support deleting more than 1 'latest' version 3113 Delete delete = new Delete(row1); 3114 delete.addColumn(fam1, qual); 3115 delete.addColumn(fam1, qual); 3116 region.delete(delete); 3117 3118 Get get = new Get(row1); 3119 get.addFamily(fam1); 3120 Result r = region.get(get); 3121 assertEquals(0, r.size()); 3122 } 3123 3124 @Test 3125 public void testDelete_CheckFamily() throws IOException { 3126 byte[] row1 = Bytes.toBytes("row1"); 3127 byte[] fam1 = Bytes.toBytes("fam1"); 3128 byte[] fam2 = Bytes.toBytes("fam2"); 3129 byte[] fam3 = Bytes.toBytes("fam3"); 3130 byte[] fam4 = Bytes.toBytes("fam4"); 3131 3132 // Setting up region 3133 this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3); 3134 List<Cell> kvs = new ArrayList<>(); 3135 kvs.add(new KeyValue(row1, fam4, null, null)); 3136 3137 byte[] forUnitTestsOnly = Bytes.toBytes("ForUnitTestsOnly"); 3138 3139 // testing existing family 3140 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<>(Bytes.BYTES_COMPARATOR); 3141 deleteMap.put(fam2, kvs); 3142 region.delete(new Delete(forUnitTestsOnly, HConstants.LATEST_TIMESTAMP, deleteMap)); 3143 3144 // testing non existing family 3145 NavigableMap<byte[], List<Cell>> deleteMap2 = new TreeMap<>(Bytes.BYTES_COMPARATOR); 3146 deleteMap2.put(fam4, kvs); 3147 assertThrows("Family " + Bytes.toString(fam4) + " does exist", 3148 NoSuchColumnFamilyException.class, 3149 () -> region.delete(new Delete(forUnitTestsOnly, HConstants.LATEST_TIMESTAMP, deleteMap2))); 3150 } 3151 3152 @Test 3153 public void testDelete_mixed() throws IOException, InterruptedException { 3154 byte[] fam = Bytes.toBytes("info"); 3155 byte[][] families = { fam }; 3156 this.region = initHRegion(tableName, method, CONF, families); 3157 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge()); 3158 3159 byte[] row = Bytes.toBytes("table_name"); 3160 // column names 3161 byte[] serverinfo = Bytes.toBytes("serverinfo"); 3162 byte[] splitA = Bytes.toBytes("splitA"); 3163 byte[] splitB = Bytes.toBytes("splitB"); 3164 3165 // add some data: 3166 Put put = new Put(row); 3167 put.addColumn(fam, splitA, Bytes.toBytes("reference_A")); 3168 region.put(put); 3169 3170 put = new Put(row); 3171 put.addColumn(fam, splitB, Bytes.toBytes("reference_B")); 3172 region.put(put); 3173 3174 put = new Put(row); 3175 put.addColumn(fam, serverinfo, Bytes.toBytes("ip_address")); 3176 region.put(put); 3177 3178 // ok now delete a split: 3179 Delete delete = new Delete(row); 3180 delete.addColumns(fam, splitA); 3181 region.delete(delete); 3182 3183 // assert some things: 3184 Get get = new Get(row).addColumn(fam, serverinfo); 3185 Result result = region.get(get); 3186 assertEquals(1, result.size()); 3187 3188 get = new Get(row).addColumn(fam, splitA); 3189 result = region.get(get); 3190 assertEquals(0, result.size()); 3191 3192 get = new Get(row).addColumn(fam, splitB); 3193 result = region.get(get); 3194 assertEquals(1, result.size()); 3195 3196 // Assert that after a delete, I can put. 3197 put = new Put(row); 3198 put.addColumn(fam, splitA, Bytes.toBytes("reference_A")); 3199 region.put(put); 3200 get = new Get(row); 3201 result = region.get(get); 3202 assertEquals(3, result.size()); 3203 3204 // Now delete all... then test I can add stuff back 3205 delete = new Delete(row); 3206 region.delete(delete); 3207 assertEquals(0, region.get(get).size()); 3208 3209 region.put(new Put(row).addColumn(fam, splitA, Bytes.toBytes("reference_A"))); 3210 result = region.get(get); 3211 assertEquals(1, result.size()); 3212 } 3213 3214 @Test 3215 public void testDeleteRowWithFutureTs() throws IOException { 3216 byte[] fam = Bytes.toBytes("info"); 3217 byte[][] families = { fam }; 3218 this.region = initHRegion(tableName, method, CONF, families); 3219 byte[] row = Bytes.toBytes("table_name"); 3220 // column names 3221 byte[] serverinfo = Bytes.toBytes("serverinfo"); 3222 3223 // add data in the far future 3224 Put put = new Put(row); 3225 put.addColumn(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value")); 3226 region.put(put); 3227 3228 // now delete something in the present 3229 Delete delete = new Delete(row); 3230 region.delete(delete); 3231 3232 // make sure we still see our data 3233 Get get = new Get(row).addColumn(fam, serverinfo); 3234 Result result = region.get(get); 3235 assertEquals(1, result.size()); 3236 3237 // delete the future row 3238 delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3); 3239 region.delete(delete); 3240 3241 // make sure it is gone 3242 get = new Get(row).addColumn(fam, serverinfo); 3243 result = region.get(get); 3244 assertEquals(0, result.size()); 3245 } 3246 3247 /** 3248 * Tests that the special LATEST_TIMESTAMP option for puts gets replaced by the actual timestamp 3249 */ 3250 @Test 3251 public void testPutWithLatestTS() throws IOException { 3252 byte[] fam = Bytes.toBytes("info"); 3253 byte[][] families = { fam }; 3254 this.region = initHRegion(tableName, method, CONF, families); 3255 byte[] row = Bytes.toBytes("row1"); 3256 // column names 3257 byte[] qual = Bytes.toBytes("qual"); 3258 3259 // add data with LATEST_TIMESTAMP, put without WAL 3260 Put put = new Put(row); 3261 put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value")); 3262 region.put(put); 3263 3264 // Make sure it shows up with an actual timestamp 3265 Get get = new Get(row).addColumn(fam, qual); 3266 Result result = region.get(get); 3267 assertEquals(1, result.size()); 3268 Cell kv = result.rawCells()[0]; 3269 LOG.info("Got: " + kv); 3270 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp", 3271 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP); 3272 3273 // Check same with WAL enabled (historically these took different 3274 // code paths, so check both) 3275 row = Bytes.toBytes("row2"); 3276 put = new Put(row); 3277 put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value")); 3278 region.put(put); 3279 3280 // Make sure it shows up with an actual timestamp 3281 get = new Get(row).addColumn(fam, qual); 3282 result = region.get(get); 3283 assertEquals(1, result.size()); 3284 kv = result.rawCells()[0]; 3285 LOG.info("Got: " + kv); 3286 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp", 3287 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP); 3288 } 3289 3290 /** 3291 * Tests that there is server-side filtering for invalid timestamp upper bound. Note that the 3292 * timestamp lower bound is automatically handled for us by the TTL field. 3293 */ 3294 @Test 3295 public void testPutWithTsSlop() throws IOException { 3296 byte[] fam = Bytes.toBytes("info"); 3297 byte[][] families = { fam }; 3298 3299 // add data with a timestamp that is too recent for range. Ensure assert 3300 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000); 3301 this.region = initHRegion(tableName, method, CONF, families); 3302 boolean caughtExcep = false; 3303 try { 3304 // no TS specified == use latest. should not error 3305 region.put(new Put(row).addColumn(fam, Bytes.toBytes("qual"), Bytes.toBytes("value"))); 3306 // TS out of range. should error 3307 region.put(new Put(row).addColumn(fam, Bytes.toBytes("qual"), 3308 EnvironmentEdgeManager.currentTime() + 2000, Bytes.toBytes("value"))); 3309 fail("Expected IOE for TS out of configured timerange"); 3310 } catch (FailedSanityCheckException ioe) { 3311 LOG.debug("Received expected exception", ioe); 3312 caughtExcep = true; 3313 } 3314 assertTrue("Should catch FailedSanityCheckException", caughtExcep); 3315 } 3316 3317 @Test 3318 public void testScanner_DeleteOneFamilyNotAnother() throws IOException { 3319 byte[] fam1 = Bytes.toBytes("columnA"); 3320 byte[] fam2 = Bytes.toBytes("columnB"); 3321 this.region = initHRegion(tableName, method, CONF, fam1, fam2); 3322 byte[] rowA = Bytes.toBytes("rowA"); 3323 byte[] rowB = Bytes.toBytes("rowB"); 3324 3325 byte[] value = Bytes.toBytes("value"); 3326 3327 Delete delete = new Delete(rowA); 3328 delete.addFamily(fam1); 3329 3330 region.delete(delete); 3331 3332 // now create data. 3333 Put put = new Put(rowA); 3334 put.addColumn(fam2, null, value); 3335 region.put(put); 3336 3337 put = new Put(rowB); 3338 put.addColumn(fam1, null, value); 3339 put.addColumn(fam2, null, value); 3340 region.put(put); 3341 3342 Scan scan = new Scan(); 3343 scan.addFamily(fam1).addFamily(fam2); 3344 try (InternalScanner s = region.getScanner(scan)) { 3345 List<Cell> results = new ArrayList<>(); 3346 s.next(results); 3347 assertTrue(CellUtil.matchingRows(results.get(0), rowA)); 3348 3349 results.clear(); 3350 s.next(results); 3351 assertTrue(CellUtil.matchingRows(results.get(0), rowB)); 3352 } 3353 } 3354 3355 @Test 3356 public void testDataInMemoryWithoutWAL() throws IOException { 3357 FileSystem fs = FileSystem.get(CONF); 3358 Path rootDir = new Path(dir + "testDataInMemoryWithoutWAL"); 3359 FSHLog hLog = new FSHLog(fs, rootDir, "testDataInMemoryWithoutWAL", CONF); 3360 hLog.init(); 3361 // This chunk creation is done throughout the code base. Do we want to move it into core? 3362 // It is missing from this test. W/o it we NPE. 3363 region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, hLog, 3364 COLUMN_FAMILY_BYTES); 3365 3366 Cell originalCell = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row) 3367 .setFamily(COLUMN_FAMILY_BYTES).setQualifier(qual1) 3368 .setTimestamp(EnvironmentEdgeManager.currentTime()).setType(KeyValue.Type.Put.getCode()) 3369 .setValue(value1).build(); 3370 final long originalSize = originalCell.getSerializedSize(); 3371 3372 Cell addCell = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row) 3373 .setFamily(COLUMN_FAMILY_BYTES).setQualifier(qual1) 3374 .setTimestamp(EnvironmentEdgeManager.currentTime()).setType(KeyValue.Type.Put.getCode()) 3375 .setValue(Bytes.toBytes("xxxxxxxxxx")).build(); 3376 final long addSize = addCell.getSerializedSize(); 3377 3378 LOG.info("originalSize:" + originalSize + ", addSize:" + addSize); 3379 // start test. We expect that the addPut's durability will be replaced 3380 // by originalPut's durability. 3381 3382 // case 1: 3383 testDataInMemoryWithoutWAL(region, 3384 new Put(row).add(originalCell).setDurability(Durability.SKIP_WAL), 3385 new Put(row).add(addCell).setDurability(Durability.SKIP_WAL), originalSize + addSize); 3386 3387 // case 2: 3388 testDataInMemoryWithoutWAL(region, 3389 new Put(row).add(originalCell).setDurability(Durability.SKIP_WAL), 3390 new Put(row).add(addCell).setDurability(Durability.SYNC_WAL), originalSize + addSize); 3391 3392 // case 3: 3393 testDataInMemoryWithoutWAL(region, 3394 new Put(row).add(originalCell).setDurability(Durability.SYNC_WAL), 3395 new Put(row).add(addCell).setDurability(Durability.SKIP_WAL), 0); 3396 3397 // case 4: 3398 testDataInMemoryWithoutWAL(region, 3399 new Put(row).add(originalCell).setDurability(Durability.SYNC_WAL), 3400 new Put(row).add(addCell).setDurability(Durability.SYNC_WAL), 0); 3401 } 3402 3403 private static void testDataInMemoryWithoutWAL(HRegion region, Put originalPut, final Put addPut, 3404 long delta) throws IOException { 3405 final long initSize = region.getDataInMemoryWithoutWAL(); 3406 // save normalCPHost and replaced by mockedCPHost 3407 RegionCoprocessorHost normalCPHost = region.getCoprocessorHost(); 3408 RegionCoprocessorHost mockedCPHost = mock(RegionCoprocessorHost.class); 3409 // Because the preBatchMutate returns void, we can't do usual Mockito when...then form. Must 3410 // do below format (from Mockito doc). 3411 doAnswer(new Answer<Void>() { 3412 @Override 3413 public Void answer(InvocationOnMock invocation) throws Throwable { 3414 MiniBatchOperationInProgress<Mutation> mb = invocation.getArgument(0); 3415 mb.addOperationsFromCP(0, new Mutation[] { addPut }); 3416 return null; 3417 } 3418 }).when(mockedCPHost).preBatchMutate(isA(MiniBatchOperationInProgress.class)); 3419 ColumnFamilyDescriptorBuilder builder = 3420 ColumnFamilyDescriptorBuilder.newBuilder(COLUMN_FAMILY_BYTES); 3421 ScanInfo info = new ScanInfo(CONF, builder.build(), Long.MAX_VALUE, Long.MAX_VALUE, 3422 region.getCellComparator()); 3423 when(mockedCPHost.preFlushScannerOpen(any(HStore.class), any())).thenReturn(info); 3424 3425 when(mockedCPHost.preFlush(any(), any(StoreScanner.class), any())) 3426 .thenAnswer(i -> i.getArgument(1)); 3427 region.setCoprocessorHost(mockedCPHost); 3428 3429 region.put(originalPut); 3430 region.setCoprocessorHost(normalCPHost); 3431 final long finalSize = region.getDataInMemoryWithoutWAL(); 3432 assertEquals("finalSize:" + finalSize + ", initSize:" + initSize + ", delta:" + delta, 3433 finalSize, initSize + delta); 3434 } 3435 3436 @Test 3437 public void testDeleteColumns_PostInsert() throws IOException, InterruptedException { 3438 Delete delete = new Delete(row); 3439 delete.addColumns(fam1, qual1); 3440 doTestDelete_AndPostInsert(delete); 3441 } 3442 3443 @Test 3444 public void testaddFamily_PostInsert() throws IOException, InterruptedException { 3445 Delete delete = new Delete(row); 3446 delete.addFamily(fam1); 3447 doTestDelete_AndPostInsert(delete); 3448 } 3449 3450 public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException { 3451 this.region = initHRegion(tableName, method, CONF, fam1); 3452 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge()); 3453 Put put = new Put(row); 3454 put.addColumn(fam1, qual1, value1); 3455 region.put(put); 3456 3457 // now delete the value: 3458 region.delete(delete); 3459 3460 // ok put data: 3461 put = new Put(row); 3462 put.addColumn(fam1, qual1, value2); 3463 region.put(put); 3464 3465 // ok get: 3466 Get get = new Get(row); 3467 get.addColumn(fam1, qual1); 3468 3469 Result r = region.get(get); 3470 assertEquals(1, r.size()); 3471 assertArrayEquals(value2, r.getValue(fam1, qual1)); 3472 3473 // next: 3474 Scan scan = new Scan().withStartRow(row); 3475 scan.addColumn(fam1, qual1); 3476 try (InternalScanner s = region.getScanner(scan)) { 3477 List<Cell> results = new ArrayList<>(); 3478 assertEquals(false, s.next(results)); 3479 assertEquals(1, results.size()); 3480 Cell kv = results.get(0); 3481 3482 assertArrayEquals(value2, CellUtil.cloneValue(kv)); 3483 assertArrayEquals(fam1, CellUtil.cloneFamily(kv)); 3484 assertArrayEquals(qual1, CellUtil.cloneQualifier(kv)); 3485 assertArrayEquals(row, CellUtil.cloneRow(kv)); 3486 } 3487 } 3488 3489 @Test 3490 public void testDelete_CheckTimestampUpdated() throws IOException { 3491 byte[] row1 = Bytes.toBytes("row1"); 3492 byte[] col1 = Bytes.toBytes("col1"); 3493 byte[] col2 = Bytes.toBytes("col2"); 3494 byte[] col3 = Bytes.toBytes("col3"); 3495 3496 byte[] forUnitTestsOnly = Bytes.toBytes("ForUnitTestsOnly"); 3497 3498 // Setting up region 3499 this.region = initHRegion(tableName, method, CONF, fam1); 3500 // Building checkerList 3501 List<Cell> kvs = new ArrayList<>(); 3502 kvs.add(new KeyValue(row1, fam1, col1, null)); 3503 kvs.add(new KeyValue(row1, fam1, col2, null)); 3504 kvs.add(new KeyValue(row1, fam1, col3, null)); 3505 3506 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<>(Bytes.BYTES_COMPARATOR); 3507 deleteMap.put(fam1, kvs); 3508 region.delete(new Delete(forUnitTestsOnly, HConstants.LATEST_TIMESTAMP, deleteMap)); 3509 3510 // extract the key values out the memstore: 3511 // This is kinda hacky, but better than nothing... 3512 long now = EnvironmentEdgeManager.currentTime(); 3513 AbstractMemStore memstore = (AbstractMemStore) region.getStore(fam1).memstore; 3514 Cell firstCell = memstore.getActive().first(); 3515 assertTrue(firstCell.getTimestamp() <= now); 3516 now = firstCell.getTimestamp(); 3517 for (Cell cell : memstore.getActive().getCellSet()) { 3518 assertTrue(cell.getTimestamp() <= now); 3519 now = cell.getTimestamp(); 3520 } 3521 } 3522 3523 // //////////////////////////////////////////////////////////////////////////// 3524 // Get tests 3525 // //////////////////////////////////////////////////////////////////////////// 3526 @Test 3527 public void testGet_FamilyChecker() throws IOException { 3528 byte[] row1 = Bytes.toBytes("row1"); 3529 byte[] fam1 = Bytes.toBytes("fam1"); 3530 byte[] fam2 = Bytes.toBytes("False"); 3531 byte[] col1 = Bytes.toBytes("col1"); 3532 3533 // Setting up region 3534 this.region = initHRegion(tableName, method, CONF, fam1); 3535 Get get = new Get(row1); 3536 get.addColumn(fam2, col1); 3537 3538 // Test 3539 try { 3540 region.get(get); 3541 fail("Expecting DoNotRetryIOException in get but did not get any"); 3542 } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) { 3543 LOG.info("Got expected DoNotRetryIOException successfully"); 3544 } 3545 } 3546 3547 @Test 3548 public void testGet_Basic() throws IOException { 3549 byte[] row1 = Bytes.toBytes("row1"); 3550 byte[] fam1 = Bytes.toBytes("fam1"); 3551 byte[] col1 = Bytes.toBytes("col1"); 3552 byte[] col2 = Bytes.toBytes("col2"); 3553 byte[] col3 = Bytes.toBytes("col3"); 3554 byte[] col4 = Bytes.toBytes("col4"); 3555 byte[] col5 = Bytes.toBytes("col5"); 3556 3557 // Setting up region 3558 this.region = initHRegion(tableName, method, CONF, fam1); 3559 // Add to memstore 3560 Put put = new Put(row1); 3561 put.addColumn(fam1, col1, null); 3562 put.addColumn(fam1, col2, null); 3563 put.addColumn(fam1, col3, null); 3564 put.addColumn(fam1, col4, null); 3565 put.addColumn(fam1, col5, null); 3566 region.put(put); 3567 3568 Get get = new Get(row1); 3569 get.addColumn(fam1, col2); 3570 get.addColumn(fam1, col4); 3571 // Expected result 3572 KeyValue kv1 = new KeyValue(row1, fam1, col2); 3573 KeyValue kv2 = new KeyValue(row1, fam1, col4); 3574 KeyValue[] expected = { kv1, kv2 }; 3575 3576 // Test 3577 Result res = region.get(get); 3578 assertEquals(expected.length, res.size()); 3579 for (int i = 0; i < res.size(); i++) { 3580 assertTrue(CellUtil.matchingRows(expected[i], res.rawCells()[i])); 3581 assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i])); 3582 assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i])); 3583 } 3584 3585 // Test using a filter on a Get 3586 Get g = new Get(row1); 3587 final int count = 2; 3588 g.setFilter(new ColumnCountGetFilter(count)); 3589 res = region.get(g); 3590 assertEquals(count, res.size()); 3591 } 3592 3593 @Test 3594 public void testGet_Empty() throws IOException { 3595 byte[] row = Bytes.toBytes("row"); 3596 byte[] fam = Bytes.toBytes("fam"); 3597 3598 this.region = initHRegion(tableName, method, CONF, fam); 3599 Get get = new Get(row); 3600 get.addFamily(fam); 3601 Result r = region.get(get); 3602 3603 assertTrue(r.isEmpty()); 3604 } 3605 3606 @Test 3607 public void testGetWithFilter() throws IOException, InterruptedException { 3608 byte[] row1 = Bytes.toBytes("row1"); 3609 byte[] fam1 = Bytes.toBytes("fam1"); 3610 byte[] col1 = Bytes.toBytes("col1"); 3611 byte[] value1 = Bytes.toBytes("value1"); 3612 byte[] value2 = Bytes.toBytes("value2"); 3613 3614 final int maxVersions = 3; 3615 TableDescriptor tableDescriptor = 3616 TableDescriptorBuilder.newBuilder(TableName.valueOf("testFilterAndColumnTracker")) 3617 .setColumnFamily( 3618 ColumnFamilyDescriptorBuilder.newBuilder(fam1).setMaxVersions(maxVersions).build()) 3619 .build(); 3620 ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null, 3621 MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT); 3622 RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 3623 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + ".log"); 3624 final WAL wal = HBaseTestingUtil.createWal(TEST_UTIL.getConfiguration(), logDir, info); 3625 this.region = TEST_UTIL.createLocalHRegion(info, CONF, tableDescriptor, wal); 3626 3627 // Put 4 version to memstore 3628 long ts = 0; 3629 Put put = new Put(row1, ts); 3630 put.addColumn(fam1, col1, value1); 3631 region.put(put); 3632 put = new Put(row1, ts + 1); 3633 put.addColumn(fam1, col1, Bytes.toBytes("filter1")); 3634 region.put(put); 3635 put = new Put(row1, ts + 2); 3636 put.addColumn(fam1, col1, Bytes.toBytes("filter2")); 3637 region.put(put); 3638 put = new Put(row1, ts + 3); 3639 put.addColumn(fam1, col1, value2); 3640 region.put(put); 3641 3642 Get get = new Get(row1); 3643 get.readAllVersions(); 3644 Result res = region.get(get); 3645 // Get 3 versions, the oldest version has gone from user view 3646 assertEquals(maxVersions, res.size()); 3647 3648 get.setFilter(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("value"))); 3649 res = region.get(get); 3650 // When use value filter, the oldest version should still gone from user view and it 3651 // should only return one key vaule 3652 assertEquals(1, res.size()); 3653 assertTrue(CellUtil.matchingValue(new KeyValue(row1, fam1, col1, value2), res.rawCells()[0])); 3654 assertEquals(ts + 3, res.rawCells()[0].getTimestamp()); 3655 3656 region.flush(true); 3657 region.compact(true); 3658 Thread.sleep(1000); 3659 res = region.get(get); 3660 // After flush and compact, the result should be consistent with previous result 3661 assertEquals(1, res.size()); 3662 assertTrue(CellUtil.matchingValue(new KeyValue(row1, fam1, col1, value2), res.rawCells()[0])); 3663 } 3664 3665 // //////////////////////////////////////////////////////////////////////////// 3666 // Scanner tests 3667 // //////////////////////////////////////////////////////////////////////////// 3668 @Test 3669 public void testGetScanner_WithOkFamilies() throws IOException { 3670 byte[] fam1 = Bytes.toBytes("fam1"); 3671 byte[] fam2 = Bytes.toBytes("fam2"); 3672 3673 byte[][] families = { fam1, fam2 }; 3674 3675 // Setting up region 3676 this.region = initHRegion(tableName, method, CONF, families); 3677 Scan scan = new Scan(); 3678 scan.addFamily(fam1); 3679 scan.addFamily(fam2); 3680 region.getScanner(scan).close(); 3681 } 3682 3683 @Test 3684 public void testGetScanner_WithNotOkFamilies() throws IOException { 3685 byte[] fam1 = Bytes.toBytes("fam1"); 3686 byte[] fam2 = Bytes.toBytes("fam2"); 3687 3688 byte[][] families = { fam1 }; 3689 3690 // Setting up region 3691 this.region = initHRegion(tableName, method, CONF, families); 3692 Scan scan = new Scan(); 3693 scan.addFamily(fam2); 3694 assertThrows(NoSuchColumnFamilyException.class, () -> region.getScanner(scan)); 3695 } 3696 3697 @Test 3698 public void testGetScanner_WithNoFamilies() throws IOException { 3699 byte[] row1 = Bytes.toBytes("row1"); 3700 byte[] fam1 = Bytes.toBytes("fam1"); 3701 byte[] fam2 = Bytes.toBytes("fam2"); 3702 byte[] fam3 = Bytes.toBytes("fam3"); 3703 byte[] fam4 = Bytes.toBytes("fam4"); 3704 3705 byte[][] families = { fam1, fam2, fam3, fam4 }; 3706 3707 // Setting up region 3708 this.region = initHRegion(tableName, method, CONF, families); 3709 // Putting data in Region 3710 Put put = new Put(row1); 3711 put.addColumn(fam1, null, null); 3712 put.addColumn(fam2, null, null); 3713 put.addColumn(fam3, null, null); 3714 put.addColumn(fam4, null, null); 3715 region.put(put); 3716 3717 Scan scan = null; 3718 3719 // Testing to see how many scanners that is produced by getScanner, starting with known number, 3720 // 2 - current = 1 3721 scan = new Scan(); 3722 scan.addFamily(fam2); 3723 scan.addFamily(fam4); 3724 try (RegionScannerImpl is = region.getScanner(scan)) { 3725 assertEquals(1, is.storeHeap.getHeap().size()); 3726 } 3727 3728 scan = new Scan(); 3729 try (RegionScannerImpl is = region.getScanner(scan)) { 3730 assertEquals(families.length - 1, is.storeHeap.getHeap().size()); 3731 } 3732 } 3733 3734 /** 3735 * This method tests https://issues.apache.org/jira/browse/HBASE-2516. 3736 */ 3737 @Test 3738 public void testGetScanner_WithRegionClosed() throws IOException { 3739 byte[] fam1 = Bytes.toBytes("fam1"); 3740 byte[] fam2 = Bytes.toBytes("fam2"); 3741 3742 byte[][] families = { fam1, fam2 }; 3743 3744 // Setting up region 3745 region = initHRegion(tableName, method, CONF, families); 3746 region.closed.set(true); 3747 try { 3748 assertThrows(NotServingRegionException.class, () -> region.getScanner(null)); 3749 } finally { 3750 // so we can close the region in tearDown 3751 region.closed.set(false); 3752 } 3753 } 3754 3755 @Test 3756 public void testRegionScanner_Next() throws IOException { 3757 byte[] row1 = Bytes.toBytes("row1"); 3758 byte[] row2 = Bytes.toBytes("row2"); 3759 byte[] fam1 = Bytes.toBytes("fam1"); 3760 byte[] fam2 = Bytes.toBytes("fam2"); 3761 byte[] fam3 = Bytes.toBytes("fam3"); 3762 byte[] fam4 = Bytes.toBytes("fam4"); 3763 3764 byte[][] families = { fam1, fam2, fam3, fam4 }; 3765 long ts = EnvironmentEdgeManager.currentTime(); 3766 3767 // Setting up region 3768 this.region = initHRegion(tableName, method, CONF, families); 3769 // Putting data in Region 3770 Put put = null; 3771 put = new Put(row1); 3772 put.addColumn(fam1, (byte[]) null, ts, null); 3773 put.addColumn(fam2, (byte[]) null, ts, null); 3774 put.addColumn(fam3, (byte[]) null, ts, null); 3775 put.addColumn(fam4, (byte[]) null, ts, null); 3776 region.put(put); 3777 3778 put = new Put(row2); 3779 put.addColumn(fam1, (byte[]) null, ts, null); 3780 put.addColumn(fam2, (byte[]) null, ts, null); 3781 put.addColumn(fam3, (byte[]) null, ts, null); 3782 put.addColumn(fam4, (byte[]) null, ts, null); 3783 region.put(put); 3784 3785 Scan scan = new Scan(); 3786 scan.addFamily(fam2); 3787 scan.addFamily(fam4); 3788 try (InternalScanner is = region.getScanner(scan)) { 3789 List<ExtendedCell> res = null; 3790 3791 // Result 1 3792 List<ExtendedCell> expected1 = new ArrayList<>(); 3793 expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null)); 3794 expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null)); 3795 3796 res = new ArrayList<>(); 3797 is.next(res); 3798 for (int i = 0; i < res.size(); i++) { 3799 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected1.get(i), res.get(i))); 3800 } 3801 3802 // Result 2 3803 List<ExtendedCell> expected2 = new ArrayList<>(); 3804 expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null)); 3805 expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null)); 3806 3807 res = new ArrayList<>(); 3808 is.next(res); 3809 for (int i = 0; i < res.size(); i++) { 3810 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected2.get(i), res.get(i))); 3811 } 3812 } 3813 } 3814 3815 @Test 3816 public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException { 3817 byte[] row1 = Bytes.toBytes("row1"); 3818 byte[] qf1 = Bytes.toBytes("qualifier1"); 3819 byte[] qf2 = Bytes.toBytes("qualifier2"); 3820 byte[] fam1 = Bytes.toBytes("fam1"); 3821 byte[][] families = { fam1 }; 3822 3823 long ts1 = EnvironmentEdgeManager.currentTime(); 3824 long ts2 = ts1 + 1; 3825 long ts3 = ts1 + 2; 3826 3827 // Setting up region 3828 this.region = initHRegion(tableName, method, CONF, families); 3829 // Putting data in Region 3830 Put put = null; 3831 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 3832 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 3833 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 3834 3835 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 3836 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 3837 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 3838 3839 put = new Put(row1); 3840 put.add(kv13); 3841 put.add(kv12); 3842 put.add(kv11); 3843 put.add(kv23); 3844 put.add(kv22); 3845 put.add(kv21); 3846 region.put(put); 3847 3848 // Expected 3849 List<Cell> expected = new ArrayList<>(); 3850 expected.add(kv13); 3851 expected.add(kv12); 3852 3853 Scan scan = new Scan().withStartRow(row1); 3854 scan.addColumn(fam1, qf1); 3855 scan.readVersions(MAX_VERSIONS); 3856 List<Cell> actual = new ArrayList<>(); 3857 try (InternalScanner scanner = region.getScanner(scan)) { 3858 boolean hasNext = scanner.next(actual); 3859 assertEquals(false, hasNext); 3860 3861 // Verify result 3862 for (int i = 0; i < expected.size(); i++) { 3863 assertEquals(expected.get(i), actual.get(i)); 3864 } 3865 } 3866 } 3867 3868 @Test 3869 public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException { 3870 byte[] row1 = Bytes.toBytes("row1"); 3871 byte[] qf1 = Bytes.toBytes("qualifier1"); 3872 byte[] qf2 = Bytes.toBytes("qualifier2"); 3873 byte[] fam1 = Bytes.toBytes("fam1"); 3874 byte[][] families = { fam1 }; 3875 3876 long ts1 = 1; 3877 long ts2 = ts1 + 1; 3878 long ts3 = ts1 + 2; 3879 3880 // Setting up region 3881 this.region = initHRegion(tableName, method, CONF, families); 3882 // Putting data in Region 3883 Put put = null; 3884 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 3885 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 3886 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 3887 3888 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 3889 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 3890 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 3891 3892 put = new Put(row1); 3893 put.add(kv13); 3894 put.add(kv12); 3895 put.add(kv11); 3896 put.add(kv23); 3897 put.add(kv22); 3898 put.add(kv21); 3899 region.put(put); 3900 region.flush(true); 3901 3902 // Expected 3903 List<ExtendedCell> expected = new ArrayList<>(); 3904 expected.add(kv13); 3905 expected.add(kv12); 3906 expected.add(kv23); 3907 expected.add(kv22); 3908 3909 Scan scan = new Scan().withStartRow(row1); 3910 scan.addColumn(fam1, qf1); 3911 scan.addColumn(fam1, qf2); 3912 scan.readVersions(MAX_VERSIONS); 3913 List<ExtendedCell> actual = new ArrayList<>(); 3914 try (InternalScanner scanner = region.getScanner(scan)) { 3915 boolean hasNext = scanner.next(actual); 3916 assertEquals(false, hasNext); 3917 3918 // Verify result 3919 for (int i = 0; i < expected.size(); i++) { 3920 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); 3921 } 3922 } 3923 } 3924 3925 @Test 3926 public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() 3927 throws IOException { 3928 byte[] row1 = Bytes.toBytes("row1"); 3929 byte[] fam1 = Bytes.toBytes("fam1"); 3930 byte[][] families = { fam1 }; 3931 byte[] qf1 = Bytes.toBytes("qualifier1"); 3932 byte[] qf2 = Bytes.toBytes("qualifier2"); 3933 3934 long ts1 = 1; 3935 long ts2 = ts1 + 1; 3936 long ts3 = ts1 + 2; 3937 long ts4 = ts1 + 3; 3938 3939 // Setting up region 3940 this.region = initHRegion(tableName, method, CONF, families); 3941 // Putting data in Region 3942 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null); 3943 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 3944 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 3945 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 3946 3947 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null); 3948 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 3949 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 3950 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 3951 3952 Put put = null; 3953 put = new Put(row1); 3954 put.add(kv14); 3955 put.add(kv24); 3956 region.put(put); 3957 region.flush(true); 3958 3959 put = new Put(row1); 3960 put.add(kv23); 3961 put.add(kv13); 3962 region.put(put); 3963 region.flush(true); 3964 3965 put = new Put(row1); 3966 put.add(kv22); 3967 put.add(kv12); 3968 region.put(put); 3969 region.flush(true); 3970 3971 put = new Put(row1); 3972 put.add(kv21); 3973 put.add(kv11); 3974 region.put(put); 3975 3976 // Expected 3977 List<ExtendedCell> expected = new ArrayList<>(); 3978 expected.add(kv14); 3979 expected.add(kv13); 3980 expected.add(kv12); 3981 expected.add(kv24); 3982 expected.add(kv23); 3983 expected.add(kv22); 3984 3985 Scan scan = new Scan().withStartRow(row1); 3986 scan.addColumn(fam1, qf1); 3987 scan.addColumn(fam1, qf2); 3988 int versions = 3; 3989 scan.readVersions(versions); 3990 List<ExtendedCell> actual = new ArrayList<>(); 3991 try (InternalScanner scanner = region.getScanner(scan)) { 3992 boolean hasNext = scanner.next(actual); 3993 assertEquals(false, hasNext); 3994 3995 // Verify result 3996 for (int i = 0; i < expected.size(); i++) { 3997 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); 3998 } 3999 } 4000 } 4001 4002 @Test 4003 public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException { 4004 byte[] row1 = Bytes.toBytes("row1"); 4005 byte[] qf1 = Bytes.toBytes("qualifier1"); 4006 byte[] qf2 = Bytes.toBytes("qualifier2"); 4007 byte[] fam1 = Bytes.toBytes("fam1"); 4008 byte[][] families = { fam1 }; 4009 4010 long ts1 = EnvironmentEdgeManager.currentTime(); 4011 long ts2 = ts1 + 1; 4012 long ts3 = ts1 + 2; 4013 4014 // Setting up region 4015 this.region = initHRegion(tableName, method, CONF, families); 4016 // Putting data in Region 4017 Put put = null; 4018 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 4019 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 4020 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 4021 4022 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 4023 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 4024 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 4025 4026 put = new Put(row1); 4027 put.add(kv13); 4028 put.add(kv12); 4029 put.add(kv11); 4030 put.add(kv23); 4031 put.add(kv22); 4032 put.add(kv21); 4033 region.put(put); 4034 4035 // Expected 4036 List<Cell> expected = new ArrayList<>(); 4037 expected.add(kv13); 4038 expected.add(kv12); 4039 expected.add(kv23); 4040 expected.add(kv22); 4041 4042 Scan scan = new Scan().withStartRow(row1); 4043 scan.addFamily(fam1); 4044 scan.readVersions(MAX_VERSIONS); 4045 List<Cell> actual = new ArrayList<>(); 4046 try (InternalScanner scanner = region.getScanner(scan)) { 4047 boolean hasNext = scanner.next(actual); 4048 assertEquals(false, hasNext); 4049 4050 // Verify result 4051 for (int i = 0; i < expected.size(); i++) { 4052 assertEquals(expected.get(i), actual.get(i)); 4053 } 4054 } 4055 } 4056 4057 @Test 4058 public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException { 4059 byte[] row1 = Bytes.toBytes("row1"); 4060 byte[] qf1 = Bytes.toBytes("qualifier1"); 4061 byte[] qf2 = Bytes.toBytes("qualifier2"); 4062 byte[] fam1 = Bytes.toBytes("fam1"); 4063 4064 long ts1 = 1; 4065 long ts2 = ts1 + 1; 4066 long ts3 = ts1 + 2; 4067 4068 // Setting up region 4069 this.region = initHRegion(tableName, method, CONF, fam1); 4070 // Putting data in Region 4071 Put put = null; 4072 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 4073 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 4074 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 4075 4076 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 4077 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 4078 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 4079 4080 put = new Put(row1); 4081 put.add(kv13); 4082 put.add(kv12); 4083 put.add(kv11); 4084 put.add(kv23); 4085 put.add(kv22); 4086 put.add(kv21); 4087 region.put(put); 4088 region.flush(true); 4089 4090 // Expected 4091 List<ExtendedCell> expected = new ArrayList<>(); 4092 expected.add(kv13); 4093 expected.add(kv12); 4094 expected.add(kv23); 4095 expected.add(kv22); 4096 4097 Scan scan = new Scan().withStartRow(row1); 4098 scan.addFamily(fam1); 4099 scan.readVersions(MAX_VERSIONS); 4100 List<ExtendedCell> actual = new ArrayList<>(); 4101 try (InternalScanner scanner = region.getScanner(scan)) { 4102 boolean hasNext = scanner.next(actual); 4103 assertEquals(false, hasNext); 4104 4105 // Verify result 4106 for (int i = 0; i < expected.size(); i++) { 4107 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); 4108 } 4109 } 4110 } 4111 4112 @Test 4113 public void testScanner_StopRow1542() throws IOException { 4114 byte[] family = Bytes.toBytes("testFamily"); 4115 this.region = initHRegion(tableName, method, CONF, family); 4116 byte[] row1 = Bytes.toBytes("row111"); 4117 byte[] row2 = Bytes.toBytes("row222"); 4118 byte[] row3 = Bytes.toBytes("row333"); 4119 byte[] row4 = Bytes.toBytes("row444"); 4120 byte[] row5 = Bytes.toBytes("row555"); 4121 4122 byte[] col1 = Bytes.toBytes("Pub111"); 4123 byte[] col2 = Bytes.toBytes("Pub222"); 4124 4125 Put put = new Put(row1); 4126 put.addColumn(family, col1, Bytes.toBytes(10L)); 4127 region.put(put); 4128 4129 put = new Put(row2); 4130 put.addColumn(family, col1, Bytes.toBytes(15L)); 4131 region.put(put); 4132 4133 put = new Put(row3); 4134 put.addColumn(family, col2, Bytes.toBytes(20L)); 4135 region.put(put); 4136 4137 put = new Put(row4); 4138 put.addColumn(family, col2, Bytes.toBytes(30L)); 4139 region.put(put); 4140 4141 put = new Put(row5); 4142 put.addColumn(family, col1, Bytes.toBytes(40L)); 4143 region.put(put); 4144 4145 Scan scan = new Scan().withStartRow(row3).withStopRow(row4); 4146 scan.readAllVersions(); 4147 scan.addColumn(family, col1); 4148 try (InternalScanner s = region.getScanner(scan)) { 4149 List<Cell> results = new ArrayList<>(); 4150 assertEquals(false, s.next(results)); 4151 assertEquals(0, results.size()); 4152 } 4153 } 4154 4155 @Test 4156 public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException { 4157 byte[] row1 = Bytes.toBytes("row1"); 4158 byte[] fam1 = Bytes.toBytes("fam1"); 4159 byte[] qf1 = Bytes.toBytes("qualifier1"); 4160 byte[] qf2 = Bytes.toBytes("quateslifier2"); 4161 4162 long ts1 = 1; 4163 long ts2 = ts1 + 1; 4164 long ts3 = ts1 + 2; 4165 long ts4 = ts1 + 3; 4166 4167 // Setting up region 4168 this.region = initHRegion(tableName, method, CONF, fam1); 4169 // Putting data in Region 4170 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null); 4171 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 4172 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 4173 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 4174 4175 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null); 4176 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 4177 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 4178 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 4179 4180 Put put = null; 4181 put = new Put(row1); 4182 put.add(kv14); 4183 put.add(kv24); 4184 region.put(put); 4185 region.flush(true); 4186 4187 put = new Put(row1); 4188 put.add(kv23); 4189 put.add(kv13); 4190 region.put(put); 4191 region.flush(true); 4192 4193 put = new Put(row1); 4194 put.add(kv22); 4195 put.add(kv12); 4196 region.put(put); 4197 region.flush(true); 4198 4199 put = new Put(row1); 4200 put.add(kv21); 4201 put.add(kv11); 4202 region.put(put); 4203 4204 // Expected 4205 List<KeyValue> expected = new ArrayList<>(); 4206 expected.add(kv14); 4207 expected.add(kv13); 4208 expected.add(kv12); 4209 expected.add(kv24); 4210 expected.add(kv23); 4211 expected.add(kv22); 4212 4213 Scan scan = new Scan().withStartRow(row1); 4214 int versions = 3; 4215 scan.readVersions(versions); 4216 List<ExtendedCell> actual = new ArrayList<>(); 4217 try (InternalScanner scanner = region.getScanner(scan)) { 4218 boolean hasNext = scanner.next(actual); 4219 assertEquals(false, hasNext); 4220 4221 // Verify result 4222 for (int i = 0; i < expected.size(); i++) { 4223 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); 4224 } 4225 } 4226 } 4227 4228 /** 4229 * Added for HBASE-5416 Here we test scan optimization when only subset of CFs are used in filter 4230 * conditions. 4231 */ 4232 @Test 4233 public void testScanner_JoinedScanners() throws IOException { 4234 byte[] cf_essential = Bytes.toBytes("essential"); 4235 byte[] cf_joined = Bytes.toBytes("joined"); 4236 byte[] cf_alpha = Bytes.toBytes("alpha"); 4237 this.region = initHRegion(tableName, method, CONF, cf_essential, cf_joined, cf_alpha); 4238 byte[] row1 = Bytes.toBytes("row1"); 4239 byte[] row2 = Bytes.toBytes("row2"); 4240 byte[] row3 = Bytes.toBytes("row3"); 4241 4242 byte[] col_normal = Bytes.toBytes("d"); 4243 byte[] col_alpha = Bytes.toBytes("a"); 4244 4245 byte[] filtered_val = Bytes.toBytes(3); 4246 4247 Put put = new Put(row1); 4248 put.addColumn(cf_essential, col_normal, Bytes.toBytes(1)); 4249 put.addColumn(cf_joined, col_alpha, Bytes.toBytes(1)); 4250 region.put(put); 4251 4252 put = new Put(row2); 4253 put.addColumn(cf_essential, col_alpha, Bytes.toBytes(2)); 4254 put.addColumn(cf_joined, col_normal, Bytes.toBytes(2)); 4255 put.addColumn(cf_alpha, col_alpha, Bytes.toBytes(2)); 4256 region.put(put); 4257 4258 put = new Put(row3); 4259 put.addColumn(cf_essential, col_normal, filtered_val); 4260 put.addColumn(cf_joined, col_normal, filtered_val); 4261 region.put(put); 4262 4263 // Check two things: 4264 // 1. result list contains expected values 4265 // 2. result list is sorted properly 4266 4267 Scan scan = new Scan(); 4268 Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal, 4269 CompareOperator.NOT_EQUAL, filtered_val); 4270 scan.setFilter(filter); 4271 scan.setLoadColumnFamiliesOnDemand(true); 4272 try (InternalScanner s = region.getScanner(scan)) { 4273 List<Cell> results = new ArrayList<>(); 4274 assertTrue(s.next(results)); 4275 assertEquals(1, results.size()); 4276 results.clear(); 4277 4278 assertTrue(s.next(results)); 4279 assertEquals(3, results.size()); 4280 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha)); 4281 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential)); 4282 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined)); 4283 results.clear(); 4284 4285 assertFalse(s.next(results)); 4286 assertEquals(0, results.size()); 4287 } 4288 } 4289 4290 /** 4291 * HBASE-5416 Test case when scan limits amount of KVs returned on each next() call. 4292 */ 4293 @Test 4294 public void testScanner_JoinedScannersWithLimits() throws IOException { 4295 final byte[] cf_first = Bytes.toBytes("first"); 4296 final byte[] cf_second = Bytes.toBytes("second"); 4297 4298 this.region = initHRegion(tableName, method, CONF, cf_first, cf_second); 4299 final byte[] col_a = Bytes.toBytes("a"); 4300 final byte[] col_b = Bytes.toBytes("b"); 4301 4302 Put put; 4303 4304 for (int i = 0; i < 10; i++) { 4305 put = new Put(Bytes.toBytes("r" + Integer.toString(i))); 4306 put.addColumn(cf_first, col_a, Bytes.toBytes(i)); 4307 if (i < 5) { 4308 put.addColumn(cf_first, col_b, Bytes.toBytes(i)); 4309 put.addColumn(cf_second, col_a, Bytes.toBytes(i)); 4310 put.addColumn(cf_second, col_b, Bytes.toBytes(i)); 4311 } 4312 region.put(put); 4313 } 4314 4315 Scan scan = new Scan(); 4316 scan.setLoadColumnFamiliesOnDemand(true); 4317 Filter bogusFilter = new FilterBase() { 4318 @Override 4319 public ReturnCode filterCell(final Cell ignored) throws IOException { 4320 return ReturnCode.INCLUDE; 4321 } 4322 4323 @Override 4324 public boolean isFamilyEssential(byte[] name) { 4325 return Bytes.equals(name, cf_first); 4326 } 4327 }; 4328 4329 scan.setFilter(bogusFilter); 4330 try (InternalScanner s = region.getScanner(scan)) { 4331 // Our data looks like this: 4332 // r0: first:a, first:b, second:a, second:b 4333 // r1: first:a, first:b, second:a, second:b 4334 // r2: first:a, first:b, second:a, second:b 4335 // r3: first:a, first:b, second:a, second:b 4336 // r4: first:a, first:b, second:a, second:b 4337 // r5: first:a 4338 // r6: first:a 4339 // r7: first:a 4340 // r8: first:a 4341 // r9: first:a 4342 4343 // But due to next's limit set to 3, we should get this: 4344 // r0: first:a, first:b, second:a 4345 // r0: second:b 4346 // r1: first:a, first:b, second:a 4347 // r1: second:b 4348 // r2: first:a, first:b, second:a 4349 // r2: second:b 4350 // r3: first:a, first:b, second:a 4351 // r3: second:b 4352 // r4: first:a, first:b, second:a 4353 // r4: second:b 4354 // r5: first:a 4355 // r6: first:a 4356 // r7: first:a 4357 // r8: first:a 4358 // r9: first:a 4359 4360 List<Cell> results = new ArrayList<>(); 4361 int index = 0; 4362 ScannerContext scannerContext = ScannerContext.newBuilder().setBatchLimit(3).build(); 4363 while (true) { 4364 boolean more = s.next(results, scannerContext); 4365 if ((index >> 1) < 5) { 4366 if (index % 2 == 0) { 4367 assertEquals(3, results.size()); 4368 } else { 4369 assertEquals(1, results.size()); 4370 } 4371 } else { 4372 assertEquals(1, results.size()); 4373 } 4374 results.clear(); 4375 index++; 4376 if (!more) { 4377 break; 4378 } 4379 } 4380 } 4381 } 4382 4383 @Test 4384 public void testScannerOperationId() throws IOException { 4385 region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES); 4386 Scan scan = new Scan(); 4387 try (RegionScanner scanner = region.getScanner(scan)) { 4388 assertNull(scanner.getOperationId()); 4389 } 4390 4391 String operationId = "test_operation_id_0101"; 4392 scan = new Scan().setId(operationId); 4393 try (RegionScanner scanner = region.getScanner(scan)) { 4394 assertEquals(operationId, scanner.getOperationId()); 4395 } 4396 4397 HBaseTestingUtil.closeRegionAndWAL(this.region); 4398 } 4399 4400 /** 4401 * Write an HFile block full with Cells whose qualifier that are identical between 0 and 4402 * Short.MAX_VALUE. See HBASE-13329. 4403 */ 4404 @Test 4405 public void testLongQualifier() throws Exception { 4406 byte[] family = Bytes.toBytes("family"); 4407 this.region = initHRegion(tableName, method, CONF, family); 4408 byte[] q = new byte[Short.MAX_VALUE + 2]; 4409 Arrays.fill(q, 0, q.length - 1, (byte) 42); 4410 for (byte i = 0; i < 10; i++) { 4411 Put p = new Put(Bytes.toBytes("row")); 4412 // qualifiers that differ past Short.MAX_VALUE 4413 q[q.length - 1] = i; 4414 p.addColumn(family, q, q); 4415 region.put(p); 4416 } 4417 region.flush(false); 4418 } 4419 4420 /** 4421 * Flushes the cache in a thread while scanning. The tests verify that the scan is coherent - e.g. 4422 * the returned results are always of the same or later update as the previous results. scan / 4423 * compact thread join 4424 */ 4425 @Test 4426 public void testFlushCacheWhileScanning() throws IOException, InterruptedException { 4427 byte[] family = Bytes.toBytes("family"); 4428 int numRows = 1000; 4429 int flushAndScanInterval = 10; 4430 int compactInterval = 10 * flushAndScanInterval; 4431 4432 this.region = initHRegion(tableName, method, CONF, family); 4433 FlushThread flushThread = new FlushThread(); 4434 try { 4435 flushThread.start(); 4436 4437 Scan scan = new Scan(); 4438 scan.addFamily(family); 4439 scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOperator.EQUAL, 4440 new BinaryComparator(Bytes.toBytes(5L)))); 4441 4442 int expectedCount = 0; 4443 List<Cell> res = new ArrayList<>(); 4444 4445 boolean toggle = true; 4446 for (long i = 0; i < numRows; i++) { 4447 Put put = new Put(Bytes.toBytes(i)); 4448 put.setDurability(Durability.SKIP_WAL); 4449 put.addColumn(family, qual1, Bytes.toBytes(i % 10)); 4450 region.put(put); 4451 4452 if (i != 0 && i % compactInterval == 0) { 4453 LOG.debug("iteration = " + i + " ts=" + EnvironmentEdgeManager.currentTime()); 4454 region.compact(true); 4455 } 4456 4457 if (i % 10 == 5L) { 4458 expectedCount++; 4459 } 4460 4461 if (i != 0 && i % flushAndScanInterval == 0) { 4462 res.clear(); 4463 try (InternalScanner scanner = region.getScanner(scan)) { 4464 if (toggle) { 4465 flushThread.flush(); 4466 } 4467 while (scanner.next(res)) { 4468 // ignore 4469 } 4470 } 4471 if (!toggle) { 4472 flushThread.flush(); 4473 } 4474 assertEquals( 4475 "toggle=" + toggle + "i=" + i + " ts=" + EnvironmentEdgeManager.currentTime(), 4476 expectedCount, res.size()); 4477 toggle = !toggle; 4478 } 4479 } 4480 4481 } finally { 4482 try { 4483 flushThread.done(); 4484 flushThread.join(); 4485 flushThread.checkNoError(); 4486 } catch (InterruptedException ie) { 4487 LOG.warn("Caught exception when joining with flushThread", ie); 4488 } 4489 HBaseTestingUtil.closeRegionAndWAL(this.region); 4490 this.region = null; 4491 } 4492 } 4493 4494 protected class FlushThread extends Thread { 4495 private volatile boolean done; 4496 private Throwable error = null; 4497 4498 FlushThread() { 4499 super("FlushThread"); 4500 } 4501 4502 public void done() { 4503 done = true; 4504 synchronized (this) { 4505 interrupt(); 4506 } 4507 } 4508 4509 public void checkNoError() { 4510 if (error != null) { 4511 assertNull(error); 4512 } 4513 } 4514 4515 @Override 4516 public void run() { 4517 done = false; 4518 while (!done) { 4519 synchronized (this) { 4520 try { 4521 wait(); 4522 } catch (InterruptedException ignored) { 4523 if (done) { 4524 break; 4525 } 4526 } 4527 } 4528 try { 4529 region.flush(true); 4530 } catch (IOException e) { 4531 if (!done) { 4532 LOG.error("Error while flushing cache", e); 4533 error = e; 4534 } 4535 break; 4536 } catch (Throwable t) { 4537 LOG.error("Uncaught exception", t); 4538 throw t; 4539 } 4540 } 4541 } 4542 4543 public void flush() { 4544 synchronized (this) { 4545 notify(); 4546 } 4547 } 4548 } 4549 4550 /** 4551 * So can be overridden in subclasses. 4552 */ 4553 protected int getNumQualifiersForTestWritesWhileScanning() { 4554 return 100; 4555 } 4556 4557 /** 4558 * So can be overridden in subclasses. 4559 */ 4560 protected int getTestCountForTestWritesWhileScanning() { 4561 return 100; 4562 } 4563 4564 /** 4565 * Writes very wide records and scans for the latest every time.. Flushes and compacts the region 4566 * every now and then to keep things realistic. by flush / scan / compaction when joining threads 4567 */ 4568 @Test 4569 public void testWritesWhileScanning() throws IOException, InterruptedException { 4570 int testCount = getTestCountForTestWritesWhileScanning(); 4571 int numRows = 1; 4572 int numFamilies = 10; 4573 int numQualifiers = getNumQualifiersForTestWritesWhileScanning(); 4574 int flushInterval = 7; 4575 int compactInterval = 5 * flushInterval; 4576 byte[][] families = new byte[numFamilies][]; 4577 for (int i = 0; i < numFamilies; i++) { 4578 families[i] = Bytes.toBytes("family" + i); 4579 } 4580 byte[][] qualifiers = new byte[numQualifiers][]; 4581 for (int i = 0; i < numQualifiers; i++) { 4582 qualifiers[i] = Bytes.toBytes("qual" + i); 4583 } 4584 4585 this.region = initHRegion(tableName, method, CONF, families); 4586 FlushThread flushThread = new FlushThread(); 4587 PutThread putThread = new PutThread(numRows, families, qualifiers); 4588 try { 4589 putThread.start(); 4590 putThread.waitForFirstPut(); 4591 4592 flushThread.start(); 4593 4594 Scan scan = new Scan().withStartRow(Bytes.toBytes("row0")).withStopRow(Bytes.toBytes("row1")); 4595 4596 int expectedCount = numFamilies * numQualifiers; 4597 List<Cell> res = new ArrayList<>(); 4598 4599 long prevTimestamp = 0L; 4600 for (int i = 0; i < testCount; i++) { 4601 4602 if (i != 0 && i % compactInterval == 0) { 4603 region.compact(true); 4604 for (HStore store : region.getStores()) { 4605 store.closeAndArchiveCompactedFiles(); 4606 } 4607 } 4608 4609 if (i != 0 && i % flushInterval == 0) { 4610 flushThread.flush(); 4611 } 4612 4613 boolean previousEmpty = res.isEmpty(); 4614 res.clear(); 4615 try (InternalScanner scanner = region.getScanner(scan)) { 4616 boolean moreRows; 4617 do { 4618 moreRows = scanner.next(res); 4619 } while (moreRows); 4620 } 4621 if (!res.isEmpty() || !previousEmpty || i > compactInterval) { 4622 assertEquals("i=" + i, expectedCount, res.size()); 4623 long timestamp = res.get(0).getTimestamp(); 4624 assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp, 4625 timestamp >= prevTimestamp); 4626 prevTimestamp = timestamp; 4627 } 4628 } 4629 4630 putThread.done(); 4631 4632 region.flush(true); 4633 4634 } finally { 4635 try { 4636 flushThread.done(); 4637 flushThread.join(); 4638 flushThread.checkNoError(); 4639 4640 putThread.join(); 4641 putThread.checkNoError(); 4642 } catch (InterruptedException ie) { 4643 LOG.warn("Caught exception when joining with flushThread", ie); 4644 } 4645 4646 try { 4647 HBaseTestingUtil.closeRegionAndWAL(this.region); 4648 } catch (DroppedSnapshotException dse) { 4649 // We could get this on way out because we interrupt the background flusher and it could 4650 // fail anywhere causing a DSE over in the background flusher... only it is not properly 4651 // dealt with so could still be memory hanging out when we get to here -- memory we can't 4652 // flush because the accounting is 'off' since original DSE. 4653 } 4654 this.region = null; 4655 } 4656 } 4657 4658 @Test 4659 public void testCloseAndArchiveCompactedFiles() throws IOException { 4660 byte[] CF1 = Bytes.toBytes("CF1"); 4661 byte[] CF2 = Bytes.toBytes("CF2"); 4662 this.region = initHRegion(tableName, method, CONF, CF1, CF2); 4663 for (int i = 0; i < 2; i++) { 4664 int index = i; 4665 Put put = 4666 new Put(Bytes.toBytes(index)).addColumn(CF1, Bytes.toBytes("q"), Bytes.toBytes(index)); 4667 region.put(put); 4668 region.flush(true); 4669 } 4670 4671 region.compact(true); 4672 4673 HStore store1 = region.getStore(CF1); 4674 HStore store2 = region.getStore(CF2); 4675 store1.closeAndArchiveCompactedFiles(); 4676 store2.closeAndArchiveCompactedFiles(); 4677 4678 int storefilesCount = region.getStores().stream().mapToInt(Store::getStorefilesCount).sum(); 4679 assertTrue(storefilesCount == 1); 4680 4681 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 4682 Configuration conf = region.getReadOnlyConfiguration(); 4683 RegionInfo regionInfo = region.getRegionInfo(); 4684 Path store1ArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, CF1); 4685 assertTrue(fs.exists(store1ArchiveDir)); 4686 // The archived dir of CF2 does not exist because this column family has no data at all 4687 Path store2ArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, CF2); 4688 assertFalse(fs.exists(store2ArchiveDir)); 4689 } 4690 4691 protected class PutThread extends Thread { 4692 private volatile boolean done; 4693 private volatile int numPutsFinished = 0; 4694 4695 private Throwable error = null; 4696 private int numRows; 4697 private byte[][] families; 4698 private byte[][] qualifiers; 4699 4700 private PutThread(int numRows, byte[][] families, byte[][] qualifiers) { 4701 super("PutThread"); 4702 this.numRows = numRows; 4703 this.families = families; 4704 this.qualifiers = qualifiers; 4705 } 4706 4707 /** 4708 * Block calling thread until this instance of PutThread has put at least one row. 4709 */ 4710 public void waitForFirstPut() throws InterruptedException { 4711 // wait until put thread actually puts some data 4712 while (isAlive() && numPutsFinished == 0) { 4713 checkNoError(); 4714 Thread.sleep(50); 4715 } 4716 } 4717 4718 public void done() { 4719 done = true; 4720 synchronized (this) { 4721 interrupt(); 4722 } 4723 } 4724 4725 public void checkNoError() { 4726 if (error != null) { 4727 assertNull(error); 4728 } 4729 } 4730 4731 @Override 4732 public void run() { 4733 done = false; 4734 while (!done) { 4735 try { 4736 for (int r = 0; r < numRows; r++) { 4737 byte[] row = Bytes.toBytes("row" + r); 4738 Put put = new Put(row); 4739 put.setDurability(Durability.SKIP_WAL); 4740 byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished)); 4741 for (byte[] family : families) { 4742 for (byte[] qualifier : qualifiers) { 4743 put.addColumn(family, qualifier, numPutsFinished, value); 4744 } 4745 } 4746 region.put(put); 4747 numPutsFinished++; 4748 if (numPutsFinished > 0 && numPutsFinished % 47 == 0) { 4749 LOG.debug("put iteration = {}", numPutsFinished); 4750 Delete delete = new Delete(row, (long) numPutsFinished - 30); 4751 region.delete(delete); 4752 } 4753 numPutsFinished++; 4754 } 4755 } catch (InterruptedIOException e) { 4756 // This is fine. It means we are done, or didn't get the lock on time 4757 LOG.info("Interrupted", e); 4758 } catch (IOException e) { 4759 LOG.error("Error while putting records", e); 4760 error = e; 4761 break; 4762 } 4763 } 4764 4765 } 4766 4767 } 4768 4769 /** 4770 * Writes very wide records and gets the latest row every time.. Flushes and compacts the region 4771 * aggressivly to catch issues. by flush / scan / compaction when joining threads 4772 */ 4773 @Test 4774 public void testWritesWhileGetting() throws Exception { 4775 int testCount = 50; 4776 int numRows = 1; 4777 int numFamilies = 10; 4778 int numQualifiers = 100; 4779 int compactInterval = 100; 4780 byte[][] families = new byte[numFamilies][]; 4781 for (int i = 0; i < numFamilies; i++) { 4782 families[i] = Bytes.toBytes("family" + i); 4783 } 4784 byte[][] qualifiers = new byte[numQualifiers][]; 4785 for (int i = 0; i < numQualifiers; i++) { 4786 qualifiers[i] = Bytes.toBytes("qual" + i); 4787 } 4788 4789 // This test flushes constantly and can cause many files to be created, 4790 // possibly 4791 // extending over the ulimit. Make sure compactions are aggressive in 4792 // reducing 4793 // the number of HFiles created. 4794 Configuration conf = HBaseConfiguration.create(CONF); 4795 conf.setInt("hbase.hstore.compaction.min", 1); 4796 conf.setInt("hbase.hstore.compaction.max", 1000); 4797 this.region = initHRegion(tableName, method, conf, families); 4798 PutThread putThread = null; 4799 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf); 4800 try { 4801 putThread = new PutThread(numRows, families, qualifiers); 4802 putThread.start(); 4803 putThread.waitForFirstPut(); 4804 4805 // Add a thread that flushes as fast as possible 4806 ctx.addThread(new RepeatingTestThread(ctx) { 4807 4808 @Override 4809 public void doAnAction() throws Exception { 4810 region.flush(true); 4811 // Compact regularly to avoid creating too many files and exceeding 4812 // the ulimit. 4813 region.compact(false); 4814 for (HStore store : region.getStores()) { 4815 store.closeAndArchiveCompactedFiles(); 4816 } 4817 } 4818 }); 4819 ctx.startThreads(); 4820 4821 Get get = new Get(Bytes.toBytes("row0")); 4822 Result result = null; 4823 4824 int expectedCount = numFamilies * numQualifiers; 4825 4826 long prevTimestamp = 0L; 4827 for (int i = 0; i < testCount; i++) { 4828 LOG.info("testWritesWhileGetting verify turn " + i); 4829 boolean previousEmpty = result == null || result.isEmpty(); 4830 result = region.get(get); 4831 if (!result.isEmpty() || !previousEmpty || i > compactInterval) { 4832 assertEquals("i=" + i, expectedCount, result.size()); 4833 // TODO this was removed, now what dangit?! 4834 // search looking for the qualifier in question? 4835 long timestamp = 0; 4836 for (Cell kv : result.rawCells()) { 4837 if ( 4838 CellUtil.matchingFamily(kv, families[0]) 4839 && CellUtil.matchingQualifier(kv, qualifiers[0]) 4840 ) { 4841 timestamp = kv.getTimestamp(); 4842 } 4843 } 4844 assertTrue(timestamp >= prevTimestamp); 4845 prevTimestamp = timestamp; 4846 ExtendedCell previousKV = null; 4847 4848 for (ExtendedCell kv : ClientInternalHelper.getExtendedRawCells(result)) { 4849 byte[] thisValue = CellUtil.cloneValue(kv); 4850 if (previousKV != null) { 4851 if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) { 4852 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV 4853 + "(memStoreTS:" + previousKV.getSequenceId() + ")" + ", New KV: " + kv 4854 + "(memStoreTS:" + kv.getSequenceId() + ")"); 4855 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue)); 4856 } 4857 } 4858 previousKV = kv; 4859 } 4860 } 4861 } 4862 } finally { 4863 if (putThread != null) putThread.done(); 4864 4865 region.flush(true); 4866 4867 if (putThread != null) { 4868 putThread.join(); 4869 putThread.checkNoError(); 4870 } 4871 4872 ctx.stop(); 4873 HBaseTestingUtil.closeRegionAndWAL(this.region); 4874 this.region = null; 4875 } 4876 } 4877 4878 @Test 4879 public void testHolesInMeta() throws Exception { 4880 byte[] family = Bytes.toBytes("family"); 4881 this.region = 4882 initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF, false, family); 4883 byte[] rowNotServed = Bytes.toBytes("a"); 4884 Get g = new Get(rowNotServed); 4885 try { 4886 region.get(g); 4887 fail(); 4888 } catch (WrongRegionException x) { 4889 // OK 4890 } 4891 byte[] row = Bytes.toBytes("y"); 4892 g = new Get(row); 4893 region.get(g); 4894 } 4895 4896 @Test 4897 public void testIndexesScanWithOneDeletedRow() throws IOException { 4898 byte[] family = Bytes.toBytes("family"); 4899 4900 // Setting up region 4901 this.region = initHRegion(tableName, method, CONF, family); 4902 Put put = new Put(Bytes.toBytes(1L)); 4903 put.addColumn(family, qual1, 1L, Bytes.toBytes(1L)); 4904 region.put(put); 4905 4906 region.flush(true); 4907 4908 Delete delete = new Delete(Bytes.toBytes(1L), 1L); 4909 region.delete(delete); 4910 4911 put = new Put(Bytes.toBytes(2L)); 4912 put.addColumn(family, qual1, 2L, Bytes.toBytes(2L)); 4913 region.put(put); 4914 4915 Scan idxScan = new Scan(); 4916 idxScan.addFamily(family); 4917 idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, 4918 Arrays.<Filter> asList( 4919 new SingleColumnValueFilter(family, qual1, CompareOperator.GREATER_OR_EQUAL, 4920 new BinaryComparator(Bytes.toBytes(0L))), 4921 new SingleColumnValueFilter(family, qual1, CompareOperator.LESS_OR_EQUAL, 4922 new BinaryComparator(Bytes.toBytes(3L)))))); 4923 try (InternalScanner scanner = region.getScanner(idxScan)) { 4924 List<Cell> res = new ArrayList<>(); 4925 4926 while (scanner.next(res)) { 4927 // Ignore res value. 4928 } 4929 assertEquals(1L, res.size()); 4930 } 4931 } 4932 4933 // //////////////////////////////////////////////////////////////////////////// 4934 // Bloom filter test 4935 // //////////////////////////////////////////////////////////////////////////// 4936 @Test 4937 public void testBloomFilterSize() throws IOException { 4938 byte[] fam1 = Bytes.toBytes("fam1"); 4939 byte[] qf1 = Bytes.toBytes("col"); 4940 byte[] val1 = Bytes.toBytes("value1"); 4941 // Create Table 4942 TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName) 4943 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam1) 4944 .setMaxVersions(Integer.MAX_VALUE).setBloomFilterType(BloomType.ROWCOL).build()) 4945 .build(); 4946 RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 4947 this.region = TEST_UTIL.createLocalHRegion(info, tableDescriptor); 4948 int num_unique_rows = 10; 4949 int duplicate_multiplier = 2; 4950 int num_storefiles = 4; 4951 4952 int version = 0; 4953 for (int f = 0; f < num_storefiles; f++) { 4954 for (int i = 0; i < duplicate_multiplier; i++) { 4955 for (int j = 0; j < num_unique_rows; j++) { 4956 Put put = new Put(Bytes.toBytes("row" + j)); 4957 put.setDurability(Durability.SKIP_WAL); 4958 long ts = version++; 4959 put.addColumn(fam1, qf1, ts, val1); 4960 region.put(put); 4961 } 4962 } 4963 region.flush(true); 4964 } 4965 // before compaction 4966 HStore store = region.getStore(fam1); 4967 Collection<HStoreFile> storeFiles = store.getStorefiles(); 4968 for (HStoreFile storefile : storeFiles) { 4969 StoreFileReader reader = storefile.getReader(); 4970 reader.loadFileInfo(); 4971 reader.loadBloomfilter(); 4972 assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries()); 4973 assertEquals(num_unique_rows, reader.getFilterEntries()); 4974 } 4975 4976 region.compact(true); 4977 4978 // after compaction 4979 storeFiles = store.getStorefiles(); 4980 for (HStoreFile storefile : storeFiles) { 4981 StoreFileReader reader = storefile.getReader(); 4982 reader.loadFileInfo(); 4983 reader.loadBloomfilter(); 4984 assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries()); 4985 assertEquals(num_unique_rows, reader.getFilterEntries()); 4986 } 4987 } 4988 4989 @Test 4990 public void testAllColumnsWithBloomFilter() throws IOException { 4991 byte[] TABLE = Bytes.toBytes(name.getMethodName()); 4992 byte[] FAMILY = Bytes.toBytes("family"); 4993 4994 // Create table 4995 TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(TableName.valueOf(TABLE)) 4996 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY) 4997 .setMaxVersions(Integer.MAX_VALUE).setBloomFilterType(BloomType.ROWCOL).build()) 4998 .build(); 4999 RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 5000 this.region = TEST_UTIL.createLocalHRegion(info, tableDescriptor); 5001 // For row:0, col:0: insert versions 1 through 5. 5002 byte[] row = Bytes.toBytes("row:" + 0); 5003 byte[] column = Bytes.toBytes("column:" + 0); 5004 Put put = new Put(row); 5005 put.setDurability(Durability.SKIP_WAL); 5006 for (long idx = 1; idx <= 4; idx++) { 5007 put.addColumn(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx)); 5008 } 5009 region.put(put); 5010 5011 // Flush 5012 region.flush(true); 5013 5014 // Get rows 5015 Get get = new Get(row); 5016 get.readAllVersions(); 5017 Cell[] kvs = region.get(get).rawCells(); 5018 5019 // Check if rows are correct 5020 assertEquals(4, kvs.length); 5021 checkOneCell(kvs[0], FAMILY, 0, 0, 4); 5022 checkOneCell(kvs[1], FAMILY, 0, 0, 3); 5023 checkOneCell(kvs[2], FAMILY, 0, 0, 2); 5024 checkOneCell(kvs[3], FAMILY, 0, 0, 1); 5025 } 5026 5027 /** 5028 * Testcase to cover bug-fix for HBASE-2823 Ensures correct delete when issuing delete row on 5029 * columns with bloom filter set to row+col (BloomType.ROWCOL) 5030 */ 5031 @Test 5032 public void testDeleteRowWithBloomFilter() throws IOException { 5033 byte[] familyName = Bytes.toBytes("familyName"); 5034 5035 // Create Table 5036 TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName) 5037 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(familyName) 5038 .setMaxVersions(Integer.MAX_VALUE).setBloomFilterType(BloomType.ROWCOL).build()) 5039 .build(); 5040 RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 5041 this.region = TEST_UTIL.createLocalHRegion(info, tableDescriptor); 5042 // Insert some data 5043 byte[] row = Bytes.toBytes("row1"); 5044 byte[] col = Bytes.toBytes("col1"); 5045 5046 Put put = new Put(row); 5047 put.addColumn(familyName, col, 1, Bytes.toBytes("SomeRandomValue")); 5048 region.put(put); 5049 region.flush(true); 5050 5051 Delete del = new Delete(row); 5052 region.delete(del); 5053 region.flush(true); 5054 5055 // Get remaining rows (should have none) 5056 Get get = new Get(row); 5057 get.addColumn(familyName, col); 5058 5059 Cell[] keyValues = region.get(get).rawCells(); 5060 assertEquals(0, keyValues.length); 5061 } 5062 5063 @Test 5064 public void testgetHDFSBlocksDistribution() throws Exception { 5065 HBaseTestingUtil htu = new HBaseTestingUtil(); 5066 // Why do we set the block size in this test? If we set it smaller than the kvs, then we'll 5067 // break up the file in to more pieces that can be distributed across the three nodes and we 5068 // won't be able to have the condition this test asserts; that at least one node has 5069 // a copy of all replicas -- if small block size, then blocks are spread evenly across the 5070 // the three nodes. hfilev3 with tags seems to put us over the block size. St.Ack. 5071 // final int DEFAULT_BLOCK_SIZE = 1024; 5072 // htu.getConfiguration().setLong("dfs.blocksize", DEFAULT_BLOCK_SIZE); 5073 htu.getConfiguration().setInt("dfs.replication", 2); 5074 5075 // set up a cluster with 3 nodes 5076 SingleProcessHBaseCluster cluster = null; 5077 String dataNodeHosts[] = new String[] { "host1", "host2", "host3" }; 5078 int regionServersCount = 3; 5079 5080 try { 5081 StartTestingClusterOption option = StartTestingClusterOption.builder() 5082 .numRegionServers(regionServersCount).dataNodeHosts(dataNodeHosts).build(); 5083 cluster = htu.startMiniCluster(option); 5084 byte[][] families = { fam1, fam2 }; 5085 Table ht = htu.createTable(tableName, families); 5086 5087 // Setting up region 5088 byte row[] = Bytes.toBytes("row1"); 5089 byte col[] = Bytes.toBytes("col1"); 5090 5091 Put put = new Put(row); 5092 put.addColumn(fam1, col, 1, Bytes.toBytes("test1")); 5093 put.addColumn(fam2, col, 1, Bytes.toBytes("test2")); 5094 ht.put(put); 5095 5096 HRegion firstRegion = htu.getHBaseCluster().getRegions(tableName).get(0); 5097 firstRegion.flush(true); 5098 HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution(); 5099 5100 // Given the default replication factor is 2 and we have 2 HFiles, 5101 // we will have total of 4 replica of blocks on 3 datanodes; thus there 5102 // must be at least one host that have replica for 2 HFiles. That host's 5103 // weight will be equal to the unique block weight. 5104 long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight(); 5105 StringBuilder sb = new StringBuilder(); 5106 for (String host : blocksDistribution1.getTopHosts()) { 5107 if (sb.length() > 0) sb.append(", "); 5108 sb.append(host); 5109 sb.append("="); 5110 sb.append(blocksDistribution1.getWeight(host)); 5111 } 5112 5113 String topHost = blocksDistribution1.getTopHosts().get(0); 5114 long topHostWeight = blocksDistribution1.getWeight(topHost); 5115 String msg = "uniqueBlocksWeight=" + uniqueBlocksWeight1 + ", topHostWeight=" + topHostWeight 5116 + ", topHost=" + topHost + "; " + sb.toString(); 5117 LOG.info(msg); 5118 assertTrue(msg, uniqueBlocksWeight1 == topHostWeight); 5119 5120 // use the static method to compute the value, it should be the same. 5121 // static method is used by load balancer or other components 5122 HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution( 5123 htu.getConfiguration(), firstRegion.getTableDescriptor(), firstRegion.getRegionInfo()); 5124 long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight(); 5125 5126 assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2); 5127 5128 ht.close(); 5129 } finally { 5130 if (cluster != null) { 5131 htu.shutdownMiniCluster(); 5132 } 5133 } 5134 } 5135 5136 /** 5137 * Testcase to check state of region initialization task set to ABORTED or not if any exceptions 5138 * during initialization 5139 */ 5140 @Test 5141 public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception { 5142 RegionInfo info; 5143 try { 5144 FileSystem fs = mock(FileSystem.class); 5145 when(fs.exists(any())).thenThrow(new IOException()); 5146 TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(tableName); 5147 ColumnFamilyDescriptor columnFamilyDescriptor = 5148 ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf")).build(); 5149 tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor); 5150 info = RegionInfoBuilder.newBuilder(tableName).build(); 5151 Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization"); 5152 region = HRegion.newHRegion(path, null, fs, CONF, info, tableDescriptorBuilder.build(), null); 5153 // region initialization throws IOException and set task state to ABORTED. 5154 region.initialize(); 5155 fail("Region initialization should fail due to IOException"); 5156 } catch (IOException io) { 5157 List<MonitoredTask> tasks = TaskMonitor.get().getTasks(); 5158 for (MonitoredTask monitoredTask : tasks) { 5159 if ( 5160 !(monitoredTask instanceof MonitoredRPCHandler) 5161 && monitoredTask.getDescription().contains(region.toString()) 5162 ) { 5163 assertTrue("Region state should be ABORTED.", 5164 monitoredTask.getState().equals(MonitoredTask.State.ABORTED)); 5165 break; 5166 } 5167 } 5168 } 5169 } 5170 5171 /** 5172 * Verifies that the .regioninfo file is written on region creation and that is recreated if 5173 * missing during region opening. 5174 */ 5175 @Test 5176 public void testRegionInfoFileCreation() throws IOException { 5177 Path rootDir = new Path(dir + "testRegionInfoFileCreation"); 5178 5179 TableDescriptorBuilder tableDescriptorBuilder = 5180 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())); 5181 ColumnFamilyDescriptor columnFamilyDescriptor = 5182 ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf")).build(); 5183 tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor); 5184 TableDescriptor tableDescriptor = tableDescriptorBuilder.build(); 5185 5186 RegionInfo hri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 5187 5188 // Create a region and skip the initialization (like CreateTableHandler) 5189 region = HBaseTestingUtil.createRegionAndWAL(hri, rootDir, CONF, tableDescriptor, false); 5190 Path regionDir = region.getRegionFileSystem().getRegionDir(); 5191 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 5192 HBaseTestingUtil.closeRegionAndWAL(region); 5193 5194 Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE); 5195 5196 // Verify that the .regioninfo file is present 5197 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir", 5198 fs.exists(regionInfoFile)); 5199 5200 // Try to open the region 5201 region = HRegion.openHRegion(rootDir, hri, tableDescriptor, null, CONF); 5202 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir()); 5203 HBaseTestingUtil.closeRegionAndWAL(region); 5204 5205 // Verify that the .regioninfo file is still there 5206 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir", 5207 fs.exists(regionInfoFile)); 5208 5209 // Remove the .regioninfo file and verify is recreated on region open 5210 fs.delete(regionInfoFile, true); 5211 assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir", 5212 fs.exists(regionInfoFile)); 5213 5214 region = HRegion.openHRegion(rootDir, hri, tableDescriptor, null, CONF); 5215 // region = TEST_UTIL.openHRegion(hri, htd); 5216 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir()); 5217 HBaseTestingUtil.closeRegionAndWAL(region); 5218 5219 // Verify that the .regioninfo file is still there 5220 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir", 5221 fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE))); 5222 5223 region = null; 5224 } 5225 5226 /** 5227 * TestCase for increment 5228 */ 5229 private static class Incrementer implements Runnable { 5230 private HRegion region; 5231 private final static byte[] incRow = Bytes.toBytes("incRow"); 5232 private final static byte[] family = Bytes.toBytes("family"); 5233 private final static byte[] qualifier = Bytes.toBytes("qualifier"); 5234 private final static long ONE = 1L; 5235 private int incCounter; 5236 5237 public Incrementer(HRegion region, int incCounter) { 5238 this.region = region; 5239 this.incCounter = incCounter; 5240 } 5241 5242 @Override 5243 public void run() { 5244 int count = 0; 5245 while (count < incCounter) { 5246 Increment inc = new Increment(incRow); 5247 inc.addColumn(family, qualifier, ONE); 5248 count++; 5249 try { 5250 region.increment(inc); 5251 } catch (IOException e) { 5252 LOG.info("Count=" + count + ", " + e); 5253 break; 5254 } 5255 } 5256 } 5257 } 5258 5259 /** 5260 * Test case to check increment function with memstore flushing 5261 */ 5262 @Test 5263 public void testParallelIncrementWithMemStoreFlush() throws Exception { 5264 byte[] family = Incrementer.family; 5265 this.region = initHRegion(tableName, method, CONF, family); 5266 final HRegion region = this.region; 5267 final AtomicBoolean incrementDone = new AtomicBoolean(false); 5268 Runnable flusher = new Runnable() { 5269 @Override 5270 public void run() { 5271 while (!incrementDone.get()) { 5272 try { 5273 region.flush(true); 5274 } catch (Exception e) { 5275 e.printStackTrace(); 5276 } 5277 } 5278 } 5279 }; 5280 5281 // after all increment finished, the row will increment to 20*100 = 2000 5282 int threadNum = 20; 5283 int incCounter = 100; 5284 long expected = (long) threadNum * incCounter; 5285 Thread[] incrementers = new Thread[threadNum]; 5286 Thread flushThread = new Thread(flusher); 5287 for (int i = 0; i < threadNum; i++) { 5288 incrementers[i] = new Thread(new Incrementer(this.region, incCounter)); 5289 incrementers[i].start(); 5290 } 5291 flushThread.start(); 5292 for (int i = 0; i < threadNum; i++) { 5293 incrementers[i].join(); 5294 } 5295 5296 incrementDone.set(true); 5297 flushThread.join(); 5298 5299 Get get = new Get(Incrementer.incRow); 5300 get.addColumn(Incrementer.family, Incrementer.qualifier); 5301 get.readVersions(1); 5302 Result res = this.region.get(get); 5303 List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier); 5304 5305 // we just got the latest version 5306 assertEquals(1, kvs.size()); 5307 Cell kv = kvs.get(0); 5308 assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset())); 5309 } 5310 5311 /** 5312 * TestCase for append 5313 */ 5314 private static class Appender implements Runnable { 5315 private HRegion region; 5316 private final static byte[] appendRow = Bytes.toBytes("appendRow"); 5317 private final static byte[] family = Bytes.toBytes("family"); 5318 private final static byte[] qualifier = Bytes.toBytes("qualifier"); 5319 private final static byte[] CHAR = Bytes.toBytes("a"); 5320 private int appendCounter; 5321 5322 public Appender(HRegion region, int appendCounter) { 5323 this.region = region; 5324 this.appendCounter = appendCounter; 5325 } 5326 5327 @Override 5328 public void run() { 5329 int count = 0; 5330 while (count < appendCounter) { 5331 Append app = new Append(appendRow); 5332 app.addColumn(family, qualifier, CHAR); 5333 count++; 5334 try { 5335 region.append(app); 5336 } catch (IOException e) { 5337 LOG.info("Count=" + count + ", max=" + appendCounter + ", " + e); 5338 break; 5339 } 5340 } 5341 } 5342 } 5343 5344 /** 5345 * Test case to check append function with memstore flushing 5346 */ 5347 @Test 5348 public void testParallelAppendWithMemStoreFlush() throws Exception { 5349 byte[] family = Appender.family; 5350 this.region = initHRegion(tableName, method, CONF, family); 5351 final HRegion region = this.region; 5352 final AtomicBoolean appendDone = new AtomicBoolean(false); 5353 Runnable flusher = new Runnable() { 5354 @Override 5355 public void run() { 5356 while (!appendDone.get()) { 5357 try { 5358 region.flush(true); 5359 } catch (Exception e) { 5360 e.printStackTrace(); 5361 } 5362 } 5363 } 5364 }; 5365 5366 // After all append finished, the value will append to threadNum * 5367 // appendCounter Appender.CHAR 5368 int threadNum = 20; 5369 int appendCounter = 100; 5370 byte[] expected = new byte[threadNum * appendCounter]; 5371 for (int i = 0; i < threadNum * appendCounter; i++) { 5372 System.arraycopy(Appender.CHAR, 0, expected, i, 1); 5373 } 5374 Thread[] appenders = new Thread[threadNum]; 5375 Thread flushThread = new Thread(flusher); 5376 for (int i = 0; i < threadNum; i++) { 5377 appenders[i] = new Thread(new Appender(this.region, appendCounter)); 5378 appenders[i].start(); 5379 } 5380 flushThread.start(); 5381 for (int i = 0; i < threadNum; i++) { 5382 appenders[i].join(); 5383 } 5384 5385 appendDone.set(true); 5386 flushThread.join(); 5387 5388 Get get = new Get(Appender.appendRow); 5389 get.addColumn(Appender.family, Appender.qualifier); 5390 get.readVersions(1); 5391 Result res = this.region.get(get); 5392 List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier); 5393 5394 // we just got the latest version 5395 assertEquals(1, kvs.size()); 5396 Cell kv = kvs.get(0); 5397 byte[] appendResult = new byte[kv.getValueLength()]; 5398 System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength()); 5399 assertArrayEquals(expected, appendResult); 5400 } 5401 5402 /** 5403 * Test case to check put function with memstore flushing for same row, same ts 5404 */ 5405 @Test 5406 public void testPutWithMemStoreFlush() throws Exception { 5407 byte[] family = Bytes.toBytes("family"); 5408 byte[] qualifier = Bytes.toBytes("qualifier"); 5409 byte[] row = Bytes.toBytes("putRow"); 5410 byte[] value = null; 5411 this.region = initHRegion(tableName, method, CONF, family); 5412 Put put = null; 5413 Get get = null; 5414 List<Cell> kvs = null; 5415 Result res = null; 5416 5417 put = new Put(row); 5418 value = Bytes.toBytes("value0"); 5419 put.addColumn(family, qualifier, 1234567L, value); 5420 region.put(put); 5421 get = new Get(row); 5422 get.addColumn(family, qualifier); 5423 get.readAllVersions(); 5424 res = this.region.get(get); 5425 kvs = res.getColumnCells(family, qualifier); 5426 assertEquals(1, kvs.size()); 5427 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0))); 5428 5429 region.flush(true); 5430 get = new Get(row); 5431 get.addColumn(family, qualifier); 5432 get.readAllVersions(); 5433 res = this.region.get(get); 5434 kvs = res.getColumnCells(family, qualifier); 5435 assertEquals(1, kvs.size()); 5436 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0))); 5437 5438 put = new Put(row); 5439 value = Bytes.toBytes("value1"); 5440 put.addColumn(family, qualifier, 1234567L, value); 5441 region.put(put); 5442 get = new Get(row); 5443 get.addColumn(family, qualifier); 5444 get.readAllVersions(); 5445 res = this.region.get(get); 5446 kvs = res.getColumnCells(family, qualifier); 5447 assertEquals(1, kvs.size()); 5448 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0))); 5449 5450 region.flush(true); 5451 get = new Get(row); 5452 get.addColumn(family, qualifier); 5453 get.readAllVersions(); 5454 res = this.region.get(get); 5455 kvs = res.getColumnCells(family, qualifier); 5456 assertEquals(1, kvs.size()); 5457 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0))); 5458 } 5459 5460 /** 5461 * For this test,the spied {@link AsyncFSWAL} can not work properly because of a Mockito defect 5462 * that can not deal with classes which have a field of an inner class. See discussions in 5463 * HBASE-15536.When we reuse the code of {@link AsyncFSWAL} for {@link FSHLog}, this test could 5464 * not work for {@link FSHLog} also. 5465 */ 5466 @Test 5467 @Ignore 5468 public void testDurability() throws Exception { 5469 // there are 5 x 5 cases: 5470 // table durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT) x mutation 5471 // durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT) 5472 5473 // expected cases for append and sync wal 5474 durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false); 5475 durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false); 5476 durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false); 5477 5478 durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false); 5479 durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false); 5480 durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false); 5481 5482 durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false); 5483 durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false); 5484 5485 durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false); 5486 durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false); 5487 5488 durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false); 5489 durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false); 5490 durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false); 5491 5492 // expected cases for async wal 5493 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false); 5494 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false); 5495 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false); 5496 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false); 5497 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false); 5498 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false); 5499 5500 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true); 5501 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true); 5502 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true); 5503 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true); 5504 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true); 5505 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true); 5506 5507 // expect skip wal cases 5508 durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false); 5509 durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false); 5510 durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false); 5511 durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false); 5512 durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false); 5513 durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false); 5514 5515 } 5516 5517 private void durabilityTest(String method, Durability tableDurability, 5518 Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync, 5519 final boolean expectSyncFromLogSyncer) throws Exception { 5520 Configuration conf = HBaseConfiguration.create(CONF); 5521 conf.setLong(AbstractFSWAL.WAL_SHUTDOWN_WAIT_TIMEOUT_MS, 60 * 60 * 1000); 5522 method = method + "_" + tableDurability.name() + "_" + mutationDurability.name(); 5523 byte[] family = Bytes.toBytes("family"); 5524 Path logDir = new Path(new Path(dir + method), "log"); 5525 final Configuration walConf = new Configuration(conf); 5526 CommonFSUtils.setRootDir(walConf, logDir); 5527 // XXX: The spied AsyncFSWAL can not work properly because of a Mockito defect that can not 5528 // deal with classes which have a field of an inner class. See discussions in HBASE-15536. 5529 walConf.set(WALFactory.WAL_PROVIDER, "filesystem"); 5530 final WALFactory wals = new WALFactory(walConf, HBaseTestingUtil.getRandomUUID().toString()); 5531 final WAL wal = spy(wals.getWAL(RegionInfoBuilder.newBuilder(tableName).build())); 5532 this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF, 5533 false, tableDurability, wal, new byte[][] { family }); 5534 5535 Put put = new Put(Bytes.toBytes("r1")); 5536 put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); 5537 put.setDurability(mutationDurability); 5538 region.put(put); 5539 5540 // verify append called or not 5541 verify(wal, expectAppend ? times(1) : never()).appendData((RegionInfo) any(), 5542 (WALKeyImpl) any(), (WALEdit) any()); 5543 5544 // verify sync called or not 5545 if (expectSync || expectSyncFromLogSyncer) { 5546 TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() { 5547 @Override 5548 public boolean evaluate() throws Exception { 5549 try { 5550 if (expectSync) { 5551 verify(wal, times(1)).sync(anyLong()); // Hregion calls this one 5552 } else if (expectSyncFromLogSyncer) { 5553 verify(wal, times(1)).sync(); // wal syncer calls this one 5554 } 5555 } catch (Throwable ignore) { 5556 } 5557 return true; 5558 } 5559 }); 5560 } else { 5561 // verify(wal, never()).sync(anyLong()); 5562 verify(wal, never()).sync(); 5563 } 5564 5565 HBaseTestingUtil.closeRegionAndWAL(this.region); 5566 wals.close(); 5567 this.region = null; 5568 } 5569 5570 @Test 5571 public void testRegionReplicaSecondary() throws IOException { 5572 // create a primary region, load some data and flush 5573 // create a secondary region, and do a get against that 5574 Path rootDir = new Path(dir + name.getMethodName()); 5575 CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir); 5576 5577 byte[][] families = 5578 new byte[][] { Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3") }; 5579 byte[] cq = Bytes.toBytes("cq"); 5580 TableDescriptorBuilder builder = 5581 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())); 5582 for (byte[] family : families) { 5583 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)); 5584 } 5585 TableDescriptor tableDescriptor = builder.build(); 5586 long time = EnvironmentEdgeManager.currentTime(); 5587 RegionInfo primaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5588 .setRegionId(time).setReplicaId(0).build(); 5589 RegionInfo secondaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5590 .setRegionId(time).setReplicaId(1).build(); 5591 5592 HRegion primaryRegion = null, secondaryRegion = null; 5593 5594 try { 5595 primaryRegion = HBaseTestingUtil.createRegionAndWAL(primaryHri, rootDir, 5596 TEST_UTIL.getConfiguration(), tableDescriptor); 5597 5598 // load some data 5599 putData(primaryRegion, 0, 1000, cq, families); 5600 5601 // flush region 5602 primaryRegion.flush(true); 5603 5604 // open secondary region 5605 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, tableDescriptor, null, CONF); 5606 5607 verifyData(secondaryRegion, 0, 1000, cq, families); 5608 } finally { 5609 if (primaryRegion != null) { 5610 HBaseTestingUtil.closeRegionAndWAL(primaryRegion); 5611 } 5612 if (secondaryRegion != null) { 5613 HBaseTestingUtil.closeRegionAndWAL(secondaryRegion); 5614 } 5615 } 5616 } 5617 5618 @Test 5619 public void testRegionReplicaSecondaryIsReadOnly() throws IOException { 5620 // create a primary region, load some data and flush 5621 // create a secondary region, and do a put against that 5622 Path rootDir = new Path(dir + name.getMethodName()); 5623 CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir); 5624 5625 byte[][] families = 5626 new byte[][] { Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3") }; 5627 byte[] cq = Bytes.toBytes("cq"); 5628 TableDescriptorBuilder builder = 5629 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())); 5630 for (byte[] family : families) { 5631 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)); 5632 } 5633 TableDescriptor tableDescriptor = builder.build(); 5634 long time = EnvironmentEdgeManager.currentTime(); 5635 RegionInfo primaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5636 .setRegionId(time).setReplicaId(0).build(); 5637 RegionInfo secondaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5638 .setRegionId(time).setReplicaId(1).build(); 5639 5640 HRegion primaryRegion = null, secondaryRegion = null; 5641 5642 try { 5643 primaryRegion = HBaseTestingUtil.createRegionAndWAL(primaryHri, rootDir, 5644 TEST_UTIL.getConfiguration(), tableDescriptor); 5645 5646 // load some data 5647 putData(primaryRegion, 0, 1000, cq, families); 5648 5649 // flush region 5650 primaryRegion.flush(true); 5651 5652 // open secondary region 5653 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, tableDescriptor, null, CONF); 5654 5655 try { 5656 putData(secondaryRegion, 0, 1000, cq, families); 5657 fail("Should have thrown exception"); 5658 } catch (IOException ex) { 5659 // expected 5660 } 5661 } finally { 5662 if (primaryRegion != null) { 5663 HBaseTestingUtil.closeRegionAndWAL(primaryRegion); 5664 } 5665 if (secondaryRegion != null) { 5666 HBaseTestingUtil.closeRegionAndWAL(secondaryRegion); 5667 } 5668 } 5669 } 5670 5671 static WALFactory createWALFactory(Configuration conf, Path rootDir) throws IOException { 5672 Configuration confForWAL = new Configuration(conf); 5673 confForWAL.set(HConstants.HBASE_DIR, rootDir.toString()); 5674 return new WALFactory(confForWAL, "hregion-" + RandomStringUtils.randomNumeric(8)); 5675 } 5676 5677 @Test 5678 public void testCompactionFromPrimary() throws IOException { 5679 Path rootDir = new Path(dir + name.getMethodName()); 5680 CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir); 5681 5682 byte[][] families = 5683 new byte[][] { Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3") }; 5684 byte[] cq = Bytes.toBytes("cq"); 5685 TableDescriptorBuilder builder = 5686 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())); 5687 for (byte[] family : families) { 5688 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)); 5689 } 5690 TableDescriptor tableDescriptor = builder.build(); 5691 long time = EnvironmentEdgeManager.currentTime(); 5692 RegionInfo primaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5693 .setRegionId(time).setReplicaId(0).build(); 5694 RegionInfo secondaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5695 .setRegionId(time).setReplicaId(1).build(); 5696 5697 HRegion primaryRegion = null, secondaryRegion = null; 5698 5699 try { 5700 primaryRegion = HBaseTestingUtil.createRegionAndWAL(primaryHri, rootDir, 5701 TEST_UTIL.getConfiguration(), tableDescriptor); 5702 5703 // load some data 5704 putData(primaryRegion, 0, 1000, cq, families); 5705 5706 // flush region 5707 primaryRegion.flush(true); 5708 5709 // open secondary region 5710 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, tableDescriptor, null, CONF); 5711 5712 // move the file of the primary region to the archive, simulating a compaction 5713 Collection<HStoreFile> storeFiles = primaryRegion.getStore(families[0]).getStorefiles(); 5714 HRegionFileSystem regionFs = primaryRegion.getRegionFileSystem(); 5715 StoreFileTracker sft = StoreFileTrackerFactory.create(primaryRegion.getBaseConf(), false, 5716 StoreContext.getBuilder() 5717 .withColumnFamilyDescriptor(ColumnFamilyDescriptorBuilder.newBuilder(families[0]).build()) 5718 .withFamilyStoreDirectoryPath( 5719 new Path(regionFs.getRegionDir(), Bytes.toString(families[0]))) 5720 .withRegionFileSystem(regionFs).build()); 5721 sft.removeStoreFiles(storeFiles.stream().collect(Collectors.toList())); 5722 Collection<StoreFileInfo> storeFileInfos = sft.load(); 5723 Assert.assertTrue(storeFileInfos == null || storeFileInfos.isEmpty()); 5724 5725 verifyData(secondaryRegion, 0, 1000, cq, families); 5726 } finally { 5727 if (primaryRegion != null) { 5728 HBaseTestingUtil.closeRegionAndWAL(primaryRegion); 5729 } 5730 if (secondaryRegion != null) { 5731 HBaseTestingUtil.closeRegionAndWAL(secondaryRegion); 5732 } 5733 } 5734 } 5735 5736 private void putData(int startRow, int numRows, byte[] qf, byte[]... families) 5737 throws IOException { 5738 putData(this.region, startRow, numRows, qf, families); 5739 } 5740 5741 private void putData(HRegion region, int startRow, int numRows, byte[] qf, byte[]... families) 5742 throws IOException { 5743 putData(region, Durability.SKIP_WAL, startRow, numRows, qf, families); 5744 } 5745 5746 static void putData(HRegion region, Durability durability, int startRow, int numRows, byte[] qf, 5747 byte[]... families) throws IOException { 5748 for (int i = startRow; i < startRow + numRows; i++) { 5749 Put put = new Put(Bytes.toBytes("" + i)); 5750 put.setDurability(durability); 5751 for (byte[] family : families) { 5752 put.addColumn(family, qf, null); 5753 } 5754 region.put(put); 5755 LOG.info(put.toString()); 5756 } 5757 } 5758 5759 static void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families) 5760 throws IOException { 5761 for (int i = startRow; i < startRow + numRows; i++) { 5762 byte[] row = Bytes.toBytes("" + i); 5763 Get get = new Get(row); 5764 for (byte[] family : families) { 5765 get.addColumn(family, qf); 5766 } 5767 Result result = newReg.get(get); 5768 Cell[] raw = result.rawCells(); 5769 assertEquals(families.length, result.size()); 5770 for (int j = 0; j < families.length; j++) { 5771 assertTrue(CellUtil.matchingRows(raw[j], row)); 5772 assertTrue(CellUtil.matchingFamily(raw[j], families[j])); 5773 assertTrue(CellUtil.matchingQualifier(raw[j], qf)); 5774 } 5775 } 5776 } 5777 5778 static void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException { 5779 // Now I have k, get values out and assert they are as expected. 5780 Get get = new Get(k).addFamily(family).readAllVersions(); 5781 Cell[] results = r.get(get).rawCells(); 5782 for (int j = 0; j < results.length; j++) { 5783 byte[] tmp = CellUtil.cloneValue(results[j]); 5784 // Row should be equal to value every time. 5785 assertTrue(Bytes.equals(k, tmp)); 5786 } 5787 } 5788 5789 /* 5790 * Assert first value in the passed region is <code>firstValue</code>. 5791 */ 5792 protected void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue) 5793 throws IOException { 5794 byte[][] families = { fs }; 5795 Scan scan = new Scan(); 5796 for (int i = 0; i < families.length; i++) 5797 scan.addFamily(families[i]); 5798 try (InternalScanner s = r.getScanner(scan)) { 5799 List<Cell> curVals = new ArrayList<>(); 5800 boolean first = true; 5801 OUTER_LOOP: while (s.next(curVals)) { 5802 for (Cell kv : curVals) { 5803 byte[] val = CellUtil.cloneValue(kv); 5804 byte[] curval = val; 5805 if (first) { 5806 first = false; 5807 assertTrue(Bytes.compareTo(curval, firstValue) == 0); 5808 } else { 5809 // Not asserting anything. Might as well break. 5810 break OUTER_LOOP; 5811 } 5812 } 5813 } 5814 } 5815 } 5816 5817 /** 5818 * Test that we get the expected flush results back 5819 */ 5820 @Test 5821 public void testFlushResult() throws IOException { 5822 byte[] family = Bytes.toBytes("family"); 5823 5824 this.region = initHRegion(tableName, method, family); 5825 5826 // empty memstore, flush doesn't run 5827 HRegion.FlushResult fr = region.flush(true); 5828 assertFalse(fr.isFlushSucceeded()); 5829 assertFalse(fr.isCompactionNeeded()); 5830 5831 // Flush enough files to get up to the threshold, doesn't need compactions 5832 for (int i = 0; i < 2; i++) { 5833 Put put = new Put(tableName.toBytes()).addColumn(family, family, tableName.toBytes()); 5834 region.put(put); 5835 fr = region.flush(true); 5836 assertTrue(fr.isFlushSucceeded()); 5837 assertFalse(fr.isCompactionNeeded()); 5838 } 5839 5840 // Two flushes after the threshold, compactions are needed 5841 for (int i = 0; i < 2; i++) { 5842 Put put = new Put(tableName.toBytes()).addColumn(family, family, tableName.toBytes()); 5843 region.put(put); 5844 fr = region.flush(true); 5845 assertTrue(fr.isFlushSucceeded()); 5846 assertTrue(fr.isCompactionNeeded()); 5847 } 5848 } 5849 5850 protected Configuration initSplit() { 5851 // Always compact if there is more than one store file. 5852 CONF.setInt("hbase.hstore.compactionThreshold", 2); 5853 5854 CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000); 5855 5856 // Increase the amount of time between client retries 5857 CONF.setLong("hbase.client.pause", 15 * 1000); 5858 5859 // This size should make it so we always split using the addContent 5860 // below. After adding all data, the first region is 1.3M 5861 CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128); 5862 return CONF; 5863 } 5864 5865 /** 5866 * @return A region on which you must call {@link HBaseTestingUtil#closeRegionAndWAL(HRegion)} 5867 * when done. 5868 */ 5869 protected HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf, 5870 byte[]... families) throws IOException { 5871 return initHRegion(tableName, callingMethod, conf, false, families); 5872 } 5873 5874 /** 5875 * @return A region on which you must call {@link HBaseTestingUtil#closeRegionAndWAL(HRegion)} 5876 * when done. 5877 */ 5878 private HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf, 5879 boolean isReadOnly, byte[]... families) throws IOException { 5880 return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families); 5881 } 5882 5883 private HRegion initHRegion(TableName tableName, byte[] startKey, byte[] stopKey, 5884 String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families) 5885 throws IOException { 5886 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(callingMethod + ".log"); 5887 RegionInfo hri = 5888 RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey).setEndKey(stopKey).build(); 5889 final WAL wal = HBaseTestingUtil.createWal(conf, logDir, hri); 5890 return initHRegion(tableName, startKey, stopKey, conf, isReadOnly, Durability.SYNC_WAL, wal, 5891 families); 5892 } 5893 5894 /** 5895 * @return A region on which you must call {@link HBaseTestingUtil#closeRegionAndWAL(HRegion)} 5896 * when done. 5897 */ 5898 protected HRegion initHRegion(TableName tableName, byte[] startKey, byte[] stopKey, 5899 Configuration conf, boolean isReadOnly, Durability durability, WAL wal, byte[]... families) 5900 throws IOException { 5901 ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null, 5902 MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT); 5903 return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, conf, isReadOnly, durability, 5904 wal, families); 5905 } 5906 5907 /** 5908 * Assert that the passed in Cell has expected contents for the specified row, column & timestamp. 5909 */ 5910 private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) { 5911 String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts; 5912 assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx, 5913 Bytes.toString(CellUtil.cloneRow(kv))); 5914 assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf), 5915 Bytes.toString(CellUtil.cloneFamily(kv))); 5916 assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx, 5917 Bytes.toString(CellUtil.cloneQualifier(kv))); 5918 assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp()); 5919 assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts, 5920 Bytes.toString(CellUtil.cloneValue(kv))); 5921 } 5922 5923 @Test 5924 public void testReverseScanner_FromMemStore_SingleCF_Normal() throws IOException { 5925 byte[] rowC = Bytes.toBytes("rowC"); 5926 byte[] rowA = Bytes.toBytes("rowA"); 5927 byte[] rowB = Bytes.toBytes("rowB"); 5928 byte[] cf = Bytes.toBytes("CF"); 5929 byte[][] families = { cf }; 5930 byte[] col = Bytes.toBytes("C"); 5931 long ts = 1; 5932 this.region = initHRegion(tableName, method, families); 5933 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null); 5934 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, null); 5935 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null); 5936 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null); 5937 Put put = null; 5938 put = new Put(rowC); 5939 put.add(kv1); 5940 put.add(kv11); 5941 region.put(put); 5942 put = new Put(rowA); 5943 put.add(kv2); 5944 region.put(put); 5945 put = new Put(rowB); 5946 put.add(kv3); 5947 region.put(put); 5948 5949 Scan scan = new Scan().withStartRow(rowC); 5950 scan.readVersions(5); 5951 scan.setReversed(true); 5952 try (InternalScanner scanner = region.getScanner(scan)) { 5953 List<Cell> currRow = new ArrayList<>(); 5954 boolean hasNext = scanner.next(currRow); 5955 assertEquals(2, currRow.size()); 5956 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5957 currRow.get(0).getRowLength(), rowC, 0, rowC.length)); 5958 assertTrue(hasNext); 5959 currRow.clear(); 5960 hasNext = scanner.next(currRow); 5961 assertEquals(1, currRow.size()); 5962 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5963 currRow.get(0).getRowLength(), rowB, 0, rowB.length)); 5964 assertTrue(hasNext); 5965 currRow.clear(); 5966 hasNext = scanner.next(currRow); 5967 assertEquals(1, currRow.size()); 5968 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5969 currRow.get(0).getRowLength(), rowA, 0, rowA.length)); 5970 assertFalse(hasNext); 5971 } 5972 } 5973 5974 @Test 5975 public void testReverseScanner_FromMemStore_SingleCF_LargerKey() throws IOException { 5976 byte[] rowC = Bytes.toBytes("rowC"); 5977 byte[] rowA = Bytes.toBytes("rowA"); 5978 byte[] rowB = Bytes.toBytes("rowB"); 5979 byte[] rowD = Bytes.toBytes("rowD"); 5980 byte[] cf = Bytes.toBytes("CF"); 5981 byte[][] families = { cf }; 5982 byte[] col = Bytes.toBytes("C"); 5983 long ts = 1; 5984 this.region = initHRegion(tableName, method, families); 5985 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null); 5986 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, null); 5987 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null); 5988 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null); 5989 Put put = null; 5990 put = new Put(rowC); 5991 put.add(kv1); 5992 put.add(kv11); 5993 region.put(put); 5994 put = new Put(rowA); 5995 put.add(kv2); 5996 region.put(put); 5997 put = new Put(rowB); 5998 put.add(kv3); 5999 region.put(put); 6000 6001 Scan scan = new Scan().withStartRow(rowD); 6002 List<Cell> currRow = new ArrayList<>(); 6003 scan.setReversed(true); 6004 scan.readVersions(5); 6005 try (InternalScanner scanner = region.getScanner(scan)) { 6006 boolean hasNext = scanner.next(currRow); 6007 assertEquals(2, currRow.size()); 6008 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6009 currRow.get(0).getRowLength(), rowC, 0, rowC.length)); 6010 assertTrue(hasNext); 6011 currRow.clear(); 6012 hasNext = scanner.next(currRow); 6013 assertEquals(1, currRow.size()); 6014 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6015 currRow.get(0).getRowLength(), rowB, 0, rowB.length)); 6016 assertTrue(hasNext); 6017 currRow.clear(); 6018 hasNext = scanner.next(currRow); 6019 assertEquals(1, currRow.size()); 6020 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6021 currRow.get(0).getRowLength(), rowA, 0, rowA.length)); 6022 assertFalse(hasNext); 6023 } 6024 } 6025 6026 @Test 6027 public void testReverseScanner_FromMemStore_SingleCF_FullScan() throws IOException { 6028 byte[] rowC = Bytes.toBytes("rowC"); 6029 byte[] rowA = Bytes.toBytes("rowA"); 6030 byte[] rowB = Bytes.toBytes("rowB"); 6031 byte[] cf = Bytes.toBytes("CF"); 6032 byte[][] families = { cf }; 6033 byte[] col = Bytes.toBytes("C"); 6034 long ts = 1; 6035 this.region = initHRegion(tableName, method, families); 6036 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null); 6037 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, null); 6038 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null); 6039 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null); 6040 Put put = null; 6041 put = new Put(rowC); 6042 put.add(kv1); 6043 put.add(kv11); 6044 region.put(put); 6045 put = new Put(rowA); 6046 put.add(kv2); 6047 region.put(put); 6048 put = new Put(rowB); 6049 put.add(kv3); 6050 region.put(put); 6051 Scan scan = new Scan(); 6052 List<Cell> currRow = new ArrayList<>(); 6053 scan.setReversed(true); 6054 try (InternalScanner scanner = region.getScanner(scan)) { 6055 boolean hasNext = scanner.next(currRow); 6056 assertEquals(1, currRow.size()); 6057 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6058 currRow.get(0).getRowLength(), rowC, 0, rowC.length)); 6059 assertTrue(hasNext); 6060 currRow.clear(); 6061 hasNext = scanner.next(currRow); 6062 assertEquals(1, currRow.size()); 6063 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6064 currRow.get(0).getRowLength(), rowB, 0, rowB.length)); 6065 assertTrue(hasNext); 6066 currRow.clear(); 6067 hasNext = scanner.next(currRow); 6068 assertEquals(1, currRow.size()); 6069 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6070 currRow.get(0).getRowLength(), rowA, 0, rowA.length)); 6071 assertFalse(hasNext); 6072 } 6073 } 6074 6075 @Test 6076 public void testReverseScanner_moreRowsMayExistAfter() throws IOException { 6077 // case for "INCLUDE_AND_SEEK_NEXT_ROW & SEEK_NEXT_ROW" endless loop 6078 byte[] rowA = Bytes.toBytes("rowA"); 6079 byte[] rowB = Bytes.toBytes("rowB"); 6080 byte[] rowC = Bytes.toBytes("rowC"); 6081 byte[] rowD = Bytes.toBytes("rowD"); 6082 byte[] rowE = Bytes.toBytes("rowE"); 6083 byte[] cf = Bytes.toBytes("CF"); 6084 byte[][] families = { cf }; 6085 byte[] col1 = Bytes.toBytes("col1"); 6086 byte[] col2 = Bytes.toBytes("col2"); 6087 long ts = 1; 6088 this.region = initHRegion(tableName, method, families); 6089 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null); 6090 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null); 6091 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null); 6092 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null); 6093 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null); 6094 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null); 6095 Put put = null; 6096 put = new Put(rowA); 6097 put.add(kv1); 6098 region.put(put); 6099 put = new Put(rowB); 6100 put.add(kv2); 6101 region.put(put); 6102 put = new Put(rowC); 6103 put.add(kv3); 6104 region.put(put); 6105 put = new Put(rowD); 6106 put.add(kv4_1); 6107 region.put(put); 6108 put = new Put(rowD); 6109 put.add(kv4_2); 6110 region.put(put); 6111 put = new Put(rowE); 6112 put.add(kv5); 6113 region.put(put); 6114 region.flush(true); 6115 Scan scan = new Scan().withStartRow(rowD).withStopRow(rowA); 6116 scan.addColumn(families[0], col1); 6117 scan.setReversed(true); 6118 List<Cell> currRow = new ArrayList<>(); 6119 try (InternalScanner scanner = region.getScanner(scan)) { 6120 boolean hasNext = scanner.next(currRow); 6121 assertEquals(1, currRow.size()); 6122 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6123 currRow.get(0).getRowLength(), rowD, 0, rowD.length)); 6124 assertTrue(hasNext); 6125 currRow.clear(); 6126 hasNext = scanner.next(currRow); 6127 assertEquals(1, currRow.size()); 6128 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6129 currRow.get(0).getRowLength(), rowC, 0, rowC.length)); 6130 assertTrue(hasNext); 6131 currRow.clear(); 6132 hasNext = scanner.next(currRow); 6133 assertEquals(1, currRow.size()); 6134 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6135 currRow.get(0).getRowLength(), rowB, 0, rowB.length)); 6136 assertFalse(hasNext); 6137 } 6138 6139 scan = new Scan().withStartRow(rowD).withStopRow(rowA); 6140 scan.addColumn(families[0], col2); 6141 scan.setReversed(true); 6142 currRow.clear(); 6143 try (InternalScanner scanner = region.getScanner(scan)) { 6144 boolean hasNext = scanner.next(currRow); 6145 assertEquals(1, currRow.size()); 6146 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6147 currRow.get(0).getRowLength(), rowD, 0, rowD.length)); 6148 assertTrue(hasNext); 6149 } 6150 } 6151 6152 @Test 6153 public void testReverseScanner_smaller_blocksize() throws IOException { 6154 // case to ensure no conflict with HFile index optimization 6155 byte[] rowA = Bytes.toBytes("rowA"); 6156 byte[] rowB = Bytes.toBytes("rowB"); 6157 byte[] rowC = Bytes.toBytes("rowC"); 6158 byte[] rowD = Bytes.toBytes("rowD"); 6159 byte[] rowE = Bytes.toBytes("rowE"); 6160 byte[] cf = Bytes.toBytes("CF"); 6161 byte[][] families = { cf }; 6162 byte[] col1 = Bytes.toBytes("col1"); 6163 byte[] col2 = Bytes.toBytes("col2"); 6164 long ts = 1; 6165 Configuration conf = new Configuration(CONF); 6166 conf.setInt("test.block.size", 1); 6167 this.region = initHRegion(tableName, method, conf, families); 6168 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null); 6169 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null); 6170 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null); 6171 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null); 6172 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null); 6173 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null); 6174 Put put = null; 6175 put = new Put(rowA); 6176 put.add(kv1); 6177 region.put(put); 6178 put = new Put(rowB); 6179 put.add(kv2); 6180 region.put(put); 6181 put = new Put(rowC); 6182 put.add(kv3); 6183 region.put(put); 6184 put = new Put(rowD); 6185 put.add(kv4_1); 6186 region.put(put); 6187 put = new Put(rowD); 6188 put.add(kv4_2); 6189 region.put(put); 6190 put = new Put(rowE); 6191 put.add(kv5); 6192 region.put(put); 6193 region.flush(true); 6194 Scan scan = new Scan().withStartRow(rowD).withStopRow(rowA); 6195 scan.addColumn(families[0], col1); 6196 scan.setReversed(true); 6197 List<Cell> currRow = new ArrayList<>(); 6198 try (InternalScanner scanner = region.getScanner(scan)) { 6199 boolean hasNext = scanner.next(currRow); 6200 assertEquals(1, currRow.size()); 6201 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6202 currRow.get(0).getRowLength(), rowD, 0, rowD.length)); 6203 assertTrue(hasNext); 6204 currRow.clear(); 6205 hasNext = scanner.next(currRow); 6206 assertEquals(1, currRow.size()); 6207 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6208 currRow.get(0).getRowLength(), rowC, 0, rowC.length)); 6209 assertTrue(hasNext); 6210 currRow.clear(); 6211 hasNext = scanner.next(currRow); 6212 assertEquals(1, currRow.size()); 6213 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6214 currRow.get(0).getRowLength(), rowB, 0, rowB.length)); 6215 assertFalse(hasNext); 6216 } 6217 6218 scan = new Scan().withStartRow(rowD).withStopRow(rowA); 6219 scan.addColumn(families[0], col2); 6220 scan.setReversed(true); 6221 currRow.clear(); 6222 try (InternalScanner scanner = region.getScanner(scan)) { 6223 boolean hasNext = scanner.next(currRow); 6224 assertEquals(1, currRow.size()); 6225 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6226 currRow.get(0).getRowLength(), rowD, 0, rowD.length)); 6227 assertTrue(hasNext); 6228 } 6229 } 6230 6231 @Test 6232 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1() throws IOException { 6233 byte[] row0 = Bytes.toBytes("row0"); // 1 kv 6234 byte[] row1 = Bytes.toBytes("row1"); // 2 kv 6235 byte[] row2 = Bytes.toBytes("row2"); // 4 kv 6236 byte[] row3 = Bytes.toBytes("row3"); // 2 kv 6237 byte[] row4 = Bytes.toBytes("row4"); // 5 kv 6238 byte[] row5 = Bytes.toBytes("row5"); // 2 kv 6239 byte[] cf1 = Bytes.toBytes("CF1"); 6240 byte[] cf2 = Bytes.toBytes("CF2"); 6241 byte[] cf3 = Bytes.toBytes("CF3"); 6242 byte[][] families = { cf1, cf2, cf3 }; 6243 byte[] col = Bytes.toBytes("C"); 6244 long ts = 1; 6245 Configuration conf = new Configuration(CONF); 6246 // disable compactions in this test. 6247 conf.setInt("hbase.hstore.compactionThreshold", 10000); 6248 this.region = initHRegion(tableName, method, conf, families); 6249 // kv naming style: kv(row number) totalKvCountInThisRow seq no 6250 KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put, null); 6251 KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put, null); 6252 KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1, KeyValue.Type.Put, null); 6253 KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null); 6254 KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put, null); 6255 KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put, null); 6256 KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4, KeyValue.Type.Put, null); 6257 KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put, null); 6258 KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4, KeyValue.Type.Put, null); 6259 KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put, null); 6260 KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put, null); 6261 KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5, KeyValue.Type.Put, null); 6262 KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put, null); 6263 KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3, KeyValue.Type.Put, null); 6264 KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put, null); 6265 KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put, null); 6266 // hfiles(cf1/cf2) :"row1"(1 kv) / "row2"(1 kv) / "row4"(2 kv) 6267 Put put = null; 6268 put = new Put(row1); 6269 put.add(kv1_2_1); 6270 region.put(put); 6271 put = new Put(row2); 6272 put.add(kv2_4_1); 6273 region.put(put); 6274 put = new Put(row4); 6275 put.add(kv4_5_4); 6276 put.add(kv4_5_5); 6277 region.put(put); 6278 region.flush(true); 6279 // hfiles(cf1/cf3) : "row1" (1 kvs) / "row2" (1 kv) / "row4" (2 kv) 6280 put = new Put(row4); 6281 put.add(kv4_5_1); 6282 put.add(kv4_5_3); 6283 region.put(put); 6284 put = new Put(row1); 6285 put.add(kv1_2_2); 6286 region.put(put); 6287 put = new Put(row2); 6288 put.add(kv2_4_4); 6289 region.put(put); 6290 region.flush(true); 6291 // hfiles(cf1/cf3) : "row2"(2 kv) / "row3"(1 kvs) / "row4" (1 kv) 6292 put = new Put(row4); 6293 put.add(kv4_5_2); 6294 region.put(put); 6295 put = new Put(row2); 6296 put.add(kv2_4_2); 6297 put.add(kv2_4_3); 6298 region.put(put); 6299 put = new Put(row3); 6300 put.add(kv3_2_2); 6301 region.put(put); 6302 region.flush(true); 6303 // memstore(cf1/cf2/cf3) : "row0" (1 kvs) / "row3" ( 1 kv) / "row5" (max) 6304 // ( 2 kv) 6305 put = new Put(row0); 6306 put.add(kv0_1_1); 6307 region.put(put); 6308 put = new Put(row3); 6309 put.add(kv3_2_1); 6310 region.put(put); 6311 put = new Put(row5); 6312 put.add(kv5_2_1); 6313 put.add(kv5_2_2); 6314 region.put(put); 6315 // scan range = ["row4", min), skip the max "row5" 6316 Scan scan = new Scan().withStartRow(row4); 6317 scan.readVersions(5); 6318 scan.setBatch(3); 6319 scan.setReversed(true); 6320 try (InternalScanner scanner = region.getScanner(scan)) { 6321 List<Cell> currRow = new ArrayList<>(); 6322 boolean hasNext = false; 6323 // 1. scan out "row4" (5 kvs), "row5" can't be scanned out since not 6324 // included in scan range 6325 // "row4" takes 2 next() calls since batch=3 6326 hasNext = scanner.next(currRow); 6327 assertEquals(3, currRow.size()); 6328 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6329 currRow.get(0).getRowLength(), row4, 0, row4.length)); 6330 assertTrue(hasNext); 6331 currRow.clear(); 6332 hasNext = scanner.next(currRow); 6333 assertEquals(2, currRow.size()); 6334 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6335 currRow.get(0).getRowLength(), row4, 0, row4.length)); 6336 assertTrue(hasNext); 6337 // 2. scan out "row3" (2 kv) 6338 currRow.clear(); 6339 hasNext = scanner.next(currRow); 6340 assertEquals(2, currRow.size()); 6341 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6342 currRow.get(0).getRowLength(), row3, 0, row3.length)); 6343 assertTrue(hasNext); 6344 // 3. scan out "row2" (4 kvs) 6345 // "row2" takes 2 next() calls since batch=3 6346 currRow.clear(); 6347 hasNext = scanner.next(currRow); 6348 assertEquals(3, currRow.size()); 6349 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6350 currRow.get(0).getRowLength(), row2, 0, row2.length)); 6351 assertTrue(hasNext); 6352 currRow.clear(); 6353 hasNext = scanner.next(currRow); 6354 assertEquals(1, currRow.size()); 6355 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6356 currRow.get(0).getRowLength(), row2, 0, row2.length)); 6357 assertTrue(hasNext); 6358 // 4. scan out "row1" (2 kv) 6359 currRow.clear(); 6360 hasNext = scanner.next(currRow); 6361 assertEquals(2, currRow.size()); 6362 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6363 currRow.get(0).getRowLength(), row1, 0, row1.length)); 6364 assertTrue(hasNext); 6365 // 5. scan out "row0" (1 kv) 6366 currRow.clear(); 6367 hasNext = scanner.next(currRow); 6368 assertEquals(1, currRow.size()); 6369 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6370 currRow.get(0).getRowLength(), row0, 0, row0.length)); 6371 assertFalse(hasNext); 6372 } 6373 } 6374 6375 @Test 6376 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2() throws IOException { 6377 byte[] row1 = Bytes.toBytes("row1"); 6378 byte[] row2 = Bytes.toBytes("row2"); 6379 byte[] row3 = Bytes.toBytes("row3"); 6380 byte[] row4 = Bytes.toBytes("row4"); 6381 byte[] cf1 = Bytes.toBytes("CF1"); 6382 byte[] cf2 = Bytes.toBytes("CF2"); 6383 byte[] cf3 = Bytes.toBytes("CF3"); 6384 byte[] cf4 = Bytes.toBytes("CF4"); 6385 byte[][] families = { cf1, cf2, cf3, cf4 }; 6386 byte[] col = Bytes.toBytes("C"); 6387 long ts = 1; 6388 Configuration conf = new Configuration(CONF); 6389 // disable compactions in this test. 6390 conf.setInt("hbase.hstore.compactionThreshold", 10000); 6391 this.region = initHRegion(tableName, method, conf, families); 6392 KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null); 6393 KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null); 6394 KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null); 6395 KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null); 6396 // storefile1 6397 Put put = new Put(row1); 6398 put.add(kv1); 6399 region.put(put); 6400 region.flush(true); 6401 // storefile2 6402 put = new Put(row2); 6403 put.add(kv2); 6404 region.put(put); 6405 region.flush(true); 6406 // storefile3 6407 put = new Put(row3); 6408 put.add(kv3); 6409 region.put(put); 6410 region.flush(true); 6411 // memstore 6412 put = new Put(row4); 6413 put.add(kv4); 6414 region.put(put); 6415 // scan range = ["row4", min) 6416 Scan scan = new Scan().withStartRow(row4); 6417 scan.setReversed(true); 6418 scan.setBatch(10); 6419 try (InternalScanner scanner = region.getScanner(scan)) { 6420 List<Cell> currRow = new ArrayList<>(); 6421 boolean hasNext = scanner.next(currRow); 6422 assertEquals(1, currRow.size()); 6423 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6424 currRow.get(0).getRowLength(), row4, 0, row4.length)); 6425 assertTrue(hasNext); 6426 currRow.clear(); 6427 hasNext = scanner.next(currRow); 6428 assertEquals(1, currRow.size()); 6429 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6430 currRow.get(0).getRowLength(), row3, 0, row3.length)); 6431 assertTrue(hasNext); 6432 currRow.clear(); 6433 hasNext = scanner.next(currRow); 6434 assertEquals(1, currRow.size()); 6435 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6436 currRow.get(0).getRowLength(), row2, 0, row2.length)); 6437 assertTrue(hasNext); 6438 currRow.clear(); 6439 hasNext = scanner.next(currRow); 6440 assertEquals(1, currRow.size()); 6441 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6442 currRow.get(0).getRowLength(), row1, 0, row1.length)); 6443 assertFalse(hasNext); 6444 } 6445 } 6446 6447 /** 6448 * Test for HBASE-14497: Reverse Scan threw StackOverflow caused by readPt checking 6449 */ 6450 @Test 6451 public void testReverseScanner_StackOverflow() throws IOException { 6452 byte[] cf1 = Bytes.toBytes("CF1"); 6453 byte[][] families = { cf1 }; 6454 byte[] col = Bytes.toBytes("C"); 6455 Configuration conf = new Configuration(CONF); 6456 this.region = initHRegion(tableName, method, conf, families); 6457 // setup with one storefile and one memstore, to create scanner and get an earlier readPt 6458 Put put = new Put(Bytes.toBytes("19998")); 6459 put.addColumn(cf1, col, Bytes.toBytes("val")); 6460 region.put(put); 6461 region.flushcache(true, true, FlushLifeCycleTracker.DUMMY); 6462 Put put2 = new Put(Bytes.toBytes("19997")); 6463 put2.addColumn(cf1, col, Bytes.toBytes("val")); 6464 region.put(put2); 6465 6466 Scan scan = new Scan().withStartRow(Bytes.toBytes("19998")); 6467 scan.setReversed(true); 6468 try (InternalScanner scanner = region.getScanner(scan)) { 6469 // create one storefile contains many rows will be skipped 6470 // to check StoreFileScanner.seekToPreviousRow 6471 for (int i = 10000; i < 20000; i++) { 6472 Put p = new Put(Bytes.toBytes("" + i)); 6473 p.addColumn(cf1, col, Bytes.toBytes("" + i)); 6474 region.put(p); 6475 } 6476 region.flushcache(true, true, FlushLifeCycleTracker.DUMMY); 6477 6478 // create one memstore contains many rows will be skipped 6479 // to check MemStoreScanner.seekToPreviousRow 6480 for (int i = 10000; i < 20000; i++) { 6481 Put p = new Put(Bytes.toBytes("" + i)); 6482 p.addColumn(cf1, col, Bytes.toBytes("" + i)); 6483 region.put(p); 6484 } 6485 6486 List<Cell> currRow = new ArrayList<>(); 6487 boolean hasNext; 6488 do { 6489 hasNext = scanner.next(currRow); 6490 } while (hasNext); 6491 assertEquals(2, currRow.size()); 6492 assertEquals("19998", Bytes.toString(currRow.get(0).getRowArray(), 6493 currRow.get(0).getRowOffset(), currRow.get(0).getRowLength())); 6494 assertEquals("19997", Bytes.toString(currRow.get(1).getRowArray(), 6495 currRow.get(1).getRowOffset(), currRow.get(1).getRowLength())); 6496 } 6497 } 6498 6499 @Test 6500 public void testReverseScanShouldNotScanMemstoreIfReadPtLesser() throws Exception { 6501 byte[] cf1 = Bytes.toBytes("CF1"); 6502 byte[][] families = { cf1 }; 6503 byte[] col = Bytes.toBytes("C"); 6504 this.region = initHRegion(tableName, method, CONF, families); 6505 // setup with one storefile and one memstore, to create scanner and get an earlier readPt 6506 Put put = new Put(Bytes.toBytes("19996")); 6507 put.addColumn(cf1, col, Bytes.toBytes("val")); 6508 region.put(put); 6509 Put put2 = new Put(Bytes.toBytes("19995")); 6510 put2.addColumn(cf1, col, Bytes.toBytes("val")); 6511 region.put(put2); 6512 // create a reverse scan 6513 Scan scan = new Scan().withStartRow(Bytes.toBytes("19996")); 6514 scan.setReversed(true); 6515 try (RegionScannerImpl scanner = region.getScanner(scan)) { 6516 // flush the cache. This will reset the store scanner 6517 region.flushcache(true, true, FlushLifeCycleTracker.DUMMY); 6518 6519 // create one memstore contains many rows will be skipped 6520 // to check MemStoreScanner.seekToPreviousRow 6521 for (int i = 10000; i < 20000; i++) { 6522 Put p = new Put(Bytes.toBytes("" + i)); 6523 p.addColumn(cf1, col, Bytes.toBytes("" + i)); 6524 region.put(p); 6525 } 6526 List<Cell> currRow = new ArrayList<>(); 6527 boolean hasNext; 6528 boolean assertDone = false; 6529 do { 6530 hasNext = scanner.next(currRow); 6531 // With HBASE-15871, after the scanner is reset the memstore scanner should not be 6532 // added here 6533 if (!assertDone) { 6534 StoreScanner current = (StoreScanner) (scanner.storeHeap).getCurrentForTesting(); 6535 List<KeyValueScanner> scanners = current.getAllScannersForTesting(); 6536 assertEquals("There should be only one scanner the store file scanner", 1, 6537 scanners.size()); 6538 assertDone = true; 6539 } 6540 } while (hasNext); 6541 assertEquals(2, currRow.size()); 6542 assertEquals("19996", Bytes.toString(currRow.get(0).getRowArray(), 6543 currRow.get(0).getRowOffset(), currRow.get(0).getRowLength())); 6544 assertEquals("19995", Bytes.toString(currRow.get(1).getRowArray(), 6545 currRow.get(1).getRowOffset(), currRow.get(1).getRowLength())); 6546 } 6547 } 6548 6549 @Test 6550 public void testReverseScanWhenPutCellsAfterOpenReverseScan() throws Exception { 6551 byte[] cf1 = Bytes.toBytes("CF1"); 6552 byte[][] families = { cf1 }; 6553 byte[] col = Bytes.toBytes("C"); 6554 6555 this.region = initHRegion(tableName, method, CONF, families); 6556 6557 Put put = new Put(Bytes.toBytes("199996")); 6558 put.addColumn(cf1, col, Bytes.toBytes("val")); 6559 region.put(put); 6560 Put put2 = new Put(Bytes.toBytes("199995")); 6561 put2.addColumn(cf1, col, Bytes.toBytes("val")); 6562 region.put(put2); 6563 6564 // Create a reverse scan 6565 Scan scan = new Scan().withStartRow(Bytes.toBytes("199996")); 6566 scan.setReversed(true); 6567 try (RegionScannerImpl scanner = region.getScanner(scan)) { 6568 // Put a lot of cells that have sequenceIDs grater than the readPt of the reverse scan 6569 for (int i = 100000; i < 200000; i++) { 6570 Put p = new Put(Bytes.toBytes("" + i)); 6571 p.addColumn(cf1, col, Bytes.toBytes("" + i)); 6572 region.put(p); 6573 } 6574 List<Cell> currRow = new ArrayList<>(); 6575 boolean hasNext; 6576 do { 6577 hasNext = scanner.next(currRow); 6578 } while (hasNext); 6579 6580 assertEquals(2, currRow.size()); 6581 assertEquals("199996", Bytes.toString(currRow.get(0).getRowArray(), 6582 currRow.get(0).getRowOffset(), currRow.get(0).getRowLength())); 6583 assertEquals("199995", Bytes.toString(currRow.get(1).getRowArray(), 6584 currRow.get(1).getRowOffset(), currRow.get(1).getRowLength())); 6585 } 6586 } 6587 6588 @Test 6589 public void testWriteRequestsCounter() throws IOException { 6590 byte[] fam = Bytes.toBytes("info"); 6591 byte[][] families = { fam }; 6592 this.region = initHRegion(tableName, method, CONF, families); 6593 6594 Assert.assertEquals(0L, region.getWriteRequestsCount()); 6595 6596 Put put = new Put(row); 6597 put.addColumn(fam, fam, fam); 6598 6599 Assert.assertEquals(0L, region.getWriteRequestsCount()); 6600 region.put(put); 6601 Assert.assertEquals(1L, region.getWriteRequestsCount()); 6602 region.put(put); 6603 Assert.assertEquals(2L, region.getWriteRequestsCount()); 6604 region.put(put); 6605 Assert.assertEquals(3L, region.getWriteRequestsCount()); 6606 6607 region.delete(new Delete(row)); 6608 Assert.assertEquals(4L, region.getWriteRequestsCount()); 6609 } 6610 6611 @Test 6612 public void testOpenRegionWrittenToWAL() throws Exception { 6613 final ServerName serverName = ServerName.valueOf(name.getMethodName(), 100, 42); 6614 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName)); 6615 6616 TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) 6617 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)) 6618 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam2)).build(); 6619 RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build(); 6620 6621 // open the region w/o rss and wal and flush some files 6622 region = HBaseTestingUtil.createRegionAndWAL(hri, TEST_UTIL.getDataTestDir(), 6623 TEST_UTIL.getConfiguration(), htd); 6624 assertNotNull(region); 6625 6626 // create a file in fam1 for the region before opening in OpenRegionHandler 6627 region.put(new Put(Bytes.toBytes("a")).addColumn(fam1, fam1, fam1)); 6628 region.flush(true); 6629 HBaseTestingUtil.closeRegionAndWAL(region); 6630 6631 ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class); 6632 6633 // capture append() calls 6634 WAL wal = mockWAL(); 6635 when(rss.getWAL(any(RegionInfo.class))).thenReturn(wal); 6636 6637 region = 6638 HRegion.openHRegion(hri, htd, rss.getWAL(hri), TEST_UTIL.getConfiguration(), rss, null); 6639 6640 verify(wal, times(1)).appendMarker(any(RegionInfo.class), any(WALKeyImpl.class), 6641 editCaptor.capture()); 6642 6643 WALEdit edit = editCaptor.getValue(); 6644 assertNotNull(edit); 6645 assertNotNull(edit.getCells()); 6646 assertEquals(1, edit.getCells().size()); 6647 RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0)); 6648 assertNotNull(desc); 6649 6650 LOG.info("RegionEventDescriptor from WAL: " + desc); 6651 6652 assertEquals(RegionEventDescriptor.EventType.REGION_OPEN, desc.getEventType()); 6653 assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getTableName().toBytes())); 6654 assertTrue( 6655 Bytes.equals(desc.getEncodedRegionName().toByteArray(), hri.getEncodedNameAsBytes())); 6656 assertTrue(desc.getLogSequenceNumber() > 0); 6657 assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer())); 6658 assertEquals(2, desc.getStoresCount()); 6659 6660 StoreDescriptor store = desc.getStores(0); 6661 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1)); 6662 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1)); 6663 assertEquals(1, store.getStoreFileCount()); // 1store file 6664 assertFalse(store.getStoreFile(0).contains("/")); // ensure path is relative 6665 6666 store = desc.getStores(1); 6667 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2)); 6668 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2)); 6669 assertEquals(0, store.getStoreFileCount()); // no store files 6670 } 6671 6672 // Helper for test testOpenRegionWrittenToWALForLogReplay 6673 static class HRegionWithSeqId extends HRegion { 6674 public HRegionWithSeqId(final Path tableDir, final WAL wal, final FileSystem fs, 6675 final Configuration confParam, final RegionInfo regionInfo, final TableDescriptor htd, 6676 final RegionServerServices rsServices) { 6677 super(tableDir, wal, fs, confParam, regionInfo, htd, rsServices); 6678 } 6679 6680 @Override 6681 protected long getNextSequenceId(WAL wal) throws IOException { 6682 return 42; 6683 } 6684 } 6685 6686 @Test 6687 public void testFlushedFileWithNoTags() throws Exception { 6688 final TableName tableName = TableName.valueOf(name.getMethodName()); 6689 TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName) 6690 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)).build(); 6691 RegionInfo info = RegionInfoBuilder.newBuilder(tableName).build(); 6692 Path path = TEST_UTIL.getDataTestDir(getClass().getSimpleName()); 6693 region = HBaseTestingUtil.createRegionAndWAL(info, path, TEST_UTIL.getConfiguration(), 6694 tableDescriptor); 6695 Put put = new Put(Bytes.toBytes("a-b-0-0")); 6696 put.addColumn(fam1, qual1, Bytes.toBytes("c1-value")); 6697 region.put(put); 6698 region.flush(true); 6699 HStore store = region.getStore(fam1); 6700 Collection<HStoreFile> storefiles = store.getStorefiles(); 6701 for (HStoreFile sf : storefiles) { 6702 assertFalse("Tags should not be present ", 6703 sf.getReader().getHFileReader().getFileContext().isIncludesTags()); 6704 } 6705 } 6706 6707 /** 6708 * Utility method to setup a WAL mock. 6709 * <p/> 6710 * Needs to do the bit where we close latch on the WALKeyImpl on append else test hangs. 6711 * @return a mock WAL 6712 */ 6713 private WAL mockWAL() throws IOException { 6714 WAL wal = mock(WAL.class); 6715 when(wal.appendData(any(RegionInfo.class), any(WALKeyImpl.class), any(WALEdit.class))) 6716 .thenAnswer(new Answer<Long>() { 6717 @Override 6718 public Long answer(InvocationOnMock invocation) throws Throwable { 6719 WALKeyImpl key = invocation.getArgument(1); 6720 MultiVersionConcurrencyControl.WriteEntry we = key.getMvcc().begin(); 6721 key.setWriteEntry(we); 6722 return 1L; 6723 } 6724 }); 6725 when(wal.appendMarker(any(RegionInfo.class), any(WALKeyImpl.class), any(WALEdit.class))) 6726 .thenAnswer(new Answer<Long>() { 6727 @Override 6728 public Long answer(InvocationOnMock invocation) throws Throwable { 6729 WALKeyImpl key = invocation.getArgument(1); 6730 MultiVersionConcurrencyControl.WriteEntry we = key.getMvcc().begin(); 6731 key.setWriteEntry(we); 6732 return 1L; 6733 } 6734 }); 6735 return wal; 6736 } 6737 6738 @Test 6739 public void testCloseRegionWrittenToWAL() throws Exception { 6740 Path rootDir = new Path(dir + name.getMethodName()); 6741 CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir); 6742 6743 final ServerName serverName = ServerName.valueOf("testCloseRegionWrittenToWAL", 100, 42); 6744 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName)); 6745 6746 TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) 6747 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)) 6748 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam2)).build(); 6749 RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build(); 6750 6751 ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class); 6752 6753 // capture append() calls 6754 WAL wal = mockWAL(); 6755 when(rss.getWAL(any(RegionInfo.class))).thenReturn(wal); 6756 6757 // create the region 6758 region = HBaseTestingUtil.createRegionAndWAL(hri, rootDir, CONF, htd); 6759 HBaseTestingUtil.closeRegionAndWAL(region); 6760 region = null; 6761 // open the region first and then close it 6762 HRegion.openHRegion(hri, htd, rss.getWAL(hri), TEST_UTIL.getConfiguration(), rss, null).close(); 6763 6764 // 2 times, one for region open, the other close region 6765 verify(wal, times(2)).appendMarker(any(RegionInfo.class), (WALKeyImpl) any(WALKeyImpl.class), 6766 editCaptor.capture()); 6767 6768 WALEdit edit = editCaptor.getAllValues().get(1); 6769 assertNotNull(edit); 6770 assertNotNull(edit.getCells()); 6771 assertEquals(1, edit.getCells().size()); 6772 RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0)); 6773 assertNotNull(desc); 6774 6775 LOG.info("RegionEventDescriptor from WAL: " + desc); 6776 6777 assertEquals(RegionEventDescriptor.EventType.REGION_CLOSE, desc.getEventType()); 6778 assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getTableName().toBytes())); 6779 assertTrue( 6780 Bytes.equals(desc.getEncodedRegionName().toByteArray(), hri.getEncodedNameAsBytes())); 6781 assertTrue(desc.getLogSequenceNumber() > 0); 6782 assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer())); 6783 assertEquals(2, desc.getStoresCount()); 6784 6785 StoreDescriptor store = desc.getStores(0); 6786 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1)); 6787 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1)); 6788 assertEquals(0, store.getStoreFileCount()); // no store files 6789 6790 store = desc.getStores(1); 6791 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2)); 6792 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2)); 6793 assertEquals(0, store.getStoreFileCount()); // no store files 6794 } 6795 6796 /** 6797 * Test RegionTooBusyException thrown when region is busy 6798 */ 6799 @Test 6800 public void testRegionTooBusy() throws IOException { 6801 byte[] family = Bytes.toBytes("family"); 6802 long defaultBusyWaitDuration = 6803 CONF.getLong("hbase.busy.wait.duration", HRegion.DEFAULT_BUSY_WAIT_DURATION); 6804 CONF.setLong("hbase.busy.wait.duration", 1000); 6805 region = initHRegion(tableName, method, CONF, family); 6806 final AtomicBoolean stopped = new AtomicBoolean(true); 6807 Thread t = new Thread(new Runnable() { 6808 @Override 6809 public void run() { 6810 try { 6811 region.lock.writeLock().lock(); 6812 stopped.set(false); 6813 while (!stopped.get()) { 6814 Thread.sleep(100); 6815 } 6816 } catch (InterruptedException ie) { 6817 } finally { 6818 region.lock.writeLock().unlock(); 6819 } 6820 } 6821 }); 6822 t.start(); 6823 Get get = new Get(row); 6824 try { 6825 while (stopped.get()) { 6826 Thread.sleep(100); 6827 } 6828 region.get(get); 6829 fail("Should throw RegionTooBusyException"); 6830 } catch (InterruptedException ie) { 6831 fail("test interrupted"); 6832 } catch (RegionTooBusyException e) { 6833 // Good, expected 6834 } finally { 6835 stopped.set(true); 6836 try { 6837 t.join(); 6838 } catch (Throwable e) { 6839 } 6840 6841 HBaseTestingUtil.closeRegionAndWAL(region); 6842 region = null; 6843 CONF.setLong("hbase.busy.wait.duration", defaultBusyWaitDuration); 6844 } 6845 } 6846 6847 @Test 6848 public void testCellTTLs() throws IOException { 6849 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge(); 6850 EnvironmentEdgeManager.injectEdge(edge); 6851 6852 final byte[] row = Bytes.toBytes("testRow"); 6853 final byte[] q1 = Bytes.toBytes("q1"); 6854 final byte[] q2 = Bytes.toBytes("q2"); 6855 final byte[] q3 = Bytes.toBytes("q3"); 6856 final byte[] q4 = Bytes.toBytes("q4"); 6857 6858 // 10 seconds 6859 TableDescriptor tableDescriptor = 6860 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) 6861 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam1).setTimeToLive(10).build()) 6862 .build(); 6863 6864 Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); 6865 conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS); 6866 6867 region = HBaseTestingUtil.createRegionAndWAL( 6868 RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(), 6869 TEST_UTIL.getDataTestDir(), conf, tableDescriptor); 6870 assertNotNull(region); 6871 long now = EnvironmentEdgeManager.currentTime(); 6872 // Add a cell that will expire in 5 seconds via cell TTL 6873 region.put(new Put(row).add(new KeyValue(row, fam1, q1, now, HConstants.EMPTY_BYTE_ARRAY, 6874 new ArrayBackedTag[] { 6875 // TTL tags specify ts in milliseconds 6876 new ArrayBackedTag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) }))); 6877 // Add a cell that will expire after 10 seconds via family setting 6878 region.put(new Put(row).addColumn(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY)); 6879 // Add a cell that will expire in 15 seconds via cell TTL 6880 region.put(new Put(row).add(new KeyValue(row, fam1, q3, now + 10000 - 1, 6881 HConstants.EMPTY_BYTE_ARRAY, new ArrayBackedTag[] { 6882 // TTL tags specify ts in milliseconds 6883 new ArrayBackedTag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) }))); 6884 // Add a cell that will expire in 20 seconds via family setting 6885 region.put(new Put(row).addColumn(fam1, q4, now + 10000 - 1, HConstants.EMPTY_BYTE_ARRAY)); 6886 6887 // Flush so we are sure store scanning gets this right 6888 region.flush(true); 6889 6890 // A query at time T+0 should return all cells 6891 Result r = region.get(new Get(row)); 6892 assertNotNull(r.getValue(fam1, q1)); 6893 assertNotNull(r.getValue(fam1, q2)); 6894 assertNotNull(r.getValue(fam1, q3)); 6895 assertNotNull(r.getValue(fam1, q4)); 6896 6897 // Increment time to T+5 seconds 6898 edge.incrementTime(5000); 6899 6900 r = region.get(new Get(row)); 6901 assertNull(r.getValue(fam1, q1)); 6902 assertNotNull(r.getValue(fam1, q2)); 6903 assertNotNull(r.getValue(fam1, q3)); 6904 assertNotNull(r.getValue(fam1, q4)); 6905 6906 // Increment time to T+10 seconds 6907 edge.incrementTime(5000); 6908 6909 r = region.get(new Get(row)); 6910 assertNull(r.getValue(fam1, q1)); 6911 assertNull(r.getValue(fam1, q2)); 6912 assertNotNull(r.getValue(fam1, q3)); 6913 assertNotNull(r.getValue(fam1, q4)); 6914 6915 // Increment time to T+15 seconds 6916 edge.incrementTime(5000); 6917 6918 r = region.get(new Get(row)); 6919 assertNull(r.getValue(fam1, q1)); 6920 assertNull(r.getValue(fam1, q2)); 6921 assertNull(r.getValue(fam1, q3)); 6922 assertNotNull(r.getValue(fam1, q4)); 6923 6924 // Increment time to T+20 seconds 6925 edge.incrementTime(10000); 6926 6927 r = region.get(new Get(row)); 6928 assertNull(r.getValue(fam1, q1)); 6929 assertNull(r.getValue(fam1, q2)); 6930 assertNull(r.getValue(fam1, q3)); 6931 assertNull(r.getValue(fam1, q4)); 6932 6933 // Fun with disappearing increments 6934 6935 // Start at 1 6936 region.put(new Put(row).addColumn(fam1, q1, Bytes.toBytes(1L))); 6937 r = region.get(new Get(row)); 6938 byte[] val = r.getValue(fam1, q1); 6939 assertNotNull(val); 6940 assertEquals(1L, Bytes.toLong(val)); 6941 6942 // Increment with a TTL of 5 seconds 6943 Increment incr = new Increment(row).addColumn(fam1, q1, 1L); 6944 incr.setTTL(5000); 6945 region.increment(incr); // 2 6946 6947 // New value should be 2 6948 r = region.get(new Get(row)); 6949 val = r.getValue(fam1, q1); 6950 assertNotNull(val); 6951 assertEquals(2L, Bytes.toLong(val)); 6952 6953 // Increment time to T+25 seconds 6954 edge.incrementTime(5000); 6955 6956 // Value should be back to 1 6957 r = region.get(new Get(row)); 6958 val = r.getValue(fam1, q1); 6959 assertNotNull(val); 6960 assertEquals(1L, Bytes.toLong(val)); 6961 6962 // Increment time to T+30 seconds 6963 edge.incrementTime(5000); 6964 6965 // Original value written at T+20 should be gone now via family TTL 6966 r = region.get(new Get(row)); 6967 assertNull(r.getValue(fam1, q1)); 6968 } 6969 6970 @Test 6971 public void testTTLsUsingSmallHeartBeatCells() throws IOException { 6972 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge(); 6973 EnvironmentEdgeManager.injectEdge(edge); 6974 6975 final byte[] row = Bytes.toBytes("testRow"); 6976 final byte[] q1 = Bytes.toBytes("q1"); 6977 final byte[] q2 = Bytes.toBytes("q2"); 6978 final byte[] q3 = Bytes.toBytes("q3"); 6979 final byte[] q4 = Bytes.toBytes("q4"); 6980 final byte[] q5 = Bytes.toBytes("q5"); 6981 final byte[] q6 = Bytes.toBytes("q6"); 6982 final byte[] q7 = Bytes.toBytes("q7"); 6983 final byte[] q8 = Bytes.toBytes("q8"); 6984 6985 // 10 seconds 6986 int ttlSecs = 10; 6987 TableDescriptor tableDescriptor = 6988 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily( 6989 ColumnFamilyDescriptorBuilder.newBuilder(fam1).setTimeToLive(ttlSecs).build()).build(); 6990 6991 Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); 6992 conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS); 6993 // using small heart beat cells 6994 conf.setLong(StoreScanner.HBASE_CELLS_SCANNED_PER_HEARTBEAT_CHECK, 2); 6995 6996 region = HBaseTestingUtil.createRegionAndWAL( 6997 RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(), 6998 TEST_UTIL.getDataTestDir(), conf, tableDescriptor); 6999 assertNotNull(region); 7000 long now = EnvironmentEdgeManager.currentTime(); 7001 // Add a cell that will expire in 5 seconds via cell TTL 7002 region.put(new Put(row).addColumn(fam1, q1, now, HConstants.EMPTY_BYTE_ARRAY)); 7003 region.put(new Put(row).addColumn(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY)); 7004 region.put(new Put(row).addColumn(fam1, q3, now, HConstants.EMPTY_BYTE_ARRAY)); 7005 // Add a cell that will expire after 10 seconds via family setting 7006 region 7007 .put(new Put(row).addColumn(fam1, q4, now + ttlSecs * 1000 + 1, HConstants.EMPTY_BYTE_ARRAY)); 7008 region 7009 .put(new Put(row).addColumn(fam1, q5, now + ttlSecs * 1000 + 1, HConstants.EMPTY_BYTE_ARRAY)); 7010 7011 region.put(new Put(row).addColumn(fam1, q6, now, HConstants.EMPTY_BYTE_ARRAY)); 7012 region.put(new Put(row).addColumn(fam1, q7, now, HConstants.EMPTY_BYTE_ARRAY)); 7013 region 7014 .put(new Put(row).addColumn(fam1, q8, now + ttlSecs * 1000 + 1, HConstants.EMPTY_BYTE_ARRAY)); 7015 7016 // Flush so we are sure store scanning gets this right 7017 region.flush(true); 7018 7019 // A query at time T+0 should return all cells 7020 checkScan(8); 7021 region.delete(new Delete(row).addColumn(fam1, q8)); 7022 7023 // Increment time to T+ttlSecs seconds 7024 edge.incrementTime(ttlSecs * 1000); 7025 checkScan(2); 7026 } 7027 7028 private void checkScan(int expectCellSize) throws IOException { 7029 Scan s = new Scan().withStartRow(row); 7030 ScannerContext.Builder contextBuilder = ScannerContext.newBuilder(true); 7031 ScannerContext scannerContext = contextBuilder.build(); 7032 try (RegionScanner scanner = region.getScanner(s)) { 7033 List<Cell> kvs = new ArrayList<>(); 7034 scanner.next(kvs, scannerContext); 7035 assertEquals(expectCellSize, kvs.size()); 7036 } 7037 } 7038 7039 @Test 7040 public void testIncrementTimestampsAreMonotonic() throws IOException { 7041 region = initHRegion(tableName, method, CONF, fam1); 7042 ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); 7043 EnvironmentEdgeManager.injectEdge(edge); 7044 7045 edge.setValue(10); 7046 Increment inc = new Increment(row); 7047 inc.setDurability(Durability.SKIP_WAL); 7048 inc.addColumn(fam1, qual1, 1L); 7049 region.increment(inc); 7050 7051 Result result = region.get(new Get(row)); 7052 Cell c = result.getColumnLatestCell(fam1, qual1); 7053 assertNotNull(c); 7054 assertEquals(10L, c.getTimestamp()); 7055 7056 edge.setValue(1); // clock goes back 7057 region.increment(inc); 7058 result = region.get(new Get(row)); 7059 c = result.getColumnLatestCell(fam1, qual1); 7060 assertEquals(11L, c.getTimestamp()); 7061 assertEquals(2L, Bytes.toLong(c.getValueArray(), c.getValueOffset(), c.getValueLength())); 7062 } 7063 7064 @Test 7065 public void testAppendTimestampsAreMonotonic() throws IOException { 7066 region = initHRegion(tableName, method, CONF, fam1); 7067 ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); 7068 EnvironmentEdgeManager.injectEdge(edge); 7069 7070 edge.setValue(10); 7071 Append a = new Append(row); 7072 a.setDurability(Durability.SKIP_WAL); 7073 a.addColumn(fam1, qual1, qual1); 7074 region.append(a); 7075 7076 Result result = region.get(new Get(row)); 7077 Cell c = result.getColumnLatestCell(fam1, qual1); 7078 assertNotNull(c); 7079 assertEquals(10L, c.getTimestamp()); 7080 7081 edge.setValue(1); // clock goes back 7082 region.append(a); 7083 result = region.get(new Get(row)); 7084 c = result.getColumnLatestCell(fam1, qual1); 7085 assertEquals(11L, c.getTimestamp()); 7086 7087 byte[] expected = new byte[qual1.length * 2]; 7088 System.arraycopy(qual1, 0, expected, 0, qual1.length); 7089 System.arraycopy(qual1, 0, expected, qual1.length, qual1.length); 7090 7091 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(), expected, 0, 7092 expected.length)); 7093 } 7094 7095 @Test 7096 public void testCheckAndMutateTimestampsAreMonotonic() throws IOException { 7097 region = initHRegion(tableName, method, CONF, fam1); 7098 ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); 7099 EnvironmentEdgeManager.injectEdge(edge); 7100 7101 edge.setValue(10); 7102 Put p = new Put(row); 7103 p.setDurability(Durability.SKIP_WAL); 7104 p.addColumn(fam1, qual1, qual1); 7105 region.put(p); 7106 7107 Result result = region.get(new Get(row)); 7108 Cell c = result.getColumnLatestCell(fam1, qual1); 7109 assertNotNull(c); 7110 assertEquals(10L, c.getTimestamp()); 7111 7112 edge.setValue(1); // clock goes back 7113 p = new Put(row); 7114 p.setDurability(Durability.SKIP_WAL); 7115 p.addColumn(fam1, qual1, qual2); 7116 region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(qual1), p); 7117 result = region.get(new Get(row)); 7118 c = result.getColumnLatestCell(fam1, qual1); 7119 assertEquals(10L, c.getTimestamp()); 7120 7121 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(), qual2, 0, 7122 qual2.length)); 7123 } 7124 7125 @Test 7126 public void testBatchMutateWithWrongRegionException() throws Exception { 7127 final byte[] a = Bytes.toBytes("a"); 7128 final byte[] b = Bytes.toBytes("b"); 7129 final byte[] c = Bytes.toBytes("c"); // exclusive 7130 7131 int prevLockTimeout = CONF.getInt("hbase.rowlock.wait.duration", 30000); 7132 CONF.setInt("hbase.rowlock.wait.duration", 1000); 7133 region = initHRegion(tableName, a, c, method, CONF, false, fam1); 7134 7135 Mutation[] mutations = new Mutation[] { 7136 new Put(a).add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a) 7137 .setFamily(fam1).setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), 7138 // this is outside the region boundary 7139 new Put(c).add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(c) 7140 .setFamily(fam1).setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Type.Put).build()), 7141 new Put(b) 7142 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1) 7143 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()) }; 7144 7145 OperationStatus[] status = region.batchMutate(mutations); 7146 assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode()); 7147 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, status[1].getOperationStatusCode()); 7148 assertEquals(OperationStatusCode.SUCCESS, status[2].getOperationStatusCode()); 7149 7150 // test with a row lock held for a long time 7151 final CountDownLatch obtainedRowLock = new CountDownLatch(1); 7152 ExecutorService exec = Executors.newFixedThreadPool(2); 7153 Future<Void> f1 = exec.submit(new Callable<Void>() { 7154 @Override 7155 public Void call() throws Exception { 7156 LOG.info("Acquiring row lock"); 7157 RowLock rl = region.getRowLock(b); 7158 obtainedRowLock.countDown(); 7159 LOG.info("Waiting for 5 seconds before releasing lock"); 7160 Threads.sleep(5000); 7161 LOG.info("Releasing row lock"); 7162 rl.release(); 7163 return null; 7164 } 7165 }); 7166 obtainedRowLock.await(30, TimeUnit.SECONDS); 7167 7168 Future<Void> f2 = exec.submit(new Callable<Void>() { 7169 @Override 7170 public Void call() throws Exception { 7171 Mutation[] mutations = new Mutation[] { 7172 new Put(a) 7173 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a).setFamily(fam1) 7174 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), 7175 new Put(b) 7176 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1) 7177 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), }; 7178 7179 // this will wait for the row lock, and it will eventually succeed 7180 OperationStatus[] status = region.batchMutate(mutations); 7181 assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode()); 7182 assertEquals(OperationStatusCode.SUCCESS, status[1].getOperationStatusCode()); 7183 return null; 7184 } 7185 }); 7186 7187 f1.get(); 7188 f2.get(); 7189 7190 CONF.setInt("hbase.rowlock.wait.duration", prevLockTimeout); 7191 } 7192 7193 @Test 7194 public void testBatchMutateWithZeroRowLockWait() throws Exception { 7195 final byte[] a = Bytes.toBytes("a"); 7196 final byte[] b = Bytes.toBytes("b"); 7197 final byte[] c = Bytes.toBytes("c"); // exclusive 7198 7199 Configuration conf = new Configuration(CONF); 7200 conf.setInt("hbase.rowlock.wait.duration", 0); 7201 final RegionInfo hri = 7202 RegionInfoBuilder.newBuilder(tableName).setStartKey(a).setEndKey(c).build(); 7203 final TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName) 7204 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)).build(); 7205 region = HRegion.createHRegion(hri, TEST_UTIL.getDataTestDir(), conf, htd, 7206 HBaseTestingUtil.createWal(conf, TEST_UTIL.getDataTestDirOnTestFS(method + ".log"), hri)); 7207 7208 Mutation[] mutations = new Mutation[] { 7209 new Put(a).add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a) 7210 .setFamily(fam1).setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), 7211 new Put(b) 7212 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1) 7213 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()) }; 7214 7215 OperationStatus[] status = region.batchMutate(mutations); 7216 assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode()); 7217 assertEquals(OperationStatusCode.SUCCESS, status[1].getOperationStatusCode()); 7218 7219 // test with a row lock held for a long time 7220 final CountDownLatch obtainedRowLock = new CountDownLatch(1); 7221 ExecutorService exec = Executors.newFixedThreadPool(2); 7222 Future<Void> f1 = exec.submit(new Callable<Void>() { 7223 @Override 7224 public Void call() throws Exception { 7225 LOG.info("Acquiring row lock"); 7226 RowLock rl = region.getRowLock(b); 7227 obtainedRowLock.countDown(); 7228 LOG.info("Waiting for 5 seconds before releasing lock"); 7229 Threads.sleep(5000); 7230 LOG.info("Releasing row lock"); 7231 rl.release(); 7232 return null; 7233 } 7234 }); 7235 obtainedRowLock.await(30, TimeUnit.SECONDS); 7236 7237 Future<Void> f2 = exec.submit(new Callable<Void>() { 7238 @Override 7239 public Void call() throws Exception { 7240 Mutation[] mutations = new Mutation[] { 7241 new Put(a) 7242 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a).setFamily(fam1) 7243 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), 7244 new Put(b) 7245 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1) 7246 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), }; 7247 // when handling row b we are going to spin on the failure to get the row lock 7248 // until the lock above is released, but we will still succeed so long as that 7249 // takes less time then the test time out. 7250 OperationStatus[] status = region.batchMutate(mutations); 7251 assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode()); 7252 assertEquals(OperationStatusCode.SUCCESS, status[1].getOperationStatusCode()); 7253 return null; 7254 } 7255 }); 7256 7257 f1.get(); 7258 f2.get(); 7259 } 7260 7261 @Test 7262 public void testCheckAndRowMutateTimestampsAreMonotonic() throws IOException { 7263 region = initHRegion(tableName, method, CONF, fam1); 7264 ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); 7265 EnvironmentEdgeManager.injectEdge(edge); 7266 7267 edge.setValue(10); 7268 Put p = new Put(row); 7269 p.setDurability(Durability.SKIP_WAL); 7270 p.addColumn(fam1, qual1, qual1); 7271 region.put(p); 7272 7273 Result result = region.get(new Get(row)); 7274 Cell c = result.getColumnLatestCell(fam1, qual1); 7275 assertNotNull(c); 7276 assertEquals(10L, c.getTimestamp()); 7277 7278 edge.setValue(1); // clock goes back 7279 p = new Put(row); 7280 p.setDurability(Durability.SKIP_WAL); 7281 p.addColumn(fam1, qual1, qual2); 7282 RowMutations rm = new RowMutations(row); 7283 rm.add(p); 7284 assertTrue(region.checkAndRowMutate(row, fam1, qual1, CompareOperator.EQUAL, 7285 new BinaryComparator(qual1), rm)); 7286 result = region.get(new Get(row)); 7287 c = result.getColumnLatestCell(fam1, qual1); 7288 assertEquals(10L, c.getTimestamp()); 7289 LOG.info( 7290 "c value " + Bytes.toStringBinary(c.getValueArray(), c.getValueOffset(), c.getValueLength())); 7291 7292 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(), qual2, 0, 7293 qual2.length)); 7294 } 7295 7296 private HRegion initHRegion(TableName tableName, String callingMethod, byte[]... families) 7297 throws IOException { 7298 return initHRegion(tableName, callingMethod, HBaseConfiguration.create(), families); 7299 } 7300 7301 /** 7302 * HBASE-16429 Make sure no stuck if roll writer when ring buffer is filled with appends 7303 * @throws IOException if IO error occurred during test 7304 */ 7305 @Test 7306 public void testWritesWhileRollWriter() throws IOException { 7307 int testCount = 10; 7308 int numRows = 1024; 7309 int numFamilies = 2; 7310 int numQualifiers = 2; 7311 final byte[][] families = new byte[numFamilies][]; 7312 for (int i = 0; i < numFamilies; i++) { 7313 families[i] = Bytes.toBytes("family" + i); 7314 } 7315 final byte[][] qualifiers = new byte[numQualifiers][]; 7316 for (int i = 0; i < numQualifiers; i++) { 7317 qualifiers[i] = Bytes.toBytes("qual" + i); 7318 } 7319 7320 CONF.setInt("hbase.regionserver.wal.disruptor.event.count", 2); 7321 this.region = initHRegion(tableName, method, CONF, families); 7322 try { 7323 List<Thread> threads = new ArrayList<>(); 7324 for (int i = 0; i < numRows; i++) { 7325 final int count = i; 7326 Thread t = new Thread(new Runnable() { 7327 7328 @Override 7329 public void run() { 7330 byte[] row = Bytes.toBytes("row" + count); 7331 Put put = new Put(row); 7332 put.setDurability(Durability.SYNC_WAL); 7333 byte[] value = Bytes.toBytes(String.valueOf(count)); 7334 for (byte[] family : families) { 7335 for (byte[] qualifier : qualifiers) { 7336 put.addColumn(family, qualifier, count, value); 7337 } 7338 } 7339 try { 7340 region.put(put); 7341 } catch (IOException e) { 7342 throw new RuntimeException(e); 7343 } 7344 } 7345 }); 7346 threads.add(t); 7347 } 7348 for (Thread t : threads) { 7349 t.start(); 7350 } 7351 7352 for (int i = 0; i < testCount; i++) { 7353 region.getWAL().rollWriter(); 7354 Thread.yield(); 7355 } 7356 } finally { 7357 try { 7358 HBaseTestingUtil.closeRegionAndWAL(this.region); 7359 CONF.setInt("hbase.regionserver.wal.disruptor.event.count", 16 * 1024); 7360 } catch (DroppedSnapshotException dse) { 7361 // We could get this on way out because we interrupt the background flusher and it could 7362 // fail anywhere causing a DSE over in the background flusher... only it is not properly 7363 // dealt with so could still be memory hanging out when we get to here -- memory we can't 7364 // flush because the accounting is 'off' since original DSE. 7365 } 7366 this.region = null; 7367 } 7368 } 7369 7370 @Test 7371 public void testMutateRow() throws Exception { 7372 final byte[] row = Bytes.toBytes("row"); 7373 final byte[] q1 = Bytes.toBytes("q1"); 7374 final byte[] q2 = Bytes.toBytes("q2"); 7375 final byte[] q3 = Bytes.toBytes("q3"); 7376 final byte[] q4 = Bytes.toBytes("q4"); 7377 final String v1 = "v1"; 7378 7379 region = initHRegion(tableName, method, CONF, fam1); 7380 7381 // Initial values 7382 region 7383 .batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q2, Bytes.toBytes("toBeDeleted")), 7384 new Put(row).addColumn(fam1, q3, Bytes.toBytes(5L)), 7385 new Put(row).addColumn(fam1, q4, Bytes.toBytes("a")), }); 7386 7387 // Do mutateRow 7388 Result result = region.mutateRow( 7389 new RowMutations(row).add(Arrays.asList(new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1)), 7390 new Delete(row).addColumns(fam1, q2), new Increment(row).addColumn(fam1, q3, 1), 7391 new Append(row).addColumn(fam1, q4, Bytes.toBytes("b"))))); 7392 7393 assertNotNull(result); 7394 assertEquals(6L, Bytes.toLong(result.getValue(fam1, q3))); 7395 assertEquals("ab", Bytes.toString(result.getValue(fam1, q4))); 7396 7397 // Verify the value 7398 result = region.get(new Get(row)); 7399 assertEquals(v1, Bytes.toString(result.getValue(fam1, q1))); 7400 assertNull(result.getValue(fam1, q2)); 7401 assertEquals(6L, Bytes.toLong(result.getValue(fam1, q3))); 7402 assertEquals("ab", Bytes.toString(result.getValue(fam1, q4))); 7403 } 7404 7405 @Test 7406 public void testMutateRowInParallel() throws Exception { 7407 final int numReaderThreads = 100; 7408 final CountDownLatch latch = new CountDownLatch(numReaderThreads); 7409 7410 final byte[] row = Bytes.toBytes("row"); 7411 final byte[] q1 = Bytes.toBytes("q1"); 7412 final byte[] q2 = Bytes.toBytes("q2"); 7413 final byte[] q3 = Bytes.toBytes("q3"); 7414 final byte[] q4 = Bytes.toBytes("q4"); 7415 final String v1 = "v1"; 7416 final String v2 = "v2"; 7417 7418 // We need to ensure the timestamp of the delete operation is more than the previous one 7419 final AtomicLong deleteTimestamp = new AtomicLong(); 7420 7421 region = initHRegion(tableName, method, CONF, fam1); 7422 7423 // Initial values 7424 region.batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1)) 7425 .addColumn(fam1, q2, deleteTimestamp.getAndIncrement(), Bytes.toBytes(v2)) 7426 .addColumn(fam1, q3, Bytes.toBytes(1L)).addColumn(fam1, q4, Bytes.toBytes("a")) }); 7427 7428 final AtomicReference<AssertionError> assertionError = new AtomicReference<>(); 7429 7430 // Writer thread 7431 Thread writerThread = new Thread(() -> { 7432 try { 7433 while (true) { 7434 // If all the reader threads finish, then stop the writer thread 7435 if (latch.await(0, TimeUnit.MILLISECONDS)) { 7436 return; 7437 } 7438 7439 // Execute the mutations. This should be done atomically 7440 region.mutateRow(new RowMutations(row) 7441 .add(Arrays.asList(new Put(row).addColumn(fam1, q1, Bytes.toBytes(v2)), 7442 new Delete(row).addColumns(fam1, q2, deleteTimestamp.getAndIncrement()), 7443 new Increment(row).addColumn(fam1, q3, 1L), 7444 new Append(row).addColumn(fam1, q4, Bytes.toBytes("b"))))); 7445 7446 // We need to ensure the timestamps of the Increment/Append operations are more than the 7447 // previous ones 7448 Result result = region.get(new Get(row).addColumn(fam1, q3).addColumn(fam1, q4)); 7449 long tsIncrement = result.getColumnLatestCell(fam1, q3).getTimestamp(); 7450 long tsAppend = result.getColumnLatestCell(fam1, q4).getTimestamp(); 7451 7452 // Put the initial values 7453 region.batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1)) 7454 .addColumn(fam1, q2, deleteTimestamp.getAndIncrement(), Bytes.toBytes(v2)) 7455 .addColumn(fam1, q3, tsIncrement + 1, Bytes.toBytes(1L)) 7456 .addColumn(fam1, q4, tsAppend + 1, Bytes.toBytes("a")) }); 7457 } 7458 } catch (Exception e) { 7459 assertionError.set(new AssertionError(e)); 7460 } 7461 }); 7462 writerThread.start(); 7463 7464 // Reader threads 7465 for (int i = 0; i < numReaderThreads; i++) { 7466 new Thread(() -> { 7467 try { 7468 for (int j = 0; j < 10000; j++) { 7469 // Verify the values 7470 Result result = region.get(new Get(row)); 7471 7472 // The values should be equals to either the initial values or the values after 7473 // executing the mutations 7474 String q1Value = Bytes.toString(result.getValue(fam1, q1)); 7475 if (v1.equals(q1Value)) { 7476 assertEquals(v2, Bytes.toString(result.getValue(fam1, q2))); 7477 assertEquals(1L, Bytes.toLong(result.getValue(fam1, q3))); 7478 assertEquals("a", Bytes.toString(result.getValue(fam1, q4))); 7479 } else if (v2.equals(q1Value)) { 7480 assertNull(Bytes.toString(result.getValue(fam1, q2))); 7481 assertEquals(2L, Bytes.toLong(result.getValue(fam1, q3))); 7482 assertEquals("ab", Bytes.toString(result.getValue(fam1, q4))); 7483 } else { 7484 fail("the qualifier " + Bytes.toString(q1) + " should be " + v1 + " or " + v2 7485 + ", but " + q1Value); 7486 } 7487 } 7488 } catch (Exception e) { 7489 assertionError.set(new AssertionError(e)); 7490 } catch (AssertionError e) { 7491 assertionError.set(e); 7492 } 7493 7494 latch.countDown(); 7495 }).start(); 7496 } 7497 7498 writerThread.join(); 7499 7500 if (assertionError.get() != null) { 7501 throw assertionError.get(); 7502 } 7503 } 7504 7505 @Test 7506 public void testMutateRow_WriteRequestCount() throws Exception { 7507 byte[] row1 = Bytes.toBytes("row1"); 7508 byte[] fam1 = Bytes.toBytes("fam1"); 7509 byte[] qf1 = Bytes.toBytes("qualifier"); 7510 byte[] val1 = Bytes.toBytes("value1"); 7511 7512 RowMutations rm = new RowMutations(row1); 7513 Put put = new Put(row1); 7514 put.addColumn(fam1, qf1, val1); 7515 rm.add(put); 7516 7517 this.region = initHRegion(tableName, method, CONF, fam1); 7518 long wrcBeforeMutate = this.region.writeRequestsCount.longValue(); 7519 this.region.mutateRow(rm); 7520 long wrcAfterMutate = this.region.writeRequestsCount.longValue(); 7521 Assert.assertEquals(wrcBeforeMutate + rm.getMutations().size(), wrcAfterMutate); 7522 } 7523 7524 @Test 7525 public void testBulkLoadReplicationEnabled() throws IOException { 7526 TEST_UTIL.getConfiguration().setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true); 7527 final ServerName serverName = ServerName.valueOf(name.getMethodName(), 100, 42); 7528 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName)); 7529 7530 TableDescriptor tableDescriptor = 7531 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) 7532 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)).build(); 7533 RegionInfo hri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 7534 region = HRegion.openHRegion(hri, tableDescriptor, rss.getWAL(hri), 7535 TEST_UTIL.getConfiguration(), rss, null); 7536 7537 assertTrue(region.conf.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, false)); 7538 String plugins = region.conf.get(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, ""); 7539 String replicationCoprocessorClass = ReplicationObserver.class.getCanonicalName(); 7540 assertTrue(plugins.contains(replicationCoprocessorClass)); 7541 assertTrue(region.getCoprocessorHost().getCoprocessors() 7542 .contains(ReplicationObserver.class.getSimpleName())); 7543 } 7544 7545 /** 7546 * The same as HRegion class, the only difference is that instantiateHStore will create a 7547 * different HStore - HStoreForTesting. [HBASE-8518] 7548 */ 7549 public static class HRegionForTesting extends HRegion { 7550 7551 public HRegionForTesting(final Path tableDir, final WAL wal, final FileSystem fs, 7552 final Configuration confParam, final RegionInfo regionInfo, final TableDescriptor htd, 7553 final RegionServerServices rsServices) { 7554 this(new HRegionFileSystem(confParam, fs, tableDir, regionInfo), wal, confParam, htd, 7555 rsServices); 7556 } 7557 7558 public HRegionForTesting(HRegionFileSystem fs, WAL wal, Configuration confParam, 7559 TableDescriptor htd, RegionServerServices rsServices) { 7560 super(fs, wal, confParam, htd, rsServices); 7561 } 7562 7563 /** 7564 * Create HStore instance. 7565 * @return If Mob is enabled, return HMobStore, otherwise return HStoreForTesting. 7566 */ 7567 @Override 7568 protected HStore instantiateHStore(final ColumnFamilyDescriptor family, boolean warmup) 7569 throws IOException { 7570 if (family.isMobEnabled()) { 7571 if (HFile.getFormatVersion(this.conf) < HFile.MIN_FORMAT_VERSION_WITH_TAGS) { 7572 throw new IOException("A minimum HFile version of " + HFile.MIN_FORMAT_VERSION_WITH_TAGS 7573 + " is required for MOB feature. Consider setting " + HFile.FORMAT_VERSION_KEY 7574 + " accordingly."); 7575 } 7576 return new HMobStore(this, family, this.conf, warmup); 7577 } 7578 return new HStoreForTesting(this, family, this.conf, warmup); 7579 } 7580 } 7581 7582 /** 7583 * HStoreForTesting is merely the same as HStore, the difference is in the doCompaction method of 7584 * HStoreForTesting there is a checkpoint "hbase.hstore.compaction.complete" which doesn't let 7585 * hstore compaction complete. In the former edition, this config is set in HStore class inside 7586 * compact method, though this is just for testing, otherwise it doesn't do any help. In 7587 * HBASE-8518, we try to get rid of all "hbase.hstore.compaction.complete" config (except for 7588 * testing code). 7589 */ 7590 public static class HStoreForTesting extends HStore { 7591 7592 protected HStoreForTesting(final HRegion region, final ColumnFamilyDescriptor family, 7593 final Configuration confParam, boolean warmup) throws IOException { 7594 super(region, family, confParam, warmup); 7595 } 7596 7597 @Override 7598 protected List<HStoreFile> doCompaction(CompactionRequestImpl cr, 7599 Collection<HStoreFile> filesToCompact, User user, long compactionStartTime, 7600 List<Path> newFiles) throws IOException { 7601 // let compaction incomplete. 7602 if (!this.conf.getBoolean("hbase.hstore.compaction.complete", true)) { 7603 LOG.warn("hbase.hstore.compaction.complete is set to false"); 7604 List<HStoreFile> sfs = new ArrayList<>(newFiles.size()); 7605 final boolean evictOnClose = 7606 getCacheConfig() != null ? getCacheConfig().shouldEvictOnClose() : true; 7607 for (Path newFile : newFiles) { 7608 // Create storefile around what we wrote with a reader on it. 7609 HStoreFile sf = storeEngine.createStoreFileAndReader(newFile); 7610 sf.closeStoreFile(evictOnClose); 7611 sfs.add(sf); 7612 } 7613 return sfs; 7614 } 7615 return super.doCompaction(cr, filesToCompact, user, compactionStartTime, newFiles); 7616 } 7617 } 7618 7619 @Test 7620 public void testCloseNoInterrupt() throws Exception { 7621 byte[] cf1 = Bytes.toBytes("CF1"); 7622 byte[][] families = { cf1 }; 7623 final int SLEEP_TIME = 10 * 1000; 7624 7625 Configuration conf = new Configuration(CONF); 7626 // Disable close thread interrupt and server abort behavior 7627 conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, false); 7628 conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, 1000); 7629 region = initHRegion(tableName, method, conf, families); 7630 7631 final CountDownLatch latch = new CountDownLatch(1); 7632 final AtomicBoolean holderInterrupted = new AtomicBoolean(); 7633 Thread holder = new Thread(new Runnable() { 7634 @Override 7635 public void run() { 7636 try { 7637 LOG.info("Starting region operation holder"); 7638 region.startRegionOperation(Operation.SCAN); 7639 latch.countDown(); 7640 try { 7641 Thread.sleep(SLEEP_TIME); 7642 } catch (InterruptedException e) { 7643 LOG.info("Interrupted"); 7644 holderInterrupted.set(true); 7645 } 7646 } catch (Exception e) { 7647 throw new RuntimeException(e); 7648 } finally { 7649 try { 7650 region.closeRegionOperation(); 7651 } catch (IOException e) { 7652 } 7653 LOG.info("Stopped region operation holder"); 7654 } 7655 } 7656 }); 7657 7658 holder.start(); 7659 latch.await(); 7660 HBaseTestingUtil.closeRegionAndWAL(region); 7661 region = null; 7662 holder.join(); 7663 7664 assertFalse("Region lock holder should not have been interrupted", holderInterrupted.get()); 7665 } 7666 7667 @Test 7668 public void testCloseInterrupt() throws Exception { 7669 byte[] cf1 = Bytes.toBytes("CF1"); 7670 byte[][] families = { cf1 }; 7671 final int SLEEP_TIME = 10 * 1000; 7672 7673 Configuration conf = new Configuration(CONF); 7674 // Enable close thread interrupt and server abort behavior 7675 conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, true); 7676 // Speed up the unit test, no need to wait default 10 seconds. 7677 conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, 1000); 7678 region = initHRegion(tableName, method, conf, families); 7679 7680 final CountDownLatch latch = new CountDownLatch(1); 7681 final AtomicBoolean holderInterrupted = new AtomicBoolean(); 7682 Thread holder = new Thread(new Runnable() { 7683 @Override 7684 public void run() { 7685 try { 7686 LOG.info("Starting region operation holder"); 7687 region.startRegionOperation(Operation.SCAN); 7688 latch.countDown(); 7689 try { 7690 Thread.sleep(SLEEP_TIME); 7691 } catch (InterruptedException e) { 7692 LOG.info("Interrupted"); 7693 holderInterrupted.set(true); 7694 } 7695 } catch (Exception e) { 7696 throw new RuntimeException(e); 7697 } finally { 7698 try { 7699 region.closeRegionOperation(); 7700 } catch (IOException e) { 7701 } 7702 LOG.info("Stopped region operation holder"); 7703 } 7704 } 7705 }); 7706 7707 holder.start(); 7708 latch.await(); 7709 region.close(); 7710 region = null; 7711 holder.join(); 7712 7713 assertTrue("Region lock holder was not interrupted", holderInterrupted.get()); 7714 } 7715 7716 @Test 7717 public void testCloseAbort() throws Exception { 7718 byte[] cf1 = Bytes.toBytes("CF1"); 7719 byte[][] families = { cf1 }; 7720 final int SLEEP_TIME = 10 * 1000; 7721 7722 Configuration conf = new Configuration(CONF); 7723 // Enable close thread interrupt and server abort behavior. 7724 conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, true); 7725 // Set the abort interval to a fraction of sleep time so we are guaranteed to be aborted. 7726 conf.setInt(HRegion.CLOSE_WAIT_TIME, SLEEP_TIME / 2); 7727 // Set the wait interval to a fraction of sleep time so we are guaranteed to be interrupted. 7728 conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, SLEEP_TIME / 4); 7729 region = initHRegion(tableName, method, conf, families); 7730 RegionServerServices rsServices = mock(RegionServerServices.class); 7731 when(rsServices.getServerName()).thenReturn(ServerName.valueOf("localhost", 1000, 1000)); 7732 region.rsServices = rsServices; 7733 7734 final CountDownLatch latch = new CountDownLatch(1); 7735 Thread holder = new Thread(new Runnable() { 7736 @Override 7737 public void run() { 7738 try { 7739 LOG.info("Starting region operation holder"); 7740 region.startRegionOperation(Operation.SCAN); 7741 latch.countDown(); 7742 // Hold the lock for SLEEP_TIME seconds no matter how many times we are interrupted. 7743 int timeRemaining = SLEEP_TIME; 7744 while (timeRemaining > 0) { 7745 long start = EnvironmentEdgeManager.currentTime(); 7746 try { 7747 Thread.sleep(timeRemaining); 7748 } catch (InterruptedException e) { 7749 LOG.info("Interrupted"); 7750 } 7751 long end = EnvironmentEdgeManager.currentTime(); 7752 timeRemaining -= end - start; 7753 if (timeRemaining < 0) { 7754 timeRemaining = 0; 7755 } 7756 if (timeRemaining > 0) { 7757 LOG.info("Sleeping again, remaining time " + timeRemaining + " ms"); 7758 } 7759 } 7760 } catch (Exception e) { 7761 throw new RuntimeException(e); 7762 } finally { 7763 try { 7764 region.closeRegionOperation(); 7765 } catch (IOException e) { 7766 } 7767 LOG.info("Stopped region operation holder"); 7768 } 7769 } 7770 }); 7771 7772 holder.start(); 7773 latch.await(); 7774 assertThrows(IOException.class, () -> region.close()); 7775 holder.join(); 7776 7777 // Verify the region tried to abort the server 7778 verify(rsServices, atLeast(1)).abort(anyString(), any()); 7779 } 7780 7781 @Test 7782 public void testInterruptProtection() throws Exception { 7783 byte[] cf1 = Bytes.toBytes("CF1"); 7784 byte[][] families = { cf1 }; 7785 final int SLEEP_TIME = 10 * 1000; 7786 7787 Configuration conf = new Configuration(CONF); 7788 // Enable close thread interrupt and server abort behavior. 7789 conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, true); 7790 conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, 1000); 7791 region = initHRegion(tableName, method, conf, families); 7792 7793 final CountDownLatch latch = new CountDownLatch(1); 7794 final AtomicBoolean holderInterrupted = new AtomicBoolean(); 7795 Thread holder = new Thread(new Runnable() { 7796 @Override 7797 public void run() { 7798 try { 7799 LOG.info("Starting region operation holder"); 7800 region.startRegionOperation(Operation.SCAN); 7801 LOG.info("Protecting against interrupts"); 7802 region.disableInterrupts(); 7803 try { 7804 latch.countDown(); 7805 try { 7806 Thread.sleep(SLEEP_TIME); 7807 } catch (InterruptedException e) { 7808 LOG.info("Interrupted"); 7809 holderInterrupted.set(true); 7810 } 7811 } finally { 7812 region.enableInterrupts(); 7813 } 7814 } catch (Exception e) { 7815 throw new RuntimeException(e); 7816 } finally { 7817 try { 7818 region.closeRegionOperation(); 7819 } catch (IOException e) { 7820 } 7821 LOG.info("Stopped region operation holder"); 7822 } 7823 } 7824 }); 7825 7826 holder.start(); 7827 latch.await(); 7828 region.close(); 7829 region = null; 7830 holder.join(); 7831 7832 assertFalse("Region lock holder should not have been interrupted", holderInterrupted.get()); 7833 } 7834 7835 @Test 7836 public void testRegionOnCoprocessorsChange() throws IOException { 7837 byte[] cf1 = Bytes.toBytes("CF1"); 7838 byte[][] families = { cf1 }; 7839 7840 Configuration conf = new Configuration(CONF); 7841 region = initHRegion(tableName, method, conf, families); 7842 assertNull(region.getCoprocessorHost()); 7843 7844 // set and verify the system coprocessors for region and user region 7845 Configuration newConf = new Configuration(conf); 7846 newConf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, MetaTableMetrics.class.getName()); 7847 newConf.set(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, 7848 NoOpRegionCoprocessor.class.getName()); 7849 // trigger configuration change 7850 region.onConfigurationChange(newConf); 7851 assertTrue(region.getCoprocessorHost() != null); 7852 Set<String> coprocessors = region.getCoprocessorHost().getCoprocessors(); 7853 assertTrue(coprocessors.size() == 2); 7854 assertTrue(region.getCoprocessorHost().getCoprocessors() 7855 .contains(MetaTableMetrics.class.getSimpleName())); 7856 assertTrue(region.getCoprocessorHost().getCoprocessors() 7857 .contains(NoOpRegionCoprocessor.class.getSimpleName())); 7858 7859 // remove region coprocessor and keep only user region coprocessor 7860 newConf.unset(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY); 7861 region.onConfigurationChange(newConf); 7862 assertTrue(region.getCoprocessorHost() != null); 7863 coprocessors = region.getCoprocessorHost().getCoprocessors(); 7864 assertTrue(coprocessors.size() == 1); 7865 assertTrue(region.getCoprocessorHost().getCoprocessors() 7866 .contains(NoOpRegionCoprocessor.class.getSimpleName())); 7867 } 7868 7869 @Test 7870 public void testRegionOnCoprocessorsWithoutChange() throws IOException { 7871 byte[] cf1 = Bytes.toBytes("CF1"); 7872 byte[][] families = { cf1 }; 7873 7874 Configuration conf = new Configuration(CONF); 7875 conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, 7876 MetaTableMetrics.class.getCanonicalName()); 7877 region = initHRegion(tableName, method, conf, families); 7878 // region service is null in unit test, we need to load the coprocessor once 7879 region.setCoprocessorHost(new RegionCoprocessorHost(region, null, conf)); 7880 RegionCoprocessor regionCoprocessor = 7881 region.getCoprocessorHost().findCoprocessor(MetaTableMetrics.class.getName()); 7882 7883 // simulate when other configuration may have changed and onConfigurationChange execute once 7884 region.onConfigurationChange(conf); 7885 RegionCoprocessor regionCoprocessorAfterOnConfigurationChange = 7886 region.getCoprocessorHost().findCoprocessor(MetaTableMetrics.class.getName()); 7887 assertEquals(regionCoprocessor, regionCoprocessorAfterOnConfigurationChange); 7888 } 7889 7890 public static class NoOpRegionCoprocessor implements RegionCoprocessor, RegionObserver { 7891 // a empty region coprocessor class 7892 } 7893}