001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.apache.hadoop.hbase.HBaseTestingUtil.COLUMNS; 021import static org.apache.hadoop.hbase.HBaseTestingUtil.fam1; 022import static org.apache.hadoop.hbase.HBaseTestingUtil.fam2; 023import static org.apache.hadoop.hbase.HBaseTestingUtil.fam3; 024import static org.junit.Assert.assertArrayEquals; 025import static org.junit.Assert.assertEquals; 026import static org.junit.Assert.assertFalse; 027import static org.junit.Assert.assertNotNull; 028import static org.junit.Assert.assertNull; 029import static org.junit.Assert.assertThrows; 030import static org.junit.Assert.assertTrue; 031import static org.junit.Assert.fail; 032import static org.mockito.ArgumentMatchers.any; 033import static org.mockito.ArgumentMatchers.anyLong; 034import static org.mockito.ArgumentMatchers.anyString; 035import static org.mockito.ArgumentMatchers.isA; 036import static org.mockito.Mockito.atLeast; 037import static org.mockito.Mockito.doAnswer; 038import static org.mockito.Mockito.doThrow; 039import static org.mockito.Mockito.mock; 040import static org.mockito.Mockito.never; 041import static org.mockito.Mockito.spy; 042import static org.mockito.Mockito.times; 043import static org.mockito.Mockito.verify; 044import static org.mockito.Mockito.when; 045 046import java.io.IOException; 047import java.io.InterruptedIOException; 048import java.math.BigDecimal; 049import java.security.PrivilegedExceptionAction; 050import java.util.ArrayList; 051import java.util.Arrays; 052import java.util.Collection; 053import java.util.List; 054import java.util.Map; 055import java.util.NavigableMap; 056import java.util.Objects; 057import java.util.Set; 058import java.util.TreeMap; 059import java.util.concurrent.Callable; 060import java.util.concurrent.CountDownLatch; 061import java.util.concurrent.ExecutorService; 062import java.util.concurrent.Executors; 063import java.util.concurrent.Future; 064import java.util.concurrent.TimeUnit; 065import java.util.concurrent.atomic.AtomicBoolean; 066import java.util.concurrent.atomic.AtomicInteger; 067import java.util.concurrent.atomic.AtomicLong; 068import java.util.concurrent.atomic.AtomicReference; 069import org.apache.commons.lang3.RandomStringUtils; 070import org.apache.hadoop.conf.Configuration; 071import org.apache.hadoop.fs.FSDataOutputStream; 072import org.apache.hadoop.fs.FileStatus; 073import org.apache.hadoop.fs.FileSystem; 074import org.apache.hadoop.fs.Path; 075import org.apache.hadoop.hbase.ArrayBackedTag; 076import org.apache.hadoop.hbase.Cell; 077import org.apache.hadoop.hbase.Cell.Type; 078import org.apache.hadoop.hbase.CellBuilderFactory; 079import org.apache.hadoop.hbase.CellBuilderType; 080import org.apache.hadoop.hbase.CellUtil; 081import org.apache.hadoop.hbase.CompareOperator; 082import org.apache.hadoop.hbase.CompatibilitySingletonFactory; 083import org.apache.hadoop.hbase.DoNotRetryIOException; 084import org.apache.hadoop.hbase.DroppedSnapshotException; 085import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; 086import org.apache.hadoop.hbase.HBaseClassTestRule; 087import org.apache.hadoop.hbase.HBaseConfiguration; 088import org.apache.hadoop.hbase.HBaseTestingUtil; 089import org.apache.hadoop.hbase.HConstants; 090import org.apache.hadoop.hbase.HConstants.OperationStatusCode; 091import org.apache.hadoop.hbase.HDFSBlocksDistribution; 092import org.apache.hadoop.hbase.KeyValue; 093import org.apache.hadoop.hbase.MultithreadedTestUtil; 094import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread; 095import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread; 096import org.apache.hadoop.hbase.NotServingRegionException; 097import org.apache.hadoop.hbase.PrivateCellUtil; 098import org.apache.hadoop.hbase.RegionTooBusyException; 099import org.apache.hadoop.hbase.ServerName; 100import org.apache.hadoop.hbase.SingleProcessHBaseCluster; 101import org.apache.hadoop.hbase.StartTestingClusterOption; 102import org.apache.hadoop.hbase.TableName; 103import org.apache.hadoop.hbase.TagType; 104import org.apache.hadoop.hbase.Waiter; 105import org.apache.hadoop.hbase.client.Append; 106import org.apache.hadoop.hbase.client.CheckAndMutate; 107import org.apache.hadoop.hbase.client.CheckAndMutateResult; 108import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 109import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 110import org.apache.hadoop.hbase.client.Delete; 111import org.apache.hadoop.hbase.client.Durability; 112import org.apache.hadoop.hbase.client.Get; 113import org.apache.hadoop.hbase.client.Increment; 114import org.apache.hadoop.hbase.client.Mutation; 115import org.apache.hadoop.hbase.client.Put; 116import org.apache.hadoop.hbase.client.RegionInfo; 117import org.apache.hadoop.hbase.client.RegionInfoBuilder; 118import org.apache.hadoop.hbase.client.Result; 119import org.apache.hadoop.hbase.client.RowMutations; 120import org.apache.hadoop.hbase.client.Scan; 121import org.apache.hadoop.hbase.client.Table; 122import org.apache.hadoop.hbase.client.TableDescriptor; 123import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 124import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; 125import org.apache.hadoop.hbase.coprocessor.MetaTableMetrics; 126import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor; 127import org.apache.hadoop.hbase.coprocessor.RegionObserver; 128import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException; 129import org.apache.hadoop.hbase.filter.BigDecimalComparator; 130import org.apache.hadoop.hbase.filter.BinaryComparator; 131import org.apache.hadoop.hbase.filter.ColumnCountGetFilter; 132import org.apache.hadoop.hbase.filter.Filter; 133import org.apache.hadoop.hbase.filter.FilterBase; 134import org.apache.hadoop.hbase.filter.FilterList; 135import org.apache.hadoop.hbase.filter.NullComparator; 136import org.apache.hadoop.hbase.filter.PrefixFilter; 137import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter; 138import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; 139import org.apache.hadoop.hbase.filter.SubstringComparator; 140import org.apache.hadoop.hbase.filter.ValueFilter; 141import org.apache.hadoop.hbase.io.TimeRange; 142import org.apache.hadoop.hbase.io.hfile.HFile; 143import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler; 144import org.apache.hadoop.hbase.monitoring.MonitoredTask; 145import org.apache.hadoop.hbase.monitoring.TaskMonitor; 146import org.apache.hadoop.hbase.regionserver.Region.Operation; 147import org.apache.hadoop.hbase.regionserver.Region.RowLock; 148import org.apache.hadoop.hbase.regionserver.TestHStore.FaultyFileSystem; 149import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl; 150import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL; 151import org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL; 152import org.apache.hadoop.hbase.regionserver.wal.FSHLog; 153import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource; 154import org.apache.hadoop.hbase.regionserver.wal.WALUtil; 155import org.apache.hadoop.hbase.replication.regionserver.ReplicationObserver; 156import org.apache.hadoop.hbase.security.User; 157import org.apache.hadoop.hbase.test.MetricsAssertHelper; 158import org.apache.hadoop.hbase.testclassification.LargeTests; 159import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; 160import org.apache.hadoop.hbase.util.Bytes; 161import org.apache.hadoop.hbase.util.CommonFSUtils; 162import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 163import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; 164import org.apache.hadoop.hbase.util.HFileArchiveUtil; 165import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge; 166import org.apache.hadoop.hbase.util.ManualEnvironmentEdge; 167import org.apache.hadoop.hbase.util.Threads; 168import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; 169import org.apache.hadoop.hbase.wal.FaultyFSLog; 170import org.apache.hadoop.hbase.wal.NettyAsyncFSWALConfigHelper; 171import org.apache.hadoop.hbase.wal.WAL; 172import org.apache.hadoop.hbase.wal.WALEdit; 173import org.apache.hadoop.hbase.wal.WALFactory; 174import org.apache.hadoop.hbase.wal.WALKeyImpl; 175import org.apache.hadoop.hbase.wal.WALProvider; 176import org.apache.hadoop.hbase.wal.WALProvider.Writer; 177import org.apache.hadoop.hbase.wal.WALSplitUtil; 178import org.apache.hadoop.hbase.wal.WALStreamReader; 179import org.junit.After; 180import org.junit.Assert; 181import org.junit.Before; 182import org.junit.ClassRule; 183import org.junit.Ignore; 184import org.junit.Rule; 185import org.junit.Test; 186import org.junit.experimental.categories.Category; 187import org.junit.rules.ExpectedException; 188import org.junit.rules.TestName; 189import org.mockito.ArgumentCaptor; 190import org.mockito.ArgumentMatcher; 191import org.mockito.invocation.InvocationOnMock; 192import org.mockito.stubbing.Answer; 193import org.slf4j.Logger; 194import org.slf4j.LoggerFactory; 195 196import org.apache.hbase.thirdparty.com.google.common.collect.Lists; 197import org.apache.hbase.thirdparty.com.google.protobuf.ByteString; 198import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup; 199import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup; 200import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel; 201 202import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 203import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor; 204import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor; 205import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.FlushAction; 206import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor; 207import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor; 208import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.StoreDescriptor; 209 210/** 211 * Basic stand-alone testing of HRegion. No clusters! A lot of the meta information for an HRegion 212 * now lives inside other HRegions or in the HBaseMaster, so only basic testing is possible. 213 */ 214@Category({ VerySlowRegionServerTests.class, LargeTests.class }) 215@SuppressWarnings("deprecation") 216public class TestHRegion { 217 218 @ClassRule 219 public static final HBaseClassTestRule CLASS_RULE = 220 HBaseClassTestRule.forClass(TestHRegion.class); 221 222 // Do not spin up clusters in here. If you need to spin up a cluster, do it 223 // over in TestHRegionOnCluster. 224 private static final Logger LOG = LoggerFactory.getLogger(TestHRegion.class); 225 @Rule 226 public TestName name = new TestName(); 227 @Rule 228 public final ExpectedException thrown = ExpectedException.none(); 229 230 private static final String COLUMN_FAMILY = "MyCF"; 231 private static final byte[] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY); 232 private static final EventLoopGroup GROUP = new NioEventLoopGroup(); 233 234 HRegion region = null; 235 // Do not run unit tests in parallel (? Why not? It don't work? Why not? St.Ack) 236 protected static HBaseTestingUtil TEST_UTIL; 237 public static Configuration CONF; 238 private String dir; 239 private final int MAX_VERSIONS = 2; 240 241 // Test names 242 protected TableName tableName; 243 protected String method; 244 protected final byte[] qual = Bytes.toBytes("qual"); 245 protected final byte[] qual1 = Bytes.toBytes("qual1"); 246 protected final byte[] qual2 = Bytes.toBytes("qual2"); 247 protected final byte[] qual3 = Bytes.toBytes("qual3"); 248 protected final byte[] value = Bytes.toBytes("value"); 249 protected final byte[] value1 = Bytes.toBytes("value1"); 250 protected final byte[] value2 = Bytes.toBytes("value2"); 251 protected final byte[] row = Bytes.toBytes("rowA"); 252 protected final byte[] row2 = Bytes.toBytes("rowB"); 253 254 protected final MetricsAssertHelper metricsAssertHelper = 255 CompatibilitySingletonFactory.getInstance(MetricsAssertHelper.class); 256 257 @Before 258 public void setup() throws IOException { 259 TEST_UTIL = new HBaseTestingUtil(); 260 CONF = TEST_UTIL.getConfiguration(); 261 NettyAsyncFSWALConfigHelper.setEventLoopConfig(CONF, GROUP, NioSocketChannel.class); 262 dir = TEST_UTIL.getDataTestDir("TestHRegion").toString(); 263 method = name.getMethodName(); 264 tableName = TableName.valueOf(method); 265 CONF.set(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, String.valueOf(0.09)); 266 CONF.setLong(AbstractFSWAL.WAL_SYNC_TIMEOUT_MS, 10000); 267 } 268 269 @After 270 public void tearDown() throws IOException { 271 // Region may have been closed, but it is still no harm if we close it again here using HTU. 272 HBaseTestingUtil.closeRegionAndWAL(region); 273 EnvironmentEdgeManagerTestHelper.reset(); 274 LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir()); 275 TEST_UTIL.cleanupTestDir(); 276 } 277 278 /** 279 * Test that I can use the max flushed sequence id after the close. 280 */ 281 @Test 282 public void testSequenceId() throws IOException { 283 region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES); 284 assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId()); 285 // Weird. This returns 0 if no store files or no edits. Afraid to change it. 286 assertEquals(0, (long) region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES)); 287 HBaseTestingUtil.closeRegionAndWAL(this.region); 288 assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId()); 289 assertEquals(0, (long) region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES)); 290 HRegion oldRegion = region; 291 try { 292 // Open region again. 293 region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES); 294 byte[] value = Bytes.toBytes(method); 295 // Make a random put against our cf. 296 Put put = new Put(value); 297 put.addColumn(COLUMN_FAMILY_BYTES, null, value); 298 region.put(put); 299 // No flush yet so init numbers should still be in place. 300 assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId()); 301 assertEquals(0, (long) region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES)); 302 region.flush(true); 303 long max = region.getMaxFlushedSeqId(); 304 HBaseTestingUtil.closeRegionAndWAL(this.region); 305 assertEquals(max, region.getMaxFlushedSeqId()); 306 this.region = null; 307 } finally { 308 HBaseTestingUtil.closeRegionAndWAL(oldRegion); 309 } 310 } 311 312 /** 313 * Test for Bug 2 of HBASE-10466. "Bug 2: Conditions for the first flush of region close 314 * (so-called pre-flush) If memstoreSize is smaller than a certain value, or when region close 315 * starts a flush is ongoing, the first flush is skipped and only the second flush takes place. 316 * However, two flushes are required in case previous flush fails and leaves some data in 317 * snapshot. The bug could cause loss of data in current memstore. The fix is removing all 318 * conditions except abort check so we ensure 2 flushes for region close." 319 */ 320 @Test 321 public void testCloseCarryingSnapshot() throws IOException { 322 region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES); 323 HStore store = region.getStore(COLUMN_FAMILY_BYTES); 324 // Get some random bytes. 325 byte[] value = Bytes.toBytes(method); 326 // Make a random put against our cf. 327 Put put = new Put(value); 328 put.addColumn(COLUMN_FAMILY_BYTES, null, value); 329 // First put something in current memstore, which will be in snapshot after flusher.prepare() 330 region.put(put); 331 StoreFlushContext storeFlushCtx = store.createFlushContext(12345, FlushLifeCycleTracker.DUMMY); 332 storeFlushCtx.prepare(); 333 // Second put something in current memstore 334 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value); 335 region.put(put); 336 // Close with something in memstore and something in the snapshot. Make sure all is cleared. 337 HBaseTestingUtil.closeRegionAndWAL(region); 338 assertEquals(0, region.getMemStoreDataSize()); 339 region = null; 340 } 341 342 /* 343 * This test is for verifying memstore snapshot size is correctly updated in case of rollback See 344 * HBASE-10845 345 */ 346 @Test 347 public void testMemstoreSnapshotSize() throws IOException { 348 class MyFaultyFSLog extends FaultyFSLog { 349 StoreFlushContext storeFlushCtx; 350 351 public MyFaultyFSLog(FileSystem fs, Path rootDir, String logName, Configuration conf) 352 throws IOException { 353 super(fs, rootDir, logName, conf); 354 } 355 356 void setStoreFlushCtx(StoreFlushContext storeFlushCtx) { 357 this.storeFlushCtx = storeFlushCtx; 358 } 359 360 @Override 361 protected void doSync(long txid, boolean forceSync) throws IOException { 362 storeFlushCtx.prepare(); 363 super.doSync(txid, forceSync); 364 } 365 } 366 367 FileSystem fs = FileSystem.get(CONF); 368 Path rootDir = new Path(dir + "testMemstoreSnapshotSize"); 369 MyFaultyFSLog faultyLog = new MyFaultyFSLog(fs, rootDir, "testMemstoreSnapshotSize", CONF); 370 faultyLog.init(); 371 region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, faultyLog, 372 COLUMN_FAMILY_BYTES); 373 374 HStore store = region.getStore(COLUMN_FAMILY_BYTES); 375 // Get some random bytes. 376 byte[] value = Bytes.toBytes(method); 377 faultyLog.setStoreFlushCtx(store.createFlushContext(12345, FlushLifeCycleTracker.DUMMY)); 378 379 Put put = new Put(value); 380 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value); 381 faultyLog.setFailureType(FaultyFSLog.FailureType.SYNC); 382 boolean threwIOE = false; 383 try { 384 region.put(put); 385 } catch (IOException ioe) { 386 threwIOE = true; 387 } finally { 388 assertTrue("The regionserver should have thrown an exception", threwIOE); 389 } 390 MemStoreSize mss = store.getFlushableSize(); 391 assertTrue("flushable size should be zero, but it is " + mss, mss.getDataSize() == 0); 392 } 393 394 /** 395 * Create a WAL outside of the usual helper in 396 * {@link HBaseTestingUtil#createWal(Configuration, Path, RegionInfo)} because that method doesn't 397 * play nicely with FaultyFileSystem. Call this method before overriding {@code fs.file.impl}. 398 * @param callingMethod a unique component for the path, probably the name of the test method. 399 */ 400 private static WAL createWALCompatibleWithFaultyFileSystem(String callingMethod, 401 Configuration conf, TableName tableName) throws IOException { 402 final Path logDir = TEST_UTIL.getDataTestDirOnTestFS(callingMethod + ".log"); 403 final Configuration walConf = new Configuration(conf); 404 CommonFSUtils.setRootDir(walConf, logDir); 405 return new WALFactory(walConf, callingMethod) 406 .getWAL(RegionInfoBuilder.newBuilder(tableName).build()); 407 } 408 409 @Test 410 public void testMemstoreSizeAccountingWithFailedPostBatchMutate() throws IOException { 411 String testName = "testMemstoreSizeAccountingWithFailedPostBatchMutate"; 412 FileSystem fs = FileSystem.get(CONF); 413 Path rootDir = new Path(dir + testName); 414 FSHLog hLog = new FSHLog(fs, rootDir, testName, CONF); 415 hLog.init(); 416 region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, hLog, 417 COLUMN_FAMILY_BYTES); 418 HStore store = region.getStore(COLUMN_FAMILY_BYTES); 419 assertEquals(0, region.getMemStoreDataSize()); 420 421 // Put one value 422 byte[] value = Bytes.toBytes(method); 423 Put put = new Put(value); 424 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value); 425 region.put(put); 426 long onePutSize = region.getMemStoreDataSize(); 427 assertTrue(onePutSize > 0); 428 429 RegionCoprocessorHost mockedCPHost = mock(RegionCoprocessorHost.class); 430 doThrow(new IOException()).when(mockedCPHost).postBatchMutate(any()); 431 region.setCoprocessorHost(mockedCPHost); 432 433 put = new Put(value); 434 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("dfg"), value); 435 try { 436 region.put(put); 437 fail("Should have failed with IOException"); 438 } catch (IOException expected) { 439 } 440 long expectedSize = onePutSize * 2; 441 assertEquals("memstoreSize should be incremented", expectedSize, region.getMemStoreDataSize()); 442 assertEquals("flushable size should be incremented", expectedSize, 443 store.getFlushableSize().getDataSize()); 444 445 region.setCoprocessorHost(null); 446 } 447 448 /** 449 * A test case of HBASE-21041 450 */ 451 @Test 452 public void testFlushAndMemstoreSizeCounting() throws Exception { 453 byte[] family = Bytes.toBytes("family"); 454 this.region = initHRegion(tableName, method, CONF, family); 455 for (byte[] row : HBaseTestingUtil.ROWS) { 456 Put put = new Put(row); 457 put.addColumn(family, family, row); 458 region.put(put); 459 } 460 region.flush(true); 461 // After flush, data size should be zero 462 assertEquals(0, region.getMemStoreDataSize()); 463 // After flush, a new active mutable segment is created, so the heap size 464 // should equal to MutableSegment.DEEP_OVERHEAD 465 assertEquals(MutableSegment.DEEP_OVERHEAD, region.getMemStoreHeapSize()); 466 // After flush, offheap should be zero 467 assertEquals(0, region.getMemStoreOffHeapSize()); 468 } 469 470 /** 471 * Test we do not lose data if we fail a flush and then close. Part of HBase-10466. Tests the 472 * following from the issue description: "Bug 1: Wrong calculation of HRegion.memstoreSize: When a 473 * flush fails, data to be flushed is kept in each MemStore's snapshot and wait for next flush 474 * attempt to continue on it. But when the next flush succeeds, the counter of total memstore size 475 * in HRegion is always deduced by the sum of current memstore sizes instead of snapshots left 476 * from previous failed flush. This calculation is problematic that almost every time there is 477 * failed flush, HRegion.memstoreSize gets reduced by a wrong value. If region flush could not 478 * proceed for a couple cycles, the size in current memstore could be much larger than the 479 * snapshot. It's likely to drift memstoreSize much smaller than expected. In extreme case, if the 480 * error accumulates to even bigger than HRegion's memstore size limit, any further flush is 481 * skipped because flush does not do anything if memstoreSize is not larger than 0." 482 */ 483 @Test 484 public void testFlushSizeAccounting() throws Exception { 485 final Configuration conf = HBaseConfiguration.create(CONF); 486 final WAL wal = createWALCompatibleWithFaultyFileSystem(method, conf, tableName); 487 // Only retry once. 488 conf.setInt("hbase.hstore.flush.retries.number", 1); 489 final User user = User.createUserForTesting(conf, method, new String[] { "foo" }); 490 // Inject our faulty LocalFileSystem 491 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class); 492 user.runAs(new PrivilegedExceptionAction<Object>() { 493 @Override 494 public Object run() throws Exception { 495 // Make sure it worked (above is sensitive to caching details in hadoop core) 496 FileSystem fs = FileSystem.get(conf); 497 Assert.assertEquals(FaultyFileSystem.class, fs.getClass()); 498 FaultyFileSystem ffs = (FaultyFileSystem) fs; 499 HRegion region = null; 500 try { 501 // Initialize region 502 region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, wal, 503 COLUMN_FAMILY_BYTES); 504 long size = region.getMemStoreDataSize(); 505 Assert.assertEquals(0, size); 506 // Put one item into memstore. Measure the size of one item in memstore. 507 Put p1 = new Put(row); 508 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[]) null)); 509 region.put(p1); 510 final long sizeOfOnePut = region.getMemStoreDataSize(); 511 // Fail a flush which means the current memstore will hang out as memstore 'snapshot'. 512 try { 513 LOG.info("Flushing"); 514 region.flush(true); 515 Assert.fail("Didn't bubble up IOE!"); 516 } catch (DroppedSnapshotException dse) { 517 // What we are expecting 518 region.closing.set(false); // this is needed for the rest of the test to work 519 } 520 // Make it so all writes succeed from here on out 521 ffs.fault.set(false); 522 // Check sizes. Should still be the one entry. 523 Assert.assertEquals(sizeOfOnePut, region.getMemStoreDataSize()); 524 // Now add two entries so that on this next flush that fails, we can see if we 525 // subtract the right amount, the snapshot size only. 526 Put p2 = new Put(row); 527 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[]) null)); 528 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[]) null)); 529 region.put(p2); 530 long expectedSize = sizeOfOnePut * 3; 531 Assert.assertEquals(expectedSize, region.getMemStoreDataSize()); 532 // Do a successful flush. It will clear the snapshot only. Thats how flushes work. 533 // If already a snapshot, we clear it else we move the memstore to be snapshot and flush 534 // it 535 region.flush(true); 536 // Make sure our memory accounting is right. 537 Assert.assertEquals(sizeOfOnePut * 2, region.getMemStoreDataSize()); 538 } finally { 539 HBaseTestingUtil.closeRegionAndWAL(region); 540 } 541 return null; 542 } 543 }); 544 FileSystem.closeAllForUGI(user.getUGI()); 545 } 546 547 @Test 548 public void testCloseWithFailingFlush() throws Exception { 549 final Configuration conf = HBaseConfiguration.create(CONF); 550 final WAL wal = createWALCompatibleWithFaultyFileSystem(method, conf, tableName); 551 // Only retry once. 552 conf.setInt("hbase.hstore.flush.retries.number", 1); 553 final User user = User.createUserForTesting(conf, this.method, new String[] { "foo" }); 554 // Inject our faulty LocalFileSystem 555 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class); 556 user.runAs(new PrivilegedExceptionAction<Object>() { 557 @Override 558 public Object run() throws Exception { 559 // Make sure it worked (above is sensitive to caching details in hadoop core) 560 FileSystem fs = FileSystem.get(conf); 561 Assert.assertEquals(FaultyFileSystem.class, fs.getClass()); 562 FaultyFileSystem ffs = (FaultyFileSystem) fs; 563 HRegion region = null; 564 try { 565 // Initialize region 566 region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, wal, 567 COLUMN_FAMILY_BYTES); 568 long size = region.getMemStoreDataSize(); 569 Assert.assertEquals(0, size); 570 // Put one item into memstore. Measure the size of one item in memstore. 571 Put p1 = new Put(row); 572 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[]) null)); 573 region.put(p1); 574 // Manufacture an outstanding snapshot -- fake a failed flush by doing prepare step only. 575 HStore store = region.getStore(COLUMN_FAMILY_BYTES); 576 StoreFlushContext storeFlushCtx = 577 store.createFlushContext(12345, FlushLifeCycleTracker.DUMMY); 578 storeFlushCtx.prepare(); 579 // Now add two entries to the foreground memstore. 580 Put p2 = new Put(row); 581 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[]) null)); 582 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[]) null)); 583 region.put(p2); 584 // Now try close on top of a failing flush. 585 HBaseTestingUtil.closeRegionAndWAL(region); 586 region = null; 587 fail(); 588 } catch (DroppedSnapshotException dse) { 589 // Expected 590 LOG.info("Expected DroppedSnapshotException"); 591 } finally { 592 // Make it so all writes succeed from here on out so can close clean 593 ffs.fault.set(false); 594 HBaseTestingUtil.closeRegionAndWAL(region); 595 } 596 return null; 597 } 598 }); 599 FileSystem.closeAllForUGI(user.getUGI()); 600 } 601 602 @Test 603 public void testCompactionAffectedByScanners() throws Exception { 604 byte[] family = Bytes.toBytes("family"); 605 this.region = initHRegion(tableName, method, CONF, family); 606 607 Put put = new Put(Bytes.toBytes("r1")); 608 put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); 609 region.put(put); 610 region.flush(true); 611 612 Scan scan = new Scan(); 613 scan.readVersions(3); 614 // open the first scanner 615 try (RegionScanner scanner1 = region.getScanner(scan)) { 616 Delete delete = new Delete(Bytes.toBytes("r1")); 617 region.delete(delete); 618 region.flush(true); 619 // open the second scanner 620 try (RegionScanner scanner2 = region.getScanner(scan)) { 621 List<Cell> results = new ArrayList<>(); 622 623 LOG.info("Smallest read point:" + region.getSmallestReadPoint()); 624 625 // make a major compaction 626 region.compact(true); 627 628 // open the third scanner 629 try (RegionScanner scanner3 = region.getScanner(scan)) { 630 // get data from scanner 1, 2, 3 after major compaction 631 scanner1.next(results); 632 LOG.info(results.toString()); 633 assertEquals(1, results.size()); 634 635 results.clear(); 636 scanner2.next(results); 637 LOG.info(results.toString()); 638 assertEquals(0, results.size()); 639 640 results.clear(); 641 scanner3.next(results); 642 LOG.info(results.toString()); 643 assertEquals(0, results.size()); 644 } 645 } 646 } 647 } 648 649 @Test 650 public void testToShowNPEOnRegionScannerReseek() throws Exception { 651 byte[] family = Bytes.toBytes("family"); 652 this.region = initHRegion(tableName, method, CONF, family); 653 654 Put put = new Put(Bytes.toBytes("r1")); 655 put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); 656 region.put(put); 657 put = new Put(Bytes.toBytes("r2")); 658 put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); 659 region.put(put); 660 region.flush(true); 661 662 Scan scan = new Scan(); 663 scan.readVersions(3); 664 // open the first scanner 665 try (RegionScanner scanner1 = region.getScanner(scan)) { 666 LOG.info("Smallest read point:" + region.getSmallestReadPoint()); 667 668 region.compact(true); 669 670 scanner1.reseek(Bytes.toBytes("r2")); 671 List<Cell> results = new ArrayList<>(); 672 scanner1.next(results); 673 Cell keyValue = results.get(0); 674 assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0); 675 scanner1.close(); 676 } 677 } 678 679 @Test 680 public void testArchiveRecoveredEditsReplay() throws Exception { 681 byte[] family = Bytes.toBytes("family"); 682 this.region = initHRegion(tableName, method, CONF, family); 683 final WALFactory wals = new WALFactory(CONF, method); 684 try { 685 Path regiondir = region.getRegionFileSystem().getRegionDir(); 686 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 687 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 688 689 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 690 691 long maxSeqId = 1050; 692 long minSeqId = 1000; 693 694 for (long i = minSeqId; i <= maxSeqId; i += 10) { 695 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); 696 fs.create(recoveredEdits); 697 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 698 699 long time = System.nanoTime(); 700 WALEdit edit = new WALEdit(); 701 edit.add( 702 new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i))); 703 writer.append(new WAL.Entry( 704 new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); 705 706 writer.close(); 707 } 708 MonitoredTask status = TaskMonitor.get().createStatus(method); 709 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 710 for (HStore store : region.getStores()) { 711 maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId - 1); 712 } 713 CONF.set("hbase.region.archive.recovered.edits", "true"); 714 CONF.set(CommonFSUtils.HBASE_WAL_DIR, "/custom_wal_dir"); 715 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status); 716 assertEquals(maxSeqId, seqId); 717 region.getMVCC().advanceTo(seqId); 718 String fakeFamilyName = recoveredEditsDir.getName(); 719 Path rootDir = new Path(CONF.get(HConstants.HBASE_DIR)); 720 Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePathForRootDir(rootDir, 721 region.getRegionInfo(), Bytes.toBytes(fakeFamilyName)); 722 FileStatus[] list = TEST_UTIL.getTestFileSystem().listStatus(storeArchiveDir); 723 assertEquals(6, list.length); 724 } finally { 725 CONF.set("hbase.region.archive.recovered.edits", "false"); 726 CONF.set(CommonFSUtils.HBASE_WAL_DIR, ""); 727 HBaseTestingUtil.closeRegionAndWAL(this.region); 728 this.region = null; 729 wals.close(); 730 } 731 } 732 733 @Test 734 public void testSkipRecoveredEditsReplay() throws Exception { 735 byte[] family = Bytes.toBytes("family"); 736 this.region = initHRegion(tableName, method, CONF, family); 737 final WALFactory wals = new WALFactory(CONF, method); 738 try { 739 Path regiondir = region.getRegionFileSystem().getRegionDir(); 740 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 741 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 742 743 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 744 745 long maxSeqId = 1050; 746 long minSeqId = 1000; 747 748 for (long i = minSeqId; i <= maxSeqId; i += 10) { 749 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); 750 fs.create(recoveredEdits); 751 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 752 753 long time = System.nanoTime(); 754 WALEdit edit = new WALEdit(); 755 edit.add( 756 new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i))); 757 writer.append(new WAL.Entry( 758 new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); 759 760 writer.close(); 761 } 762 MonitoredTask status = TaskMonitor.get().createStatus(method); 763 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 764 for (HStore store : region.getStores()) { 765 maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId - 1); 766 } 767 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status); 768 assertEquals(maxSeqId, seqId); 769 region.getMVCC().advanceTo(seqId); 770 Get get = new Get(row); 771 Result result = region.get(get); 772 for (long i = minSeqId; i <= maxSeqId; i += 10) { 773 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i)); 774 assertEquals(1, kvs.size()); 775 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0))); 776 } 777 } finally { 778 HBaseTestingUtil.closeRegionAndWAL(this.region); 779 this.region = null; 780 wals.close(); 781 } 782 } 783 784 @Test 785 public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception { 786 byte[] family = Bytes.toBytes("family"); 787 this.region = initHRegion(tableName, method, CONF, family); 788 final WALFactory wals = new WALFactory(CONF, method); 789 try { 790 Path regiondir = region.getRegionFileSystem().getRegionDir(); 791 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 792 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 793 794 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 795 796 long maxSeqId = 1050; 797 long minSeqId = 1000; 798 799 for (long i = minSeqId; i <= maxSeqId; i += 10) { 800 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); 801 fs.create(recoveredEdits); 802 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 803 804 long time = System.nanoTime(); 805 WALEdit edit = new WALEdit(); 806 edit.add( 807 new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i))); 808 writer.append(new WAL.Entry( 809 new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); 810 811 writer.close(); 812 } 813 long recoverSeqId = 1030; 814 MonitoredTask status = TaskMonitor.get().createStatus(method); 815 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 816 for (HStore store : region.getStores()) { 817 maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), recoverSeqId - 1); 818 } 819 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status); 820 assertEquals(maxSeqId, seqId); 821 region.getMVCC().advanceTo(seqId); 822 Get get = new Get(row); 823 Result result = region.get(get); 824 for (long i = minSeqId; i <= maxSeqId; i += 10) { 825 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i)); 826 if (i < recoverSeqId) { 827 assertEquals(0, kvs.size()); 828 } else { 829 assertEquals(1, kvs.size()); 830 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0))); 831 } 832 } 833 } finally { 834 HBaseTestingUtil.closeRegionAndWAL(this.region); 835 this.region = null; 836 wals.close(); 837 } 838 } 839 840 @Test 841 public void testSkipRecoveredEditsReplayAllIgnored() throws Exception { 842 byte[] family = Bytes.toBytes("family"); 843 this.region = initHRegion(tableName, method, CONF, family); 844 Path regiondir = region.getRegionFileSystem().getRegionDir(); 845 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 846 847 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 848 for (int i = 1000; i < 1050; i += 10) { 849 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); 850 FSDataOutputStream dos = fs.create(recoveredEdits); 851 dos.writeInt(i); 852 dos.close(); 853 } 854 long minSeqId = 2000; 855 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1)); 856 FSDataOutputStream dos = fs.create(recoveredEdits); 857 dos.close(); 858 859 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 860 for (HStore store : region.getStores()) { 861 maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId); 862 } 863 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, null); 864 assertEquals(minSeqId, seqId); 865 } 866 867 @Test 868 public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception { 869 byte[] family = Bytes.toBytes("family"); 870 this.region = initHRegion(tableName, method, CONF, family); 871 final WALFactory wals = new WALFactory(CONF, method); 872 try { 873 Path regiondir = region.getRegionFileSystem().getRegionDir(); 874 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 875 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 876 byte[][] columns = region.getTableDescriptor().getColumnFamilyNames().toArray(new byte[0][]); 877 878 assertEquals(0, region.getStoreFileList(columns).size()); 879 880 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 881 882 long maxSeqId = 1050; 883 long minSeqId = 1000; 884 885 for (long i = minSeqId; i <= maxSeqId; i += 10) { 886 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); 887 fs.create(recoveredEdits); 888 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 889 890 long time = System.nanoTime(); 891 WALEdit edit = null; 892 if (i == maxSeqId) { 893 edit = WALEdit.createCompaction(region.getRegionInfo(), 894 CompactionDescriptor.newBuilder().setTableName(ByteString.copyFrom(tableName.getName())) 895 .setFamilyName(ByteString.copyFrom(regionName)) 896 .setEncodedRegionName(ByteString.copyFrom(regionName)) 897 .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString()))) 898 .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName())).build()); 899 } else { 900 edit = new WALEdit(); 901 edit.add( 902 new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i))); 903 } 904 writer.append(new WAL.Entry( 905 new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); 906 writer.close(); 907 } 908 909 long recoverSeqId = 1030; 910 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 911 MonitoredTask status = TaskMonitor.get().createStatus(method); 912 for (HStore store : region.getStores()) { 913 maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), recoverSeqId - 1); 914 } 915 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status); 916 assertEquals(maxSeqId, seqId); 917 918 // assert that the files are flushed 919 assertEquals(1, region.getStoreFileList(columns).size()); 920 921 } finally { 922 HBaseTestingUtil.closeRegionAndWAL(this.region); 923 this.region = null; 924 wals.close(); 925 } 926 } 927 928 @Test 929 public void testRecoveredEditsReplayCompaction() throws Exception { 930 testRecoveredEditsReplayCompaction(false); 931 testRecoveredEditsReplayCompaction(true); 932 } 933 934 public void testRecoveredEditsReplayCompaction(boolean mismatchedRegionName) throws Exception { 935 CONF.setClass(HConstants.REGION_IMPL, HRegionForTesting.class, Region.class); 936 byte[] family = Bytes.toBytes("family"); 937 this.region = initHRegion(tableName, method, CONF, family); 938 final WALFactory wals = new WALFactory(CONF, method); 939 try { 940 Path regiondir = region.getRegionFileSystem().getRegionDir(); 941 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 942 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 943 944 long maxSeqId = 3; 945 long minSeqId = 0; 946 947 for (long i = minSeqId; i < maxSeqId; i++) { 948 Put put = new Put(Bytes.toBytes(i)); 949 put.addColumn(family, Bytes.toBytes(i), Bytes.toBytes(i)); 950 region.put(put); 951 region.flush(true); 952 } 953 954 // this will create a region with 3 files 955 assertEquals(3, region.getStore(family).getStorefilesCount()); 956 List<Path> storeFiles = new ArrayList<>(3); 957 for (HStoreFile sf : region.getStore(family).getStorefiles()) { 958 storeFiles.add(sf.getPath()); 959 } 960 961 // disable compaction completion 962 CONF.setBoolean("hbase.hstore.compaction.complete", false); 963 region.compactStores(); 964 965 // ensure that nothing changed 966 assertEquals(3, region.getStore(family).getStorefilesCount()); 967 968 // now find the compacted file, and manually add it to the recovered edits 969 Path tmpDir = new Path(region.getRegionFileSystem().getTempDir(), Bytes.toString(family)); 970 FileStatus[] files = CommonFSUtils.listStatus(fs, tmpDir); 971 String errorMsg = "Expected to find 1 file in the region temp directory " 972 + "from the compaction, could not find any"; 973 assertNotNull(errorMsg, files); 974 assertEquals(errorMsg, 1, files.length); 975 // move the file inside region dir 976 Path newFile = 977 region.getRegionFileSystem().commitStoreFile(Bytes.toString(family), files[0].getPath()); 978 979 byte[] encodedNameAsBytes = this.region.getRegionInfo().getEncodedNameAsBytes(); 980 byte[] fakeEncodedNameAsBytes = new byte[encodedNameAsBytes.length]; 981 for (int i = 0; i < encodedNameAsBytes.length; i++) { 982 // Mix the byte array to have a new encodedName 983 fakeEncodedNameAsBytes[i] = (byte) (encodedNameAsBytes[i] + 1); 984 } 985 986 CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor( 987 this.region.getRegionInfo(), mismatchedRegionName ? fakeEncodedNameAsBytes : null, family, 988 storeFiles, Lists.newArrayList(newFile), 989 region.getRegionFileSystem().getStoreDir(Bytes.toString(family))); 990 991 WALUtil.writeCompactionMarker(region.getWAL(), this.region.getReplicationScope(), 992 this.region.getRegionInfo(), compactionDescriptor, region.getMVCC(), null); 993 994 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 995 996 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000)); 997 fs.create(recoveredEdits); 998 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 999 1000 long time = System.nanoTime(); 1001 1002 writer.append(new WAL.Entry( 1003 new WALKeyImpl(regionName, tableName, 10, time, HConstants.DEFAULT_CLUSTER_ID), 1004 WALEdit.createCompaction(region.getRegionInfo(), compactionDescriptor))); 1005 writer.close(); 1006 1007 // close the region now, and reopen again 1008 region.getTableDescriptor(); 1009 region.getRegionInfo(); 1010 HBaseTestingUtil.closeRegionAndWAL(this.region); 1011 try { 1012 region = HRegion.openHRegion(region, null); 1013 } catch (WrongRegionException wre) { 1014 fail("Matching encoded region name should not have produced WrongRegionException"); 1015 } 1016 1017 // now check whether we have only one store file, the compacted one 1018 Collection<HStoreFile> sfs = region.getStore(family).getStorefiles(); 1019 for (HStoreFile sf : sfs) { 1020 LOG.info(Objects.toString(sf.getPath())); 1021 } 1022 if (!mismatchedRegionName) { 1023 assertEquals(1, region.getStore(family).getStorefilesCount()); 1024 } 1025 files = CommonFSUtils.listStatus(fs, tmpDir); 1026 assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0); 1027 1028 for (long i = minSeqId; i < maxSeqId; i++) { 1029 Get get = new Get(Bytes.toBytes(i)); 1030 Result result = region.get(get); 1031 byte[] value = result.getValue(family, Bytes.toBytes(i)); 1032 assertArrayEquals(Bytes.toBytes(i), value); 1033 } 1034 } finally { 1035 HBaseTestingUtil.closeRegionAndWAL(this.region); 1036 this.region = null; 1037 wals.close(); 1038 CONF.setClass(HConstants.REGION_IMPL, HRegion.class, Region.class); 1039 } 1040 } 1041 1042 @Test 1043 public void testFlushMarkers() throws Exception { 1044 // tests that flush markers are written to WAL and handled at recovered edits 1045 byte[] family = Bytes.toBytes("family"); 1046 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + ".log"); 1047 final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration()); 1048 CommonFSUtils.setRootDir(walConf, logDir); 1049 final WALFactory wals = new WALFactory(walConf, method); 1050 final WAL wal = wals.getWAL(RegionInfoBuilder.newBuilder(tableName).build()); 1051 1052 this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF, 1053 false, Durability.USE_DEFAULT, wal, family); 1054 try { 1055 Path regiondir = region.getRegionFileSystem().getRegionDir(); 1056 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 1057 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 1058 1059 long maxSeqId = 3; 1060 long minSeqId = 0; 1061 1062 for (long i = minSeqId; i < maxSeqId; i++) { 1063 Put put = new Put(Bytes.toBytes(i)); 1064 put.addColumn(family, Bytes.toBytes(i), Bytes.toBytes(i)); 1065 region.put(put); 1066 region.flush(true); 1067 } 1068 1069 // this will create a region with 3 files from flush 1070 assertEquals(3, region.getStore(family).getStorefilesCount()); 1071 List<String> storeFiles = new ArrayList<>(3); 1072 for (HStoreFile sf : region.getStore(family).getStorefiles()) { 1073 storeFiles.add(sf.getPath().getName()); 1074 } 1075 1076 // now verify that the flush markers are written 1077 wal.shutdown(); 1078 WALStreamReader reader = WALFactory.createStreamReader(fs, 1079 AbstractFSWALProvider.getCurrentFileName(wal), TEST_UTIL.getConfiguration()); 1080 try { 1081 List<WAL.Entry> flushDescriptors = new ArrayList<>(); 1082 long lastFlushSeqId = -1; 1083 while (true) { 1084 WAL.Entry entry = reader.next(); 1085 if (entry == null) { 1086 break; 1087 } 1088 Cell cell = entry.getEdit().getCells().get(0); 1089 if (WALEdit.isMetaEditFamily(cell)) { 1090 FlushDescriptor flushDesc = WALEdit.getFlushDescriptor(cell); 1091 assertNotNull(flushDesc); 1092 assertArrayEquals(tableName.getName(), flushDesc.getTableName().toByteArray()); 1093 if (flushDesc.getAction() == FlushAction.START_FLUSH) { 1094 assertTrue(flushDesc.getFlushSequenceNumber() > lastFlushSeqId); 1095 } else if (flushDesc.getAction() == FlushAction.COMMIT_FLUSH) { 1096 assertTrue(flushDesc.getFlushSequenceNumber() == lastFlushSeqId); 1097 } 1098 lastFlushSeqId = flushDesc.getFlushSequenceNumber(); 1099 assertArrayEquals(regionName, flushDesc.getEncodedRegionName().toByteArray()); 1100 assertEquals(1, flushDesc.getStoreFlushesCount()); // only one store 1101 StoreFlushDescriptor storeFlushDesc = flushDesc.getStoreFlushes(0); 1102 assertArrayEquals(family, storeFlushDesc.getFamilyName().toByteArray()); 1103 assertEquals("family", storeFlushDesc.getStoreHomeDir()); 1104 if (flushDesc.getAction() == FlushAction.START_FLUSH) { 1105 assertEquals(0, storeFlushDesc.getFlushOutputCount()); 1106 } else { 1107 assertEquals(1, storeFlushDesc.getFlushOutputCount()); // only one file from flush 1108 assertTrue(storeFiles.contains(storeFlushDesc.getFlushOutput(0))); 1109 } 1110 1111 flushDescriptors.add(entry); 1112 } 1113 } 1114 1115 assertEquals(3 * 2, flushDescriptors.size()); // START_FLUSH and COMMIT_FLUSH per flush 1116 1117 // now write those markers to the recovered edits again. 1118 1119 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 1120 1121 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000)); 1122 fs.create(recoveredEdits); 1123 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 1124 1125 for (WAL.Entry entry : flushDescriptors) { 1126 writer.append(entry); 1127 } 1128 writer.close(); 1129 } finally { 1130 reader.close(); 1131 } 1132 1133 // close the region now, and reopen again 1134 HBaseTestingUtil.closeRegionAndWAL(this.region); 1135 region = HRegion.openHRegion(region, null); 1136 1137 // now check whether we have can read back the data from region 1138 for (long i = minSeqId; i < maxSeqId; i++) { 1139 Get get = new Get(Bytes.toBytes(i)); 1140 Result result = region.get(get); 1141 byte[] value = result.getValue(family, Bytes.toBytes(i)); 1142 assertArrayEquals(Bytes.toBytes(i), value); 1143 } 1144 } finally { 1145 HBaseTestingUtil.closeRegionAndWAL(this.region); 1146 this.region = null; 1147 wals.close(); 1148 } 1149 } 1150 1151 static class IsFlushWALMarker implements ArgumentMatcher<WALEdit> { 1152 volatile FlushAction[] actions; 1153 1154 public IsFlushWALMarker(FlushAction... actions) { 1155 this.actions = actions; 1156 } 1157 1158 @Override 1159 public boolean matches(WALEdit edit) { 1160 List<Cell> cells = edit.getCells(); 1161 if (cells.isEmpty()) { 1162 return false; 1163 } 1164 if (WALEdit.isMetaEditFamily(cells.get(0))) { 1165 FlushDescriptor desc; 1166 try { 1167 desc = WALEdit.getFlushDescriptor(cells.get(0)); 1168 } catch (IOException e) { 1169 LOG.warn(e.toString(), e); 1170 return false; 1171 } 1172 if (desc != null) { 1173 for (FlushAction action : actions) { 1174 if (desc.getAction() == action) { 1175 return true; 1176 } 1177 } 1178 } 1179 } 1180 return false; 1181 } 1182 1183 public IsFlushWALMarker set(FlushAction... actions) { 1184 this.actions = actions; 1185 return this; 1186 } 1187 } 1188 1189 @Test 1190 public void testFlushMarkersWALFail() throws Exception { 1191 // test the cases where the WAL append for flush markers fail. 1192 byte[] family = Bytes.toBytes("family"); 1193 1194 // spy an actual WAL implementation to throw exception (was not able to mock) 1195 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + "log"); 1196 1197 final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration()); 1198 CommonFSUtils.setRootDir(walConf, logDir); 1199 // Make up a WAL that we can manipulate at append time. 1200 class FailAppendFlushMarkerWAL extends FSHLog { 1201 volatile FlushAction[] flushActions = null; 1202 1203 public FailAppendFlushMarkerWAL(FileSystem fs, Path root, String logDir, Configuration conf) 1204 throws IOException { 1205 super(fs, root, logDir, conf); 1206 } 1207 1208 @Override 1209 protected Writer createWriterInstance(FileSystem fs, Path path) throws IOException { 1210 final Writer w = super.createWriterInstance(fs, path); 1211 return new Writer() { 1212 @Override 1213 public void close() throws IOException { 1214 w.close(); 1215 } 1216 1217 @Override 1218 public void sync(boolean forceSync) throws IOException { 1219 w.sync(forceSync); 1220 } 1221 1222 @Override 1223 public void append(Entry entry) throws IOException { 1224 List<Cell> cells = entry.getEdit().getCells(); 1225 if (WALEdit.isMetaEditFamily(cells.get(0))) { 1226 FlushDescriptor desc = WALEdit.getFlushDescriptor(cells.get(0)); 1227 if (desc != null) { 1228 for (FlushAction flushAction : flushActions) { 1229 if (desc.getAction().equals(flushAction)) { 1230 throw new IOException("Failed to append flush marker! " + flushAction); 1231 } 1232 } 1233 } 1234 } 1235 w.append(entry); 1236 } 1237 1238 @Override 1239 public long getLength() { 1240 return w.getLength(); 1241 } 1242 1243 @Override 1244 public long getSyncedLength() { 1245 return w.getSyncedLength(); 1246 } 1247 }; 1248 } 1249 } 1250 FailAppendFlushMarkerWAL wal = new FailAppendFlushMarkerWAL(FileSystem.get(walConf), 1251 CommonFSUtils.getRootDir(walConf), method, walConf); 1252 wal.init(); 1253 this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF, 1254 false, Durability.USE_DEFAULT, wal, family); 1255 int i = 0; 1256 Put put = new Put(Bytes.toBytes(i)); 1257 put.setDurability(Durability.SKIP_WAL); // have to skip mocked wal 1258 put.addColumn(family, Bytes.toBytes(i), Bytes.toBytes(i)); 1259 region.put(put); 1260 1261 // 1. Test case where START_FLUSH throws exception 1262 wal.flushActions = new FlushAction[] { FlushAction.START_FLUSH }; 1263 1264 // start cache flush will throw exception 1265 try { 1266 region.flush(true); 1267 fail("This should have thrown exception"); 1268 } catch (DroppedSnapshotException unexpected) { 1269 // this should not be a dropped snapshot exception. Meaning that RS will not abort 1270 throw unexpected; 1271 } catch (IOException expected) { 1272 // expected 1273 } 1274 // The WAL is hosed now. It has two edits appended. We cannot roll the log without it 1275 // throwing a DroppedSnapshotException to force an abort. Just clean up the mess. 1276 region.close(true); 1277 wal.close(); 1278 // release the snapshot and active segment, so netty will not report memory leak 1279 for (HStore store : region.getStores()) { 1280 AbstractMemStore memstore = (AbstractMemStore) store.memstore; 1281 memstore.doClearSnapShot(); 1282 memstore.close(); 1283 } 1284 1285 // 2. Test case where START_FLUSH succeeds but COMMIT_FLUSH will throw exception 1286 wal.flushActions = new FlushAction[] { FlushAction.COMMIT_FLUSH }; 1287 wal = new FailAppendFlushMarkerWAL(FileSystem.get(walConf), CommonFSUtils.getRootDir(walConf), 1288 method, walConf); 1289 wal.init(); 1290 this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF, 1291 false, Durability.USE_DEFAULT, wal, family); 1292 region.put(put); 1293 // 3. Test case where ABORT_FLUSH will throw exception. 1294 // Even if ABORT_FLUSH throws exception, we should not fail with IOE, but continue with 1295 // DroppedSnapshotException. Below COMMIT_FLUSH will cause flush to abort 1296 wal.flushActions = new FlushAction[] { FlushAction.COMMIT_FLUSH, FlushAction.ABORT_FLUSH }; 1297 1298 // we expect this exception, since we were able to write the snapshot, but failed to 1299 // write the flush marker to WAL 1300 assertThrows(DroppedSnapshotException.class, () -> region.flush(true)); 1301 1302 region.close(true); 1303 // release the snapshot and active segment, so netty will not report memory leak 1304 for (HStore store : region.getStores()) { 1305 AbstractMemStore memstore = (AbstractMemStore) store.memstore; 1306 memstore.doClearSnapShot(); 1307 memstore.close(); 1308 } 1309 region = null; 1310 } 1311 1312 @Test 1313 public void testGetWhileRegionClose() throws IOException { 1314 Configuration hc = initSplit(); 1315 int numRows = 100; 1316 byte[][] families = { fam1, fam2, fam3 }; 1317 1318 // Setting up region 1319 this.region = initHRegion(tableName, method, hc, families); 1320 // Put data in region 1321 final int startRow = 100; 1322 putData(startRow, numRows, qual1, families); 1323 putData(startRow, numRows, qual2, families); 1324 putData(startRow, numRows, qual3, families); 1325 final AtomicBoolean done = new AtomicBoolean(false); 1326 final AtomicInteger gets = new AtomicInteger(0); 1327 GetTillDoneOrException[] threads = new GetTillDoneOrException[10]; 1328 try { 1329 // Set ten threads running concurrently getting from the region. 1330 for (int i = 0; i < threads.length / 2; i++) { 1331 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets); 1332 threads[i].setDaemon(true); 1333 threads[i].start(); 1334 } 1335 // Artificially make the condition by setting closing flag explicitly. 1336 // I can't make the issue happen with a call to region.close(). 1337 this.region.closing.set(true); 1338 for (int i = threads.length / 2; i < threads.length; i++) { 1339 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets); 1340 threads[i].setDaemon(true); 1341 threads[i].start(); 1342 } 1343 } finally { 1344 if (this.region != null) { 1345 HBaseTestingUtil.closeRegionAndWAL(this.region); 1346 this.region = null; 1347 } 1348 } 1349 done.set(true); 1350 for (GetTillDoneOrException t : threads) { 1351 try { 1352 t.join(); 1353 } catch (InterruptedException e) { 1354 e.printStackTrace(); 1355 } 1356 if (t.e != null) { 1357 LOG.info("Exception=" + t.e); 1358 assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException); 1359 } 1360 } 1361 } 1362 1363 /* 1364 * Thread that does get on single row until 'done' flag is flipped. If an exception causes us to 1365 * fail, it records it. 1366 */ 1367 class GetTillDoneOrException extends Thread { 1368 private final Get g; 1369 private final AtomicBoolean done; 1370 private final AtomicInteger count; 1371 private Exception e; 1372 1373 GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, 1374 final AtomicInteger c) { 1375 super("getter." + i); 1376 this.g = new Get(r); 1377 this.done = d; 1378 this.count = c; 1379 } 1380 1381 @Override 1382 public void run() { 1383 while (!this.done.get()) { 1384 try { 1385 assertTrue(region.get(g).size() > 0); 1386 this.count.incrementAndGet(); 1387 } catch (Exception e) { 1388 this.e = e; 1389 break; 1390 } 1391 } 1392 } 1393 } 1394 1395 /* 1396 * An involved filter test. Has multiple column families and deletes in mix. 1397 */ 1398 @Test 1399 public void testWeirdCacheBehaviour() throws Exception { 1400 final TableName tableName = TableName.valueOf(name.getMethodName()); 1401 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"), 1402 Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") }; 1403 this.region = initHRegion(tableName, method, CONF, FAMILIES); 1404 String value = "this is the value"; 1405 String value2 = "this is some other value"; 1406 String keyPrefix1 = "prefix1"; 1407 String keyPrefix2 = "prefix2"; 1408 String keyPrefix3 = "prefix3"; 1409 putRows(this.region, 3, value, keyPrefix1); 1410 putRows(this.region, 3, value, keyPrefix2); 1411 putRows(this.region, 3, value, keyPrefix3); 1412 putRows(this.region, 3, value2, keyPrefix1); 1413 putRows(this.region, 3, value2, keyPrefix2); 1414 putRows(this.region, 3, value2, keyPrefix3); 1415 System.out.println("Checking values for key: " + keyPrefix1); 1416 assertEquals("Got back incorrect number of rows from scan", 3, 1417 getNumberOfRows(keyPrefix1, value2, this.region)); 1418 System.out.println("Checking values for key: " + keyPrefix2); 1419 assertEquals("Got back incorrect number of rows from scan", 3, 1420 getNumberOfRows(keyPrefix2, value2, this.region)); 1421 System.out.println("Checking values for key: " + keyPrefix3); 1422 assertEquals("Got back incorrect number of rows from scan", 3, 1423 getNumberOfRows(keyPrefix3, value2, this.region)); 1424 deleteColumns(this.region, value2, keyPrefix1); 1425 deleteColumns(this.region, value2, keyPrefix2); 1426 deleteColumns(this.region, value2, keyPrefix3); 1427 System.out.println("Starting important checks....."); 1428 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0, 1429 getNumberOfRows(keyPrefix1, value2, this.region)); 1430 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0, 1431 getNumberOfRows(keyPrefix2, value2, this.region)); 1432 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0, 1433 getNumberOfRows(keyPrefix3, value2, this.region)); 1434 } 1435 1436 @Test 1437 public void testAppendWithReadOnlyTable() throws Exception { 1438 final TableName tableName = TableName.valueOf(name.getMethodName()); 1439 this.region = initHRegion(tableName, method, CONF, true, Bytes.toBytes("somefamily")); 1440 boolean exceptionCaught = false; 1441 Append append = new Append(Bytes.toBytes("somerow")); 1442 append.setDurability(Durability.SKIP_WAL); 1443 append.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1444 Bytes.toBytes("somevalue")); 1445 try { 1446 region.append(append); 1447 } catch (IOException e) { 1448 exceptionCaught = true; 1449 } 1450 assertTrue(exceptionCaught == true); 1451 } 1452 1453 @Test 1454 public void testIncrWithReadOnlyTable() throws Exception { 1455 final TableName tableName = TableName.valueOf(name.getMethodName()); 1456 this.region = initHRegion(tableName, method, CONF, true, Bytes.toBytes("somefamily")); 1457 boolean exceptionCaught = false; 1458 Increment inc = new Increment(Bytes.toBytes("somerow")); 1459 inc.setDurability(Durability.SKIP_WAL); 1460 inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L); 1461 try { 1462 region.increment(inc); 1463 } catch (IOException e) { 1464 exceptionCaught = true; 1465 } 1466 assertTrue(exceptionCaught == true); 1467 } 1468 1469 private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException { 1470 int count = 0; 1471 try (InternalScanner scanner = buildScanner(keyPrefix, value, r)) { 1472 boolean more = false; 1473 List<Cell> results = new ArrayList<>(); 1474 do { 1475 more = scanner.next(results); 1476 if (results != null && !results.isEmpty()) { 1477 count++; 1478 } else { 1479 break; 1480 } 1481 Delete delete = new Delete(CellUtil.cloneRow(results.get(0))); 1482 delete.addColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2")); 1483 r.delete(delete); 1484 results.clear(); 1485 } while (more); 1486 } 1487 assertEquals("Did not perform correct number of deletes", 3, count); 1488 } 1489 1490 private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception { 1491 try (InternalScanner resultScanner = buildScanner(keyPrefix, value, r)) { 1492 int numberOfResults = 0; 1493 List<Cell> results = new ArrayList<>(); 1494 boolean more = false; 1495 do { 1496 more = resultScanner.next(results); 1497 if (results != null && !results.isEmpty()) { 1498 numberOfResults++; 1499 } else { 1500 break; 1501 } 1502 for (Cell kv : results) { 1503 LOG.info("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv))); 1504 } 1505 results.clear(); 1506 } while (more); 1507 return numberOfResults; 1508 } 1509 } 1510 1511 private InternalScanner buildScanner(String keyPrefix, String value, HRegion r) 1512 throws IOException { 1513 // Defaults FilterList.Operator.MUST_PASS_ALL. 1514 FilterList allFilters = new FilterList(); 1515 allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix))); 1516 // Only return rows where this column value exists in the row. 1517 SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"), 1518 Bytes.toBytes("qual2"), CompareOperator.EQUAL, Bytes.toBytes(value)); 1519 filter.setFilterIfMissing(true); 1520 allFilters.addFilter(filter); 1521 Scan scan = new Scan(); 1522 scan.addFamily(Bytes.toBytes("trans-blob")); 1523 scan.addFamily(Bytes.toBytes("trans-type")); 1524 scan.addFamily(Bytes.toBytes("trans-date")); 1525 scan.addFamily(Bytes.toBytes("trans-tags")); 1526 scan.addFamily(Bytes.toBytes("trans-group")); 1527 scan.setFilter(allFilters); 1528 return r.getScanner(scan); 1529 } 1530 1531 private void putRows(HRegion r, int numRows, String value, String key) throws IOException { 1532 for (int i = 0; i < numRows; i++) { 1533 String row = key + "_" + i/* UUID.randomUUID().toString() */; 1534 System.out.println(String.format("Saving row: %s, with value %s", row, value)); 1535 Put put = new Put(Bytes.toBytes(row)); 1536 put.setDurability(Durability.SKIP_WAL); 1537 put.addColumn(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob")); 1538 put.addColumn(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement")); 1539 put.addColumn(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999")); 1540 put.addColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value)); 1541 put.addColumn(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId")); 1542 r.put(put); 1543 } 1544 } 1545 1546 @Test 1547 public void testFamilyWithAndWithoutColon() throws Exception { 1548 byte[] cf = Bytes.toBytes(COLUMN_FAMILY); 1549 this.region = initHRegion(tableName, method, CONF, cf); 1550 Put p = new Put(tableName.toBytes()); 1551 byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":"); 1552 p.addColumn(cfwithcolon, cfwithcolon, cfwithcolon); 1553 boolean exception = false; 1554 try { 1555 this.region.put(p); 1556 } catch (NoSuchColumnFamilyException e) { 1557 exception = true; 1558 } 1559 assertTrue(exception); 1560 } 1561 1562 @Test 1563 public void testBatchPut_whileNoRowLocksHeld() throws IOException { 1564 final Put[] puts = new Put[10]; 1565 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); 1566 long syncs = prepareRegionForBachPut(puts, source, false); 1567 1568 OperationStatus[] codes = this.region.batchMutate(puts); 1569 assertEquals(10, codes.length); 1570 for (int i = 0; i < 10; i++) { 1571 assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode()); 1572 } 1573 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source); 1574 1575 LOG.info("Next a batch put with one invalid family"); 1576 puts[5].addColumn(Bytes.toBytes("BAD_CF"), qual, value); 1577 codes = this.region.batchMutate(puts); 1578 assertEquals(10, codes.length); 1579 for (int i = 0; i < 10; i++) { 1580 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS, 1581 codes[i].getOperationStatusCode()); 1582 } 1583 1584 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source); 1585 } 1586 1587 @Test 1588 public void testBatchPut_whileMultipleRowLocksHeld() throws Exception { 1589 final Put[] puts = new Put[10]; 1590 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); 1591 long syncs = prepareRegionForBachPut(puts, source, false); 1592 1593 puts[5].addColumn(Bytes.toBytes("BAD_CF"), qual, value); 1594 1595 LOG.info("batchPut will have to break into four batches to avoid row locks"); 1596 RowLock rowLock1 = region.getRowLock(Bytes.toBytes("row_2")); 1597 RowLock rowLock2 = region.getRowLock(Bytes.toBytes("row_1")); 1598 RowLock rowLock3 = region.getRowLock(Bytes.toBytes("row_3")); 1599 RowLock rowLock4 = region.getRowLock(Bytes.toBytes("row_3"), true); 1600 1601 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF); 1602 final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<>(); 1603 final CountDownLatch startingPuts = new CountDownLatch(1); 1604 final CountDownLatch startingClose = new CountDownLatch(1); 1605 TestThread putter = new TestThread(ctx) { 1606 @Override 1607 public void doWork() throws IOException { 1608 startingPuts.countDown(); 1609 retFromThread.set(region.batchMutate(puts)); 1610 } 1611 }; 1612 LOG.info("...starting put thread while holding locks"); 1613 ctx.addThread(putter); 1614 ctx.startThreads(); 1615 1616 // Now attempt to close the region from another thread. Prior to HBASE-12565 1617 // this would cause the in-progress batchMutate operation to to fail with 1618 // exception because it use to release and re-acquire the close-guard lock 1619 // between batches. Caller then didn't get status indicating which writes succeeded. 1620 // We now expect this thread to block until the batchMutate call finishes. 1621 Thread regionCloseThread = new TestThread(ctx) { 1622 @Override 1623 public void doWork() { 1624 try { 1625 startingPuts.await(); 1626 // Give some time for the batch mutate to get in. 1627 // We don't want to race with the mutate 1628 Thread.sleep(10); 1629 startingClose.countDown(); 1630 HBaseTestingUtil.closeRegionAndWAL(region); 1631 region = null; 1632 } catch (IOException e) { 1633 throw new RuntimeException(e); 1634 } catch (InterruptedException e) { 1635 throw new RuntimeException(e); 1636 } 1637 } 1638 }; 1639 regionCloseThread.start(); 1640 1641 startingClose.await(); 1642 startingPuts.await(); 1643 Thread.sleep(100); 1644 LOG.info("...releasing row lock 1, which should let put thread continue"); 1645 rowLock1.release(); 1646 rowLock2.release(); 1647 rowLock3.release(); 1648 waitForCounter(source, "syncTimeNumOps", syncs + 1); 1649 1650 LOG.info("...joining on put thread"); 1651 ctx.stop(); 1652 regionCloseThread.join(); 1653 1654 OperationStatus[] codes = retFromThread.get(); 1655 for (int i = 0; i < codes.length; i++) { 1656 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS, 1657 codes[i].getOperationStatusCode()); 1658 } 1659 rowLock4.release(); 1660 } 1661 1662 private void waitForCounter(MetricsWALSource source, String metricName, long expectedCount) 1663 throws InterruptedException { 1664 long startWait = EnvironmentEdgeManager.currentTime(); 1665 long currentCount; 1666 while ((currentCount = metricsAssertHelper.getCounter(metricName, source)) < expectedCount) { 1667 Thread.sleep(100); 1668 if (EnvironmentEdgeManager.currentTime() - startWait > 10000) { 1669 fail(String.format("Timed out waiting for '%s' >= '%s', currentCount=%s", metricName, 1670 expectedCount, currentCount)); 1671 } 1672 } 1673 } 1674 1675 @Test 1676 public void testAtomicBatchPut() throws IOException { 1677 final Put[] puts = new Put[10]; 1678 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); 1679 long syncs = prepareRegionForBachPut(puts, source, false); 1680 1681 // 1. Straight forward case, should succeed 1682 OperationStatus[] codes = this.region.batchMutate(puts, true); 1683 assertEquals(10, codes.length); 1684 for (int i = 0; i < 10; i++) { 1685 assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode()); 1686 } 1687 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source); 1688 1689 // 2. Failed to get lock 1690 RowLock lock = region.getRowLock(Bytes.toBytes("row_" + 3)); 1691 // Method {@link HRegion#getRowLock(byte[])} is reentrant. As 'row_3' is locked in this 1692 // thread, need to run {@link HRegion#batchMutate(HRegion.BatchOperation)} in different thread 1693 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF); 1694 final AtomicReference<IOException> retFromThread = new AtomicReference<>(); 1695 final CountDownLatch finishedPuts = new CountDownLatch(1); 1696 TestThread putter = new TestThread(ctx) { 1697 @Override 1698 public void doWork() throws IOException { 1699 try { 1700 region.batchMutate(puts, true); 1701 } catch (IOException ioe) { 1702 LOG.error("test failed!", ioe); 1703 retFromThread.set(ioe); 1704 } 1705 finishedPuts.countDown(); 1706 } 1707 }; 1708 LOG.info("...starting put thread while holding locks"); 1709 ctx.addThread(putter); 1710 ctx.startThreads(); 1711 LOG.info("...waiting for batch puts while holding locks"); 1712 try { 1713 finishedPuts.await(); 1714 } catch (InterruptedException e) { 1715 LOG.error("Interrupted!", e); 1716 } finally { 1717 if (lock != null) { 1718 lock.release(); 1719 } 1720 } 1721 assertNotNull(retFromThread.get()); 1722 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source); 1723 1724 // 3. Exception thrown in validation 1725 LOG.info("Next a batch put with one invalid family"); 1726 puts[5].addColumn(Bytes.toBytes("BAD_CF"), qual, value); 1727 thrown.expect(NoSuchColumnFamilyException.class); 1728 this.region.batchMutate(puts, true); 1729 } 1730 1731 @Test 1732 public void testBatchPutWithTsSlop() throws Exception { 1733 // add data with a timestamp that is too recent for range. Ensure assert 1734 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000); 1735 final Put[] puts = new Put[10]; 1736 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); 1737 1738 long syncs = prepareRegionForBachPut(puts, source, true); 1739 1740 OperationStatus[] codes = this.region.batchMutate(puts); 1741 assertEquals(10, codes.length); 1742 for (int i = 0; i < 10; i++) { 1743 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode()); 1744 } 1745 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source); 1746 } 1747 1748 /** Returns syncs initial syncTimeNumOps */ 1749 private long prepareRegionForBachPut(final Put[] puts, final MetricsWALSource source, 1750 boolean slop) throws IOException { 1751 this.region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES); 1752 1753 LOG.info("First a batch put with all valid puts"); 1754 for (int i = 0; i < puts.length; i++) { 1755 puts[i] = slop 1756 ? new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100) 1757 : new Put(Bytes.toBytes("row_" + i)); 1758 puts[i].addColumn(COLUMN_FAMILY_BYTES, qual, value); 1759 } 1760 1761 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source); 1762 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source); 1763 return syncs; 1764 } 1765 1766 // //////////////////////////////////////////////////////////////////////////// 1767 // checkAndMutate tests 1768 // //////////////////////////////////////////////////////////////////////////// 1769 @Test 1770 @Deprecated 1771 public void testCheckAndMutate_WithEmptyRowValue() throws IOException { 1772 byte[] row1 = Bytes.toBytes("row1"); 1773 byte[] fam1 = Bytes.toBytes("fam1"); 1774 byte[] qf1 = Bytes.toBytes("qualifier"); 1775 byte[] emptyVal = new byte[] {}; 1776 byte[] val1 = Bytes.toBytes("value1"); 1777 byte[] val2 = Bytes.toBytes("value2"); 1778 1779 // Setting up region 1780 this.region = initHRegion(tableName, method, CONF, fam1); 1781 // Putting empty data in key 1782 Put put = new Put(row1); 1783 put.addColumn(fam1, qf1, emptyVal); 1784 1785 // checkAndPut with empty value 1786 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1787 new BinaryComparator(emptyVal), put); 1788 assertTrue(res); 1789 1790 // Putting data in key 1791 put = new Put(row1); 1792 put.addColumn(fam1, qf1, val1); 1793 1794 // checkAndPut with correct value 1795 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1796 new BinaryComparator(emptyVal), put); 1797 assertTrue(res); 1798 1799 // not empty anymore 1800 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1801 new BinaryComparator(emptyVal), put); 1802 assertFalse(res); 1803 1804 Delete delete = new Delete(row1); 1805 delete.addColumn(fam1, qf1); 1806 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1807 new BinaryComparator(emptyVal), delete); 1808 assertFalse(res); 1809 1810 put = new Put(row1); 1811 put.addColumn(fam1, qf1, val2); 1812 // checkAndPut with correct value 1813 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val1), 1814 put); 1815 assertTrue(res); 1816 1817 // checkAndDelete with correct value 1818 delete = new Delete(row1); 1819 delete.addColumn(fam1, qf1); 1820 delete.addColumn(fam1, qf1); 1821 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val2), 1822 delete); 1823 assertTrue(res); 1824 1825 delete = new Delete(row1); 1826 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1827 new BinaryComparator(emptyVal), delete); 1828 assertTrue(res); 1829 1830 // checkAndPut looking for a null value 1831 put = new Put(row1); 1832 put.addColumn(fam1, qf1, val1); 1833 1834 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new NullComparator(), put); 1835 assertTrue(res); 1836 } 1837 1838 @Test 1839 @Deprecated 1840 public void testCheckAndMutate_WithWrongValue() throws IOException { 1841 byte[] row1 = Bytes.toBytes("row1"); 1842 byte[] fam1 = Bytes.toBytes("fam1"); 1843 byte[] qf1 = Bytes.toBytes("qualifier"); 1844 byte[] val1 = Bytes.toBytes("value1"); 1845 byte[] val2 = Bytes.toBytes("value2"); 1846 BigDecimal bd1 = new BigDecimal(Double.MAX_VALUE); 1847 BigDecimal bd2 = new BigDecimal(Double.MIN_VALUE); 1848 1849 // Setting up region 1850 this.region = initHRegion(tableName, method, CONF, fam1); 1851 // Putting data in key 1852 Put put = new Put(row1); 1853 put.addColumn(fam1, qf1, val1); 1854 region.put(put); 1855 1856 // checkAndPut with wrong value 1857 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1858 new BinaryComparator(val2), put); 1859 assertEquals(false, res); 1860 1861 // checkAndDelete with wrong value 1862 Delete delete = new Delete(row1); 1863 delete.addFamily(fam1); 1864 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val2), 1865 put); 1866 assertEquals(false, res); 1867 1868 // Putting data in key 1869 put = new Put(row1); 1870 put.addColumn(fam1, qf1, Bytes.toBytes(bd1)); 1871 region.put(put); 1872 1873 // checkAndPut with wrong value 1874 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1875 new BigDecimalComparator(bd2), put); 1876 assertEquals(false, res); 1877 1878 // checkAndDelete with wrong value 1879 delete = new Delete(row1); 1880 delete.addFamily(fam1); 1881 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1882 new BigDecimalComparator(bd2), put); 1883 assertEquals(false, res); 1884 } 1885 1886 @Test 1887 @Deprecated 1888 public void testCheckAndMutate_WithCorrectValue() throws IOException { 1889 byte[] row1 = Bytes.toBytes("row1"); 1890 byte[] fam1 = Bytes.toBytes("fam1"); 1891 byte[] qf1 = Bytes.toBytes("qualifier"); 1892 byte[] val1 = Bytes.toBytes("value1"); 1893 BigDecimal bd1 = new BigDecimal(Double.MIN_VALUE); 1894 1895 // Setting up region 1896 this.region = initHRegion(tableName, method, CONF, fam1); 1897 // Putting data in key 1898 long now = EnvironmentEdgeManager.currentTime(); 1899 Put put = new Put(row1); 1900 put.addColumn(fam1, qf1, now, val1); 1901 region.put(put); 1902 1903 // checkAndPut with correct value 1904 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1905 new BinaryComparator(val1), put); 1906 assertEquals("First", true, res); 1907 1908 // checkAndDelete with correct value 1909 Delete delete = new Delete(row1, now + 1); 1910 delete.addColumn(fam1, qf1); 1911 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val1), 1912 delete); 1913 assertEquals("Delete", true, res); 1914 1915 // Putting data in key 1916 put = new Put(row1); 1917 put.addColumn(fam1, qf1, now + 2, Bytes.toBytes(bd1)); 1918 region.put(put); 1919 1920 // checkAndPut with correct value 1921 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1922 new BigDecimalComparator(bd1), put); 1923 assertEquals("Second put", true, res); 1924 1925 // checkAndDelete with correct value 1926 delete = new Delete(row1, now + 3); 1927 delete.addColumn(fam1, qf1); 1928 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1929 new BigDecimalComparator(bd1), delete); 1930 assertEquals("Second delete", true, res); 1931 } 1932 1933 @Test 1934 @Deprecated 1935 public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException { 1936 byte[] row1 = Bytes.toBytes("row1"); 1937 byte[] fam1 = Bytes.toBytes("fam1"); 1938 byte[] qf1 = Bytes.toBytes("qualifier"); 1939 byte[] val1 = Bytes.toBytes("value1"); 1940 byte[] val2 = Bytes.toBytes("value2"); 1941 byte[] val3 = Bytes.toBytes("value3"); 1942 byte[] val4 = Bytes.toBytes("value4"); 1943 1944 // Setting up region 1945 this.region = initHRegion(tableName, method, CONF, fam1); 1946 // Putting val3 in key 1947 Put put = new Put(row1); 1948 put.addColumn(fam1, qf1, val3); 1949 region.put(put); 1950 1951 // Test CompareOp.LESS: original = val3, compare with val3, fail 1952 boolean res = 1953 region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS, new BinaryComparator(val3), put); 1954 assertEquals(false, res); 1955 1956 // Test CompareOp.LESS: original = val3, compare with val4, fail 1957 res = 1958 region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS, new BinaryComparator(val4), put); 1959 assertEquals(false, res); 1960 1961 // Test CompareOp.LESS: original = val3, compare with val2, 1962 // succeed (now value = val2) 1963 put = new Put(row1); 1964 put.addColumn(fam1, qf1, val2); 1965 res = 1966 region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS, new BinaryComparator(val2), put); 1967 assertEquals(true, res); 1968 1969 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val3, fail 1970 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS_OR_EQUAL, 1971 new BinaryComparator(val3), put); 1972 assertEquals(false, res); 1973 1974 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val2, 1975 // succeed (value still = val2) 1976 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS_OR_EQUAL, 1977 new BinaryComparator(val2), put); 1978 assertEquals(true, res); 1979 1980 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val1, 1981 // succeed (now value = val3) 1982 put = new Put(row1); 1983 put.addColumn(fam1, qf1, val3); 1984 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS_OR_EQUAL, 1985 new BinaryComparator(val1), put); 1986 assertEquals(true, res); 1987 1988 // Test CompareOp.GREATER: original = val3, compare with val3, fail 1989 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER, 1990 new BinaryComparator(val3), put); 1991 assertEquals(false, res); 1992 1993 // Test CompareOp.GREATER: original = val3, compare with val2, fail 1994 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER, 1995 new BinaryComparator(val2), put); 1996 assertEquals(false, res); 1997 1998 // Test CompareOp.GREATER: original = val3, compare with val4, 1999 // succeed (now value = val2) 2000 put = new Put(row1); 2001 put.addColumn(fam1, qf1, val2); 2002 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER, 2003 new BinaryComparator(val4), put); 2004 assertEquals(true, res); 2005 2006 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val1, fail 2007 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER_OR_EQUAL, 2008 new BinaryComparator(val1), put); 2009 assertEquals(false, res); 2010 2011 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val2, 2012 // succeed (value still = val2) 2013 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER_OR_EQUAL, 2014 new BinaryComparator(val2), put); 2015 assertEquals(true, res); 2016 2017 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val3, succeed 2018 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER_OR_EQUAL, 2019 new BinaryComparator(val3), put); 2020 assertEquals(true, res); 2021 } 2022 2023 @Test 2024 @Deprecated 2025 public void testCheckAndPut_ThatPutWasWritten() throws IOException { 2026 byte[] row1 = Bytes.toBytes("row1"); 2027 byte[] fam1 = Bytes.toBytes("fam1"); 2028 byte[] fam2 = Bytes.toBytes("fam2"); 2029 byte[] qf1 = Bytes.toBytes("qualifier"); 2030 byte[] val1 = Bytes.toBytes("value1"); 2031 byte[] val2 = Bytes.toBytes("value2"); 2032 2033 byte[][] families = { fam1, fam2 }; 2034 2035 // Setting up region 2036 this.region = initHRegion(tableName, method, CONF, families); 2037 // Putting data in the key to check 2038 Put put = new Put(row1); 2039 put.addColumn(fam1, qf1, val1); 2040 region.put(put); 2041 2042 // Creating put to add 2043 long ts = EnvironmentEdgeManager.currentTime(); 2044 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2); 2045 put = new Put(row1); 2046 put.add(kv); 2047 2048 // checkAndPut with wrong value 2049 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 2050 new BinaryComparator(val1), put); 2051 assertEquals(true, res); 2052 2053 Get get = new Get(row1); 2054 get.addColumn(fam2, qf1); 2055 Cell[] actual = region.get(get).rawCells(); 2056 2057 Cell[] expected = { kv }; 2058 2059 assertEquals(expected.length, actual.length); 2060 for (int i = 0; i < actual.length; i++) { 2061 assertEquals(expected[i], actual[i]); 2062 } 2063 } 2064 2065 @Test 2066 @Deprecated 2067 public void testCheckAndPut_wrongRowInPut() throws IOException { 2068 this.region = initHRegion(tableName, method, CONF, COLUMNS); 2069 Put put = new Put(row2); 2070 put.addColumn(fam1, qual1, value1); 2071 try { 2072 region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(value2), 2073 put); 2074 fail(); 2075 } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) { 2076 // expected exception. 2077 } 2078 } 2079 2080 @Test 2081 @Deprecated 2082 public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException { 2083 byte[] row1 = Bytes.toBytes("row1"); 2084 byte[] fam1 = Bytes.toBytes("fam1"); 2085 byte[] fam2 = Bytes.toBytes("fam2"); 2086 byte[] qf1 = Bytes.toBytes("qualifier1"); 2087 byte[] qf2 = Bytes.toBytes("qualifier2"); 2088 byte[] qf3 = Bytes.toBytes("qualifier3"); 2089 byte[] val1 = Bytes.toBytes("value1"); 2090 byte[] val2 = Bytes.toBytes("value2"); 2091 byte[] val3 = Bytes.toBytes("value3"); 2092 byte[] emptyVal = new byte[] {}; 2093 2094 byte[][] families = { fam1, fam2 }; 2095 2096 // Setting up region 2097 this.region = initHRegion(tableName, method, CONF, families); 2098 // Put content 2099 Put put = new Put(row1); 2100 put.addColumn(fam1, qf1, val1); 2101 region.put(put); 2102 Threads.sleep(2); 2103 2104 put = new Put(row1); 2105 put.addColumn(fam1, qf1, val2); 2106 put.addColumn(fam2, qf1, val3); 2107 put.addColumn(fam2, qf2, val2); 2108 put.addColumn(fam2, qf3, val1); 2109 put.addColumn(fam1, qf3, val1); 2110 region.put(put); 2111 2112 LOG.info("get={}", region.get(new Get(row1).addColumn(fam1, qf1)).toString()); 2113 2114 // Multi-column delete 2115 Delete delete = new Delete(row1); 2116 delete.addColumn(fam1, qf1); 2117 delete.addColumn(fam2, qf1); 2118 delete.addColumn(fam1, qf3); 2119 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 2120 new BinaryComparator(val2), delete); 2121 assertEquals(true, res); 2122 2123 Get get = new Get(row1); 2124 get.addColumn(fam1, qf1); 2125 get.addColumn(fam1, qf3); 2126 get.addColumn(fam2, qf2); 2127 Result r = region.get(get); 2128 assertEquals(2, r.size()); 2129 assertArrayEquals(val1, r.getValue(fam1, qf1)); 2130 assertArrayEquals(val2, r.getValue(fam2, qf2)); 2131 2132 // Family delete 2133 delete = new Delete(row1); 2134 delete.addFamily(fam2); 2135 res = region.checkAndMutate(row1, fam2, qf1, CompareOperator.EQUAL, 2136 new BinaryComparator(emptyVal), delete); 2137 assertEquals(true, res); 2138 2139 get = new Get(row1); 2140 r = region.get(get); 2141 assertEquals(1, r.size()); 2142 assertArrayEquals(val1, r.getValue(fam1, qf1)); 2143 2144 // Row delete 2145 delete = new Delete(row1); 2146 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val1), 2147 delete); 2148 assertEquals(true, res); 2149 get = new Get(row1); 2150 r = region.get(get); 2151 assertEquals(0, r.size()); 2152 } 2153 2154 @Test 2155 @Deprecated 2156 public void testCheckAndMutate_WithFilters() throws Throwable { 2157 final byte[] FAMILY = Bytes.toBytes("fam"); 2158 2159 // Setting up region 2160 this.region = initHRegion(tableName, method, CONF, FAMILY); 2161 2162 // Put one row 2163 Put put = new Put(row); 2164 put.addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")); 2165 put.addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")); 2166 put.addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")); 2167 region.put(put); 2168 2169 // Put with success 2170 boolean ok = region.checkAndMutate(row, 2171 new FilterList( 2172 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2173 Bytes.toBytes("a")), 2174 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2175 Bytes.toBytes("b"))), 2176 new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d"))); 2177 assertTrue(ok); 2178 2179 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))); 2180 assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D")))); 2181 2182 // Put with failure 2183 ok = region.checkAndMutate(row, 2184 new FilterList( 2185 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2186 Bytes.toBytes("a")), 2187 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2188 Bytes.toBytes("c"))), 2189 new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e"))); 2190 assertFalse(ok); 2191 2192 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E"))).isEmpty()); 2193 2194 // Delete with success 2195 ok = region.checkAndMutate(row, 2196 new FilterList( 2197 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2198 Bytes.toBytes("a")), 2199 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2200 Bytes.toBytes("b"))), 2201 new Delete(row).addColumns(FAMILY, Bytes.toBytes("D"))); 2202 assertTrue(ok); 2203 2204 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))).isEmpty()); 2205 2206 // Mutate with success 2207 ok = region.checkAndRowMutate(row, 2208 new FilterList( 2209 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2210 Bytes.toBytes("a")), 2211 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2212 Bytes.toBytes("b"))), 2213 new RowMutations(row) 2214 .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e"))) 2215 .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A")))); 2216 assertTrue(ok); 2217 2218 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E"))); 2219 assertEquals("e", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("E")))); 2220 2221 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty()); 2222 } 2223 2224 @Test 2225 @Deprecated 2226 public void testCheckAndMutate_WithFiltersAndTimeRange() throws Throwable { 2227 final byte[] FAMILY = Bytes.toBytes("fam"); 2228 2229 // Setting up region 2230 this.region = initHRegion(tableName, method, CONF, FAMILY); 2231 2232 // Put with specifying the timestamp 2233 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), 100, Bytes.toBytes("a"))); 2234 2235 // Put with success 2236 boolean ok = region.checkAndMutate(row, 2237 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2238 Bytes.toBytes("a")), 2239 TimeRange.between(0, 101), 2240 new Put(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b"))); 2241 assertTrue(ok); 2242 2243 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2244 assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2245 2246 // Put with failure 2247 ok = region.checkAndMutate(row, 2248 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2249 Bytes.toBytes("a")), 2250 TimeRange.between(0, 100), 2251 new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c"))); 2252 assertFalse(ok); 2253 2254 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("C"))).isEmpty()); 2255 2256 // Mutate with success 2257 ok = region.checkAndRowMutate(row, 2258 new SingleColumnValueFilter( 2259 FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, Bytes.toBytes("a")), 2260 TimeRange.between(0, 101), 2261 new RowMutations(row) 2262 .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d"))) 2263 .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A")))); 2264 assertTrue(ok); 2265 2266 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))); 2267 assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D")))); 2268 2269 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty()); 2270 } 2271 2272 @Test 2273 @Deprecated 2274 public void testCheckAndMutate_wrongMutationType() throws Throwable { 2275 // Setting up region 2276 this.region = initHRegion(tableName, method, CONF, fam1); 2277 2278 try { 2279 region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(value1), 2280 new Increment(row).addColumn(fam1, qual1, 1)); 2281 fail("should throw DoNotRetryIOException"); 2282 } catch (DoNotRetryIOException e) { 2283 assertEquals("Unsupported mutate type: INCREMENT", e.getMessage()); 2284 } 2285 2286 try { 2287 region.checkAndMutate(row, 2288 new SingleColumnValueFilter(fam1, qual1, CompareOperator.EQUAL, value1), 2289 new Increment(row).addColumn(fam1, qual1, 1)); 2290 fail("should throw DoNotRetryIOException"); 2291 } catch (DoNotRetryIOException e) { 2292 assertEquals("Unsupported mutate type: INCREMENT", e.getMessage()); 2293 } 2294 } 2295 2296 @Test 2297 @Deprecated 2298 public void testCheckAndMutate_wrongRow() throws Throwable { 2299 final byte[] wrongRow = Bytes.toBytes("wrongRow"); 2300 2301 // Setting up region 2302 this.region = initHRegion(tableName, method, CONF, fam1); 2303 2304 try { 2305 region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(value1), 2306 new Put(wrongRow).addColumn(fam1, qual1, value1)); 2307 fail("should throw DoNotRetryIOException"); 2308 } catch (DoNotRetryIOException e) { 2309 assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>", 2310 e.getMessage()); 2311 } 2312 2313 try { 2314 region.checkAndMutate(row, 2315 new SingleColumnValueFilter(fam1, qual1, CompareOperator.EQUAL, value1), 2316 new Put(wrongRow).addColumn(fam1, qual1, value1)); 2317 fail("should throw DoNotRetryIOException"); 2318 } catch (DoNotRetryIOException e) { 2319 assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>", 2320 e.getMessage()); 2321 } 2322 2323 try { 2324 region.checkAndRowMutate(row, fam1, qual1, CompareOperator.EQUAL, 2325 new BinaryComparator(value1), 2326 new RowMutations(wrongRow).add((Mutation) new Put(wrongRow).addColumn(fam1, qual1, value1)) 2327 .add((Mutation) new Delete(wrongRow).addColumns(fam1, qual2))); 2328 fail("should throw DoNotRetryIOException"); 2329 } catch (DoNotRetryIOException e) { 2330 assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>", 2331 e.getMessage()); 2332 } 2333 2334 try { 2335 region.checkAndRowMutate(row, 2336 new SingleColumnValueFilter(fam1, qual1, CompareOperator.EQUAL, value1), 2337 new RowMutations(wrongRow).add((Mutation) new Put(wrongRow).addColumn(fam1, qual1, value1)) 2338 .add((Mutation) new Delete(wrongRow).addColumns(fam1, qual2))); 2339 fail("should throw DoNotRetryIOException"); 2340 } catch (DoNotRetryIOException e) { 2341 assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>", 2342 e.getMessage()); 2343 } 2344 } 2345 2346 @Test 2347 public void testCheckAndMutateWithEmptyRowValue() throws IOException { 2348 byte[] row1 = Bytes.toBytes("row1"); 2349 byte[] fam1 = Bytes.toBytes("fam1"); 2350 byte[] qf1 = Bytes.toBytes("qualifier"); 2351 byte[] emptyVal = new byte[] {}; 2352 byte[] val1 = Bytes.toBytes("value1"); 2353 byte[] val2 = Bytes.toBytes("value2"); 2354 2355 // Setting up region 2356 this.region = initHRegion(tableName, method, CONF, fam1); 2357 // Putting empty data in key 2358 Put put = new Put(row1); 2359 put.addColumn(fam1, qf1, emptyVal); 2360 2361 // checkAndPut with empty value 2362 CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2363 .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(put)); 2364 assertTrue(res.isSuccess()); 2365 assertNull(res.getResult()); 2366 2367 // Putting data in key 2368 put = new Put(row1); 2369 put.addColumn(fam1, qf1, val1); 2370 2371 // checkAndPut with correct value 2372 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2373 .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(put)); 2374 assertTrue(res.isSuccess()); 2375 assertNull(res.getResult()); 2376 2377 // not empty anymore 2378 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2379 .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(put)); 2380 assertFalse(res.isSuccess()); 2381 assertNull(res.getResult()); 2382 2383 Delete delete = new Delete(row1); 2384 delete.addColumn(fam1, qf1); 2385 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2386 .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(delete)); 2387 assertFalse(res.isSuccess()); 2388 assertNull(res.getResult()); 2389 2390 put = new Put(row1); 2391 put.addColumn(fam1, qf1, val2); 2392 // checkAndPut with correct value 2393 res = region.checkAndMutate( 2394 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(put)); 2395 assertTrue(res.isSuccess()); 2396 assertNull(res.getResult()); 2397 2398 // checkAndDelete with correct value 2399 delete = new Delete(row1); 2400 delete.addColumn(fam1, qf1); 2401 delete.addColumn(fam1, qf1); 2402 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2403 .ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(delete)); 2404 assertTrue(res.isSuccess()); 2405 assertNull(res.getResult()); 2406 2407 delete = new Delete(row1); 2408 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2409 .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(delete)); 2410 assertTrue(res.isSuccess()); 2411 assertNull(res.getResult()); 2412 2413 // checkAndPut looking for a null value 2414 put = new Put(row1); 2415 put.addColumn(fam1, qf1, val1); 2416 2417 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1).ifNotExists(fam1, qf1).build(put)); 2418 assertTrue(res.isSuccess()); 2419 assertNull(res.getResult()); 2420 } 2421 2422 @Test 2423 public void testCheckAndMutateWithWrongValue() throws IOException { 2424 byte[] row1 = Bytes.toBytes("row1"); 2425 byte[] fam1 = Bytes.toBytes("fam1"); 2426 byte[] qf1 = Bytes.toBytes("qualifier"); 2427 byte[] val1 = Bytes.toBytes("value1"); 2428 byte[] val2 = Bytes.toBytes("value2"); 2429 BigDecimal bd1 = new BigDecimal(Double.MAX_VALUE); 2430 BigDecimal bd2 = new BigDecimal(Double.MIN_VALUE); 2431 2432 // Setting up region 2433 this.region = initHRegion(tableName, method, CONF, fam1); 2434 // Putting data in key 2435 Put put = new Put(row1); 2436 put.addColumn(fam1, qf1, val1); 2437 region.put(put); 2438 2439 // checkAndPut with wrong value 2440 CheckAndMutateResult res = region.checkAndMutate( 2441 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(put)); 2442 assertFalse(res.isSuccess()); 2443 assertNull(res.getResult()); 2444 2445 // checkAndDelete with wrong value 2446 Delete delete = new Delete(row1); 2447 delete.addFamily(fam1); 2448 res = region.checkAndMutate( 2449 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(put)); 2450 assertFalse(res.isSuccess()); 2451 assertNull(res.getResult()); 2452 2453 // Putting data in key 2454 put = new Put(row1); 2455 put.addColumn(fam1, qf1, Bytes.toBytes(bd1)); 2456 region.put(put); 2457 2458 // checkAndPut with wrong value 2459 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2460 .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd2)).build(put)); 2461 assertFalse(res.isSuccess()); 2462 assertNull(res.getResult()); 2463 2464 // checkAndDelete with wrong value 2465 delete = new Delete(row1); 2466 delete.addFamily(fam1); 2467 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2468 .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd2)).build(delete)); 2469 assertFalse(res.isSuccess()); 2470 assertNull(res.getResult()); 2471 } 2472 2473 @Test 2474 public void testCheckAndMutateWithCorrectValue() throws IOException { 2475 byte[] row1 = Bytes.toBytes("row1"); 2476 byte[] fam1 = Bytes.toBytes("fam1"); 2477 byte[] qf1 = Bytes.toBytes("qualifier"); 2478 byte[] val1 = Bytes.toBytes("value1"); 2479 BigDecimal bd1 = new BigDecimal(Double.MIN_VALUE); 2480 2481 // Setting up region 2482 this.region = initHRegion(tableName, method, CONF, fam1); 2483 // Putting data in key 2484 long now = EnvironmentEdgeManager.currentTime(); 2485 Put put = new Put(row1); 2486 put.addColumn(fam1, qf1, now, val1); 2487 region.put(put); 2488 2489 // checkAndPut with correct value 2490 CheckAndMutateResult res = region.checkAndMutate( 2491 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(put)); 2492 assertTrue("First", res.isSuccess()); 2493 2494 // checkAndDelete with correct value 2495 Delete delete = new Delete(row1, now + 1); 2496 delete.addColumn(fam1, qf1); 2497 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2498 .ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(delete)); 2499 assertTrue("Delete", res.isSuccess()); 2500 assertNull(res.getResult()); 2501 2502 // Putting data in key 2503 put = new Put(row1); 2504 put.addColumn(fam1, qf1, now + 2, Bytes.toBytes(bd1)); 2505 region.put(put); 2506 2507 // checkAndPut with correct value 2508 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2509 .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd1)).build(put)); 2510 assertTrue("Second put", res.isSuccess()); 2511 assertNull(res.getResult()); 2512 2513 // checkAndDelete with correct value 2514 delete = new Delete(row1, now + 3); 2515 delete.addColumn(fam1, qf1); 2516 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2517 .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd1)).build(delete)); 2518 assertTrue("Second delete", res.isSuccess()); 2519 assertNull(res.getResult()); 2520 } 2521 2522 @Test 2523 public void testCheckAndMutateWithNonEqualCompareOp() throws IOException { 2524 byte[] row1 = Bytes.toBytes("row1"); 2525 byte[] fam1 = Bytes.toBytes("fam1"); 2526 byte[] qf1 = Bytes.toBytes("qualifier"); 2527 byte[] val1 = Bytes.toBytes("value1"); 2528 byte[] val2 = Bytes.toBytes("value2"); 2529 byte[] val3 = Bytes.toBytes("value3"); 2530 byte[] val4 = Bytes.toBytes("value4"); 2531 2532 // Setting up region 2533 this.region = initHRegion(tableName, method, CONF, fam1); 2534 // Putting val3 in key 2535 Put put = new Put(row1); 2536 put.addColumn(fam1, qf1, val3); 2537 region.put(put); 2538 2539 // Test CompareOp.LESS: original = val3, compare with val3, fail 2540 CheckAndMutateResult res = region.checkAndMutate( 2541 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.LESS, val3).build(put)); 2542 assertFalse(res.isSuccess()); 2543 assertNull(res.getResult()); 2544 2545 // Test CompareOp.LESS: original = val3, compare with val4, fail 2546 res = region.checkAndMutate( 2547 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.LESS, val4).build(put)); 2548 assertFalse(res.isSuccess()); 2549 assertNull(res.getResult()); 2550 2551 // Test CompareOp.LESS: original = val3, compare with val2, 2552 // succeed (now value = val2) 2553 put = new Put(row1); 2554 put.addColumn(fam1, qf1, val2); 2555 res = region.checkAndMutate( 2556 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.LESS, val2).build(put)); 2557 assertTrue(res.isSuccess()); 2558 assertNull(res.getResult()); 2559 2560 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val3, fail 2561 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2562 .ifMatches(fam1, qf1, CompareOperator.LESS_OR_EQUAL, val3).build(put)); 2563 assertFalse(res.isSuccess()); 2564 assertNull(res.getResult()); 2565 2566 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val2, 2567 // succeed (value still = val2) 2568 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2569 .ifMatches(fam1, qf1, CompareOperator.LESS_OR_EQUAL, val2).build(put)); 2570 assertTrue(res.isSuccess()); 2571 assertNull(res.getResult()); 2572 2573 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val1, 2574 // succeed (now value = val3) 2575 put = new Put(row1); 2576 put.addColumn(fam1, qf1, val3); 2577 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2578 .ifMatches(fam1, qf1, CompareOperator.LESS_OR_EQUAL, val1).build(put)); 2579 assertTrue(res.isSuccess()); 2580 assertNull(res.getResult()); 2581 2582 // Test CompareOp.GREATER: original = val3, compare with val3, fail 2583 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2584 .ifMatches(fam1, qf1, CompareOperator.GREATER, val3).build(put)); 2585 assertFalse(res.isSuccess()); 2586 assertNull(res.getResult()); 2587 2588 // Test CompareOp.GREATER: original = val3, compare with val2, fail 2589 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2590 .ifMatches(fam1, qf1, CompareOperator.GREATER, val2).build(put)); 2591 assertFalse(res.isSuccess()); 2592 assertNull(res.getResult()); 2593 2594 // Test CompareOp.GREATER: original = val3, compare with val4, 2595 // succeed (now value = val2) 2596 put = new Put(row1); 2597 put.addColumn(fam1, qf1, val2); 2598 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2599 .ifMatches(fam1, qf1, CompareOperator.GREATER, val4).build(put)); 2600 assertTrue(res.isSuccess()); 2601 assertNull(res.getResult()); 2602 2603 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val1, fail 2604 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2605 .ifMatches(fam1, qf1, CompareOperator.GREATER_OR_EQUAL, val1).build(put)); 2606 assertFalse(res.isSuccess()); 2607 assertNull(res.getResult()); 2608 2609 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val2, 2610 // succeed (value still = val2) 2611 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2612 .ifMatches(fam1, qf1, CompareOperator.GREATER_OR_EQUAL, val2).build(put)); 2613 assertTrue(res.isSuccess()); 2614 assertNull(res.getResult()); 2615 2616 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val3, succeed 2617 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2618 .ifMatches(fam1, qf1, CompareOperator.GREATER_OR_EQUAL, val3).build(put)); 2619 assertTrue(res.isSuccess()); 2620 assertNull(res.getResult()); 2621 } 2622 2623 @Test 2624 public void testCheckAndPutThatPutWasWritten() throws IOException { 2625 byte[] row1 = Bytes.toBytes("row1"); 2626 byte[] fam1 = Bytes.toBytes("fam1"); 2627 byte[] fam2 = Bytes.toBytes("fam2"); 2628 byte[] qf1 = Bytes.toBytes("qualifier"); 2629 byte[] val1 = Bytes.toBytes("value1"); 2630 byte[] val2 = Bytes.toBytes("value2"); 2631 2632 byte[][] families = { fam1, fam2 }; 2633 2634 // Setting up region 2635 this.region = initHRegion(tableName, method, CONF, families); 2636 // Putting data in the key to check 2637 Put put = new Put(row1); 2638 put.addColumn(fam1, qf1, val1); 2639 region.put(put); 2640 2641 // Creating put to add 2642 long ts = EnvironmentEdgeManager.currentTime(); 2643 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2); 2644 put = new Put(row1); 2645 put.add(kv); 2646 2647 // checkAndPut with wrong value 2648 CheckAndMutateResult res = region.checkAndMutate( 2649 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(put)); 2650 assertTrue(res.isSuccess()); 2651 assertNull(res.getResult()); 2652 2653 Get get = new Get(row1); 2654 get.addColumn(fam2, qf1); 2655 Cell[] actual = region.get(get).rawCells(); 2656 2657 Cell[] expected = { kv }; 2658 2659 assertEquals(expected.length, actual.length); 2660 for (int i = 0; i < actual.length; i++) { 2661 assertEquals(expected[i], actual[i]); 2662 } 2663 } 2664 2665 @Test 2666 public void testCheckAndDeleteThatDeleteWasWritten() throws IOException { 2667 byte[] row1 = Bytes.toBytes("row1"); 2668 byte[] fam1 = Bytes.toBytes("fam1"); 2669 byte[] fam2 = Bytes.toBytes("fam2"); 2670 byte[] qf1 = Bytes.toBytes("qualifier1"); 2671 byte[] qf2 = Bytes.toBytes("qualifier2"); 2672 byte[] qf3 = Bytes.toBytes("qualifier3"); 2673 byte[] val1 = Bytes.toBytes("value1"); 2674 byte[] val2 = Bytes.toBytes("value2"); 2675 byte[] val3 = Bytes.toBytes("value3"); 2676 byte[] emptyVal = new byte[] {}; 2677 2678 byte[][] families = { fam1, fam2 }; 2679 2680 // Setting up region 2681 this.region = initHRegion(tableName, method, CONF, families); 2682 // Put content 2683 Put put = new Put(row1); 2684 put.addColumn(fam1, qf1, val1); 2685 region.put(put); 2686 Threads.sleep(2); 2687 2688 put = new Put(row1); 2689 put.addColumn(fam1, qf1, val2); 2690 put.addColumn(fam2, qf1, val3); 2691 put.addColumn(fam2, qf2, val2); 2692 put.addColumn(fam2, qf3, val1); 2693 put.addColumn(fam1, qf3, val1); 2694 region.put(put); 2695 2696 LOG.info("get={}", region.get(new Get(row1).addColumn(fam1, qf1)).toString()); 2697 2698 // Multi-column delete 2699 Delete delete = new Delete(row1); 2700 delete.addColumn(fam1, qf1); 2701 delete.addColumn(fam2, qf1); 2702 delete.addColumn(fam1, qf3); 2703 CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2704 .ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(delete)); 2705 assertTrue(res.isSuccess()); 2706 assertNull(res.getResult()); 2707 2708 Get get = new Get(row1); 2709 get.addColumn(fam1, qf1); 2710 get.addColumn(fam1, qf3); 2711 get.addColumn(fam2, qf2); 2712 Result r = region.get(get); 2713 assertEquals(2, r.size()); 2714 assertArrayEquals(val1, r.getValue(fam1, qf1)); 2715 assertArrayEquals(val2, r.getValue(fam2, qf2)); 2716 2717 // Family delete 2718 delete = new Delete(row1); 2719 delete.addFamily(fam2); 2720 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2721 .ifMatches(fam2, qf1, CompareOperator.EQUAL, emptyVal).build(delete)); 2722 assertTrue(res.isSuccess()); 2723 assertNull(res.getResult()); 2724 2725 get = new Get(row1); 2726 r = region.get(get); 2727 assertEquals(1, r.size()); 2728 assertArrayEquals(val1, r.getValue(fam1, qf1)); 2729 2730 // Row delete 2731 delete = new Delete(row1); 2732 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2733 .ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(delete)); 2734 assertTrue(res.isSuccess()); 2735 assertNull(res.getResult()); 2736 2737 get = new Get(row1); 2738 r = region.get(get); 2739 assertEquals(0, r.size()); 2740 } 2741 2742 @Test 2743 public void testCheckAndMutateWithFilters() throws Throwable { 2744 final byte[] FAMILY = Bytes.toBytes("fam"); 2745 2746 // Setting up region 2747 this.region = initHRegion(tableName, method, CONF, FAMILY); 2748 2749 // Put one row 2750 Put put = new Put(row); 2751 put.addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")); 2752 put.addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")); 2753 put.addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")); 2754 region.put(put); 2755 2756 // Put with success 2757 CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2758 .ifMatches(new FilterList( 2759 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2760 Bytes.toBytes("a")), 2761 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2762 Bytes.toBytes("b")))) 2763 .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d")))); 2764 assertTrue(res.isSuccess()); 2765 assertNull(res.getResult()); 2766 2767 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))); 2768 assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D")))); 2769 2770 // Put with failure 2771 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2772 .ifMatches(new FilterList( 2773 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2774 Bytes.toBytes("a")), 2775 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2776 Bytes.toBytes("c")))) 2777 .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e")))); 2778 assertFalse(res.isSuccess()); 2779 assertNull(res.getResult()); 2780 2781 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E"))).isEmpty()); 2782 2783 // Delete with success 2784 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2785 .ifMatches(new FilterList( 2786 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2787 Bytes.toBytes("a")), 2788 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2789 Bytes.toBytes("b")))) 2790 .build(new Delete(row).addColumns(FAMILY, Bytes.toBytes("D")))); 2791 assertTrue(res.isSuccess()); 2792 assertNull(res.getResult()); 2793 2794 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))).isEmpty()); 2795 2796 // Mutate with success 2797 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2798 .ifMatches(new FilterList( 2799 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2800 Bytes.toBytes("a")), 2801 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2802 Bytes.toBytes("b")))) 2803 .build(new RowMutations(row) 2804 .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e"))) 2805 .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A"))))); 2806 assertTrue(res.isSuccess()); 2807 assertNull(res.getResult()); 2808 2809 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E"))); 2810 assertEquals("e", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("E")))); 2811 2812 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty()); 2813 } 2814 2815 @Test 2816 public void testCheckAndMutateWithFiltersAndTimeRange() throws Throwable { 2817 final byte[] FAMILY = Bytes.toBytes("fam"); 2818 2819 // Setting up region 2820 this.region = initHRegion(tableName, method, CONF, FAMILY); 2821 2822 // Put with specifying the timestamp 2823 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), 100, Bytes.toBytes("a"))); 2824 2825 // Put with success 2826 CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2827 .ifMatches(new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2828 Bytes.toBytes("a"))) 2829 .timeRange(TimeRange.between(0, 101)) 2830 .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")))); 2831 assertTrue(res.isSuccess()); 2832 assertNull(res.getResult()); 2833 2834 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2835 assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2836 2837 // Put with failure 2838 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2839 .ifMatches(new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2840 Bytes.toBytes("a"))) 2841 .timeRange(TimeRange.between(0, 100)) 2842 .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")))); 2843 assertFalse(res.isSuccess()); 2844 assertNull(res.getResult()); 2845 2846 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("C"))).isEmpty()); 2847 2848 // RowMutations with success 2849 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2850 .ifMatches(new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2851 Bytes.toBytes("a"))) 2852 .timeRange(TimeRange.between(0, 101)) 2853 .build(new RowMutations(row) 2854 .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d"))) 2855 .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A"))))); 2856 assertTrue(res.isSuccess()); 2857 assertNull(res.getResult()); 2858 2859 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))); 2860 assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D")))); 2861 2862 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty()); 2863 } 2864 2865 @Test 2866 public void testCheckAndIncrement() throws Throwable { 2867 final byte[] FAMILY = Bytes.toBytes("fam"); 2868 2869 // Setting up region 2870 this.region = initHRegion(tableName, method, CONF, FAMILY); 2871 2872 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a"))); 2873 2874 // CheckAndIncrement with correct value 2875 CheckAndMutateResult res = region.checkAndMutate( 2876 CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")) 2877 .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 1))); 2878 assertTrue(res.isSuccess()); 2879 assertEquals(1, Bytes.toLong(res.getResult().getValue(FAMILY, Bytes.toBytes("B")))); 2880 2881 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2882 assertEquals(1, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B")))); 2883 2884 // CheckAndIncrement with wrong value 2885 res = region.checkAndMutate( 2886 CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("b")) 2887 .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 1))); 2888 assertFalse(res.isSuccess()); 2889 assertNull(res.getResult()); 2890 2891 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2892 assertEquals(1, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B")))); 2893 2894 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c"))); 2895 2896 // CheckAndIncrement with a filter and correct value 2897 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2898 .ifMatches(new FilterList( 2899 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2900 Bytes.toBytes("a")), 2901 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL, 2902 Bytes.toBytes("c")))) 2903 .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 2))); 2904 assertTrue(res.isSuccess()); 2905 assertEquals(3, Bytes.toLong(res.getResult().getValue(FAMILY, Bytes.toBytes("B")))); 2906 2907 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2908 assertEquals(3, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B")))); 2909 2910 // CheckAndIncrement with a filter and correct value 2911 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2912 .ifMatches(new FilterList( 2913 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2914 Bytes.toBytes("b")), 2915 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL, 2916 Bytes.toBytes("d")))) 2917 .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 2))); 2918 assertFalse(res.isSuccess()); 2919 assertNull(res.getResult()); 2920 2921 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2922 assertEquals(3, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B")))); 2923 } 2924 2925 @Test 2926 public void testCheckAndAppend() throws Throwable { 2927 final byte[] FAMILY = Bytes.toBytes("fam"); 2928 2929 // Setting up region 2930 this.region = initHRegion(tableName, method, CONF, FAMILY); 2931 2932 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a"))); 2933 2934 // CheckAndAppend with correct value 2935 CheckAndMutateResult res = region.checkAndMutate( 2936 CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")) 2937 .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")))); 2938 assertTrue(res.isSuccess()); 2939 assertEquals("b", Bytes.toString(res.getResult().getValue(FAMILY, Bytes.toBytes("B")))); 2940 2941 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2942 assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2943 2944 // CheckAndAppend with wrong value 2945 res = region.checkAndMutate( 2946 CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("b")) 2947 .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")))); 2948 assertFalse(res.isSuccess()); 2949 assertNull(res.getResult()); 2950 2951 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2952 assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2953 2954 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c"))); 2955 2956 // CheckAndAppend with a filter and correct value 2957 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2958 .ifMatches(new FilterList( 2959 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2960 Bytes.toBytes("a")), 2961 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL, 2962 Bytes.toBytes("c")))) 2963 .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("bb")))); 2964 assertTrue(res.isSuccess()); 2965 assertEquals("bbb", Bytes.toString(res.getResult().getValue(FAMILY, Bytes.toBytes("B")))); 2966 2967 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2968 assertEquals("bbb", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2969 2970 // CheckAndAppend with a filter and wrong value 2971 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2972 .ifMatches(new FilterList( 2973 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2974 Bytes.toBytes("b")), 2975 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL, 2976 Bytes.toBytes("d")))) 2977 .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("bb")))); 2978 assertFalse(res.isSuccess()); 2979 assertNull(res.getResult()); 2980 2981 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2982 assertEquals("bbb", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2983 } 2984 2985 @Test 2986 public void testCheckAndIncrementAndAppend() throws Throwable { 2987 // Setting up region 2988 this.region = initHRegion(tableName, method, CONF, fam1); 2989 2990 // CheckAndMutate with Increment and Append 2991 CheckAndMutate checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, qual) 2992 .build(new RowMutations(row).add((Mutation) new Increment(row).addColumn(fam1, qual1, 1L)) 2993 .add((Mutation) new Append(row).addColumn(fam1, qual2, Bytes.toBytes("a")))); 2994 2995 CheckAndMutateResult result = region.checkAndMutate(checkAndMutate); 2996 assertTrue(result.isSuccess()); 2997 assertEquals(1L, Bytes.toLong(result.getResult().getValue(fam1, qual1))); 2998 assertEquals("a", Bytes.toString(result.getResult().getValue(fam1, qual2))); 2999 3000 Result r = region.get(new Get(row)); 3001 assertEquals(1L, Bytes.toLong(r.getValue(fam1, qual1))); 3002 assertEquals("a", Bytes.toString(r.getValue(fam1, qual2))); 3003 3004 // Set return results to false 3005 checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, qual) 3006 .build(new RowMutations(row) 3007 .add((Mutation) new Increment(row).addColumn(fam1, qual1, 1L).setReturnResults(false)) 3008 .add((Mutation) new Append(row).addColumn(fam1, qual2, Bytes.toBytes("a")) 3009 .setReturnResults(false))); 3010 3011 result = region.checkAndMutate(checkAndMutate); 3012 assertTrue(result.isSuccess()); 3013 assertNull(result.getResult().getValue(fam1, qual1)); 3014 assertNull(result.getResult().getValue(fam1, qual2)); 3015 3016 r = region.get(new Get(row)); 3017 assertEquals(2L, Bytes.toLong(r.getValue(fam1, qual1))); 3018 assertEquals("aa", Bytes.toString(r.getValue(fam1, qual2))); 3019 3020 checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, qual) 3021 .build(new RowMutations(row).add((Mutation) new Increment(row).addColumn(fam1, qual1, 1L)) 3022 .add((Mutation) new Append(row).addColumn(fam1, qual2, Bytes.toBytes("a")) 3023 .setReturnResults(false))); 3024 3025 result = region.checkAndMutate(checkAndMutate); 3026 assertTrue(result.isSuccess()); 3027 assertEquals(3L, Bytes.toLong(result.getResult().getValue(fam1, qual1))); 3028 assertNull(result.getResult().getValue(fam1, qual2)); 3029 3030 r = region.get(new Get(row)); 3031 assertEquals(3L, Bytes.toLong(r.getValue(fam1, qual1))); 3032 assertEquals("aaa", Bytes.toString(r.getValue(fam1, qual2))); 3033 } 3034 3035 @Test 3036 public void testCheckAndRowMutations() throws Throwable { 3037 final byte[] row = Bytes.toBytes("row"); 3038 final byte[] q1 = Bytes.toBytes("q1"); 3039 final byte[] q2 = Bytes.toBytes("q2"); 3040 final byte[] q3 = Bytes.toBytes("q3"); 3041 final byte[] q4 = Bytes.toBytes("q4"); 3042 final String v1 = "v1"; 3043 3044 region = initHRegion(tableName, method, CONF, fam1); 3045 3046 // Initial values 3047 region 3048 .batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q2, Bytes.toBytes("toBeDeleted")), 3049 new Put(row).addColumn(fam1, q3, Bytes.toBytes(5L)), 3050 new Put(row).addColumn(fam1, q4, Bytes.toBytes("a")), }); 3051 3052 // Do CheckAndRowMutations 3053 CheckAndMutate checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, q1).build( 3054 new RowMutations(row).add(Arrays.asList(new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1)), 3055 new Delete(row).addColumns(fam1, q2), new Increment(row).addColumn(fam1, q3, 1), 3056 new Append(row).addColumn(fam1, q4, Bytes.toBytes("b"))))); 3057 3058 CheckAndMutateResult result = region.checkAndMutate(checkAndMutate); 3059 assertTrue(result.isSuccess()); 3060 assertEquals(6L, Bytes.toLong(result.getResult().getValue(fam1, q3))); 3061 assertEquals("ab", Bytes.toString(result.getResult().getValue(fam1, q4))); 3062 3063 // Verify the value 3064 Result r = region.get(new Get(row)); 3065 assertEquals(v1, Bytes.toString(r.getValue(fam1, q1))); 3066 assertNull(r.getValue(fam1, q2)); 3067 assertEquals(6L, Bytes.toLong(r.getValue(fam1, q3))); 3068 assertEquals("ab", Bytes.toString(r.getValue(fam1, q4))); 3069 3070 // Do CheckAndRowMutations again 3071 checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, q1) 3072 .build(new RowMutations(row).add(Arrays.asList(new Delete(row).addColumns(fam1, q1), 3073 new Put(row).addColumn(fam1, q2, Bytes.toBytes(v1)), 3074 new Increment(row).addColumn(fam1, q3, 1), 3075 new Append(row).addColumn(fam1, q4, Bytes.toBytes("b"))))); 3076 3077 result = region.checkAndMutate(checkAndMutate); 3078 assertFalse(result.isSuccess()); 3079 assertNull(result.getResult()); 3080 3081 // Verify the value 3082 r = region.get(new Get(row)); 3083 assertEquals(v1, Bytes.toString(r.getValue(fam1, q1))); 3084 assertNull(r.getValue(fam1, q2)); 3085 assertEquals(6L, Bytes.toLong(r.getValue(fam1, q3))); 3086 assertEquals("ab", Bytes.toString(r.getValue(fam1, q4))); 3087 } 3088 3089 // //////////////////////////////////////////////////////////////////////////// 3090 // Delete tests 3091 // //////////////////////////////////////////////////////////////////////////// 3092 @Test 3093 public void testDelete_multiDeleteColumn() throws IOException { 3094 byte[] row1 = Bytes.toBytes("row1"); 3095 byte[] fam1 = Bytes.toBytes("fam1"); 3096 byte[] qual = Bytes.toBytes("qualifier"); 3097 byte[] value = Bytes.toBytes("value"); 3098 3099 Put put = new Put(row1); 3100 put.addColumn(fam1, qual, 1, value); 3101 put.addColumn(fam1, qual, 2, value); 3102 3103 this.region = initHRegion(tableName, method, CONF, fam1); 3104 region.put(put); 3105 3106 // We do support deleting more than 1 'latest' version 3107 Delete delete = new Delete(row1); 3108 delete.addColumn(fam1, qual); 3109 delete.addColumn(fam1, qual); 3110 region.delete(delete); 3111 3112 Get get = new Get(row1); 3113 get.addFamily(fam1); 3114 Result r = region.get(get); 3115 assertEquals(0, r.size()); 3116 } 3117 3118 @Test 3119 public void testDelete_CheckFamily() throws IOException { 3120 byte[] row1 = Bytes.toBytes("row1"); 3121 byte[] fam1 = Bytes.toBytes("fam1"); 3122 byte[] fam2 = Bytes.toBytes("fam2"); 3123 byte[] fam3 = Bytes.toBytes("fam3"); 3124 byte[] fam4 = Bytes.toBytes("fam4"); 3125 3126 // Setting up region 3127 this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3); 3128 List<Cell> kvs = new ArrayList<>(); 3129 kvs.add(new KeyValue(row1, fam4, null, null)); 3130 3131 byte[] forUnitTestsOnly = Bytes.toBytes("ForUnitTestsOnly"); 3132 3133 // testing existing family 3134 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<>(Bytes.BYTES_COMPARATOR); 3135 deleteMap.put(fam2, kvs); 3136 region.delete(new Delete(forUnitTestsOnly, HConstants.LATEST_TIMESTAMP, deleteMap)); 3137 3138 // testing non existing family 3139 NavigableMap<byte[], List<Cell>> deleteMap2 = new TreeMap<>(Bytes.BYTES_COMPARATOR); 3140 deleteMap2.put(fam4, kvs); 3141 assertThrows("Family " + Bytes.toString(fam4) + " does exist", 3142 NoSuchColumnFamilyException.class, 3143 () -> region.delete(new Delete(forUnitTestsOnly, HConstants.LATEST_TIMESTAMP, deleteMap2))); 3144 } 3145 3146 @Test 3147 public void testDelete_mixed() throws IOException, InterruptedException { 3148 byte[] fam = Bytes.toBytes("info"); 3149 byte[][] families = { fam }; 3150 this.region = initHRegion(tableName, method, CONF, families); 3151 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge()); 3152 3153 byte[] row = Bytes.toBytes("table_name"); 3154 // column names 3155 byte[] serverinfo = Bytes.toBytes("serverinfo"); 3156 byte[] splitA = Bytes.toBytes("splitA"); 3157 byte[] splitB = Bytes.toBytes("splitB"); 3158 3159 // add some data: 3160 Put put = new Put(row); 3161 put.addColumn(fam, splitA, Bytes.toBytes("reference_A")); 3162 region.put(put); 3163 3164 put = new Put(row); 3165 put.addColumn(fam, splitB, Bytes.toBytes("reference_B")); 3166 region.put(put); 3167 3168 put = new Put(row); 3169 put.addColumn(fam, serverinfo, Bytes.toBytes("ip_address")); 3170 region.put(put); 3171 3172 // ok now delete a split: 3173 Delete delete = new Delete(row); 3174 delete.addColumns(fam, splitA); 3175 region.delete(delete); 3176 3177 // assert some things: 3178 Get get = new Get(row).addColumn(fam, serverinfo); 3179 Result result = region.get(get); 3180 assertEquals(1, result.size()); 3181 3182 get = new Get(row).addColumn(fam, splitA); 3183 result = region.get(get); 3184 assertEquals(0, result.size()); 3185 3186 get = new Get(row).addColumn(fam, splitB); 3187 result = region.get(get); 3188 assertEquals(1, result.size()); 3189 3190 // Assert that after a delete, I can put. 3191 put = new Put(row); 3192 put.addColumn(fam, splitA, Bytes.toBytes("reference_A")); 3193 region.put(put); 3194 get = new Get(row); 3195 result = region.get(get); 3196 assertEquals(3, result.size()); 3197 3198 // Now delete all... then test I can add stuff back 3199 delete = new Delete(row); 3200 region.delete(delete); 3201 assertEquals(0, region.get(get).size()); 3202 3203 region.put(new Put(row).addColumn(fam, splitA, Bytes.toBytes("reference_A"))); 3204 result = region.get(get); 3205 assertEquals(1, result.size()); 3206 } 3207 3208 @Test 3209 public void testDeleteRowWithFutureTs() throws IOException { 3210 byte[] fam = Bytes.toBytes("info"); 3211 byte[][] families = { fam }; 3212 this.region = initHRegion(tableName, method, CONF, families); 3213 byte[] row = Bytes.toBytes("table_name"); 3214 // column names 3215 byte[] serverinfo = Bytes.toBytes("serverinfo"); 3216 3217 // add data in the far future 3218 Put put = new Put(row); 3219 put.addColumn(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value")); 3220 region.put(put); 3221 3222 // now delete something in the present 3223 Delete delete = new Delete(row); 3224 region.delete(delete); 3225 3226 // make sure we still see our data 3227 Get get = new Get(row).addColumn(fam, serverinfo); 3228 Result result = region.get(get); 3229 assertEquals(1, result.size()); 3230 3231 // delete the future row 3232 delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3); 3233 region.delete(delete); 3234 3235 // make sure it is gone 3236 get = new Get(row).addColumn(fam, serverinfo); 3237 result = region.get(get); 3238 assertEquals(0, result.size()); 3239 } 3240 3241 /** 3242 * Tests that the special LATEST_TIMESTAMP option for puts gets replaced by the actual timestamp 3243 */ 3244 @Test 3245 public void testPutWithLatestTS() throws IOException { 3246 byte[] fam = Bytes.toBytes("info"); 3247 byte[][] families = { fam }; 3248 this.region = initHRegion(tableName, method, CONF, families); 3249 byte[] row = Bytes.toBytes("row1"); 3250 // column names 3251 byte[] qual = Bytes.toBytes("qual"); 3252 3253 // add data with LATEST_TIMESTAMP, put without WAL 3254 Put put = new Put(row); 3255 put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value")); 3256 region.put(put); 3257 3258 // Make sure it shows up with an actual timestamp 3259 Get get = new Get(row).addColumn(fam, qual); 3260 Result result = region.get(get); 3261 assertEquals(1, result.size()); 3262 Cell kv = result.rawCells()[0]; 3263 LOG.info("Got: " + kv); 3264 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp", 3265 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP); 3266 3267 // Check same with WAL enabled (historically these took different 3268 // code paths, so check both) 3269 row = Bytes.toBytes("row2"); 3270 put = new Put(row); 3271 put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value")); 3272 region.put(put); 3273 3274 // Make sure it shows up with an actual timestamp 3275 get = new Get(row).addColumn(fam, qual); 3276 result = region.get(get); 3277 assertEquals(1, result.size()); 3278 kv = result.rawCells()[0]; 3279 LOG.info("Got: " + kv); 3280 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp", 3281 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP); 3282 } 3283 3284 /** 3285 * Tests that there is server-side filtering for invalid timestamp upper bound. Note that the 3286 * timestamp lower bound is automatically handled for us by the TTL field. 3287 */ 3288 @Test 3289 public void testPutWithTsSlop() throws IOException { 3290 byte[] fam = Bytes.toBytes("info"); 3291 byte[][] families = { fam }; 3292 3293 // add data with a timestamp that is too recent for range. Ensure assert 3294 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000); 3295 this.region = initHRegion(tableName, method, CONF, families); 3296 boolean caughtExcep = false; 3297 try { 3298 // no TS specified == use latest. should not error 3299 region.put(new Put(row).addColumn(fam, Bytes.toBytes("qual"), Bytes.toBytes("value"))); 3300 // TS out of range. should error 3301 region.put(new Put(row).addColumn(fam, Bytes.toBytes("qual"), 3302 EnvironmentEdgeManager.currentTime() + 2000, Bytes.toBytes("value"))); 3303 fail("Expected IOE for TS out of configured timerange"); 3304 } catch (FailedSanityCheckException ioe) { 3305 LOG.debug("Received expected exception", ioe); 3306 caughtExcep = true; 3307 } 3308 assertTrue("Should catch FailedSanityCheckException", caughtExcep); 3309 } 3310 3311 @Test 3312 public void testScanner_DeleteOneFamilyNotAnother() throws IOException { 3313 byte[] fam1 = Bytes.toBytes("columnA"); 3314 byte[] fam2 = Bytes.toBytes("columnB"); 3315 this.region = initHRegion(tableName, method, CONF, fam1, fam2); 3316 byte[] rowA = Bytes.toBytes("rowA"); 3317 byte[] rowB = Bytes.toBytes("rowB"); 3318 3319 byte[] value = Bytes.toBytes("value"); 3320 3321 Delete delete = new Delete(rowA); 3322 delete.addFamily(fam1); 3323 3324 region.delete(delete); 3325 3326 // now create data. 3327 Put put = new Put(rowA); 3328 put.addColumn(fam2, null, value); 3329 region.put(put); 3330 3331 put = new Put(rowB); 3332 put.addColumn(fam1, null, value); 3333 put.addColumn(fam2, null, value); 3334 region.put(put); 3335 3336 Scan scan = new Scan(); 3337 scan.addFamily(fam1).addFamily(fam2); 3338 try (InternalScanner s = region.getScanner(scan)) { 3339 List<Cell> results = new ArrayList<>(); 3340 s.next(results); 3341 assertTrue(CellUtil.matchingRows(results.get(0), rowA)); 3342 3343 results.clear(); 3344 s.next(results); 3345 assertTrue(CellUtil.matchingRows(results.get(0), rowB)); 3346 } 3347 } 3348 3349 @Test 3350 public void testDataInMemoryWithoutWAL() throws IOException { 3351 FileSystem fs = FileSystem.get(CONF); 3352 Path rootDir = new Path(dir + "testDataInMemoryWithoutWAL"); 3353 FSHLog hLog = new FSHLog(fs, rootDir, "testDataInMemoryWithoutWAL", CONF); 3354 hLog.init(); 3355 // This chunk creation is done throughout the code base. Do we want to move it into core? 3356 // It is missing from this test. W/o it we NPE. 3357 region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, hLog, 3358 COLUMN_FAMILY_BYTES); 3359 3360 Cell originalCell = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row) 3361 .setFamily(COLUMN_FAMILY_BYTES).setQualifier(qual1) 3362 .setTimestamp(EnvironmentEdgeManager.currentTime()).setType(KeyValue.Type.Put.getCode()) 3363 .setValue(value1).build(); 3364 final long originalSize = originalCell.getSerializedSize(); 3365 3366 Cell addCell = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row) 3367 .setFamily(COLUMN_FAMILY_BYTES).setQualifier(qual1) 3368 .setTimestamp(EnvironmentEdgeManager.currentTime()).setType(KeyValue.Type.Put.getCode()) 3369 .setValue(Bytes.toBytes("xxxxxxxxxx")).build(); 3370 final long addSize = addCell.getSerializedSize(); 3371 3372 LOG.info("originalSize:" + originalSize + ", addSize:" + addSize); 3373 // start test. We expect that the addPut's durability will be replaced 3374 // by originalPut's durability. 3375 3376 // case 1: 3377 testDataInMemoryWithoutWAL(region, 3378 new Put(row).add(originalCell).setDurability(Durability.SKIP_WAL), 3379 new Put(row).add(addCell).setDurability(Durability.SKIP_WAL), originalSize + addSize); 3380 3381 // case 2: 3382 testDataInMemoryWithoutWAL(region, 3383 new Put(row).add(originalCell).setDurability(Durability.SKIP_WAL), 3384 new Put(row).add(addCell).setDurability(Durability.SYNC_WAL), originalSize + addSize); 3385 3386 // case 3: 3387 testDataInMemoryWithoutWAL(region, 3388 new Put(row).add(originalCell).setDurability(Durability.SYNC_WAL), 3389 new Put(row).add(addCell).setDurability(Durability.SKIP_WAL), 0); 3390 3391 // case 4: 3392 testDataInMemoryWithoutWAL(region, 3393 new Put(row).add(originalCell).setDurability(Durability.SYNC_WAL), 3394 new Put(row).add(addCell).setDurability(Durability.SYNC_WAL), 0); 3395 } 3396 3397 private static void testDataInMemoryWithoutWAL(HRegion region, Put originalPut, final Put addPut, 3398 long delta) throws IOException { 3399 final long initSize = region.getDataInMemoryWithoutWAL(); 3400 // save normalCPHost and replaced by mockedCPHost 3401 RegionCoprocessorHost normalCPHost = region.getCoprocessorHost(); 3402 RegionCoprocessorHost mockedCPHost = mock(RegionCoprocessorHost.class); 3403 // Because the preBatchMutate returns void, we can't do usual Mockito when...then form. Must 3404 // do below format (from Mockito doc). 3405 doAnswer(new Answer<Void>() { 3406 @Override 3407 public Void answer(InvocationOnMock invocation) throws Throwable { 3408 MiniBatchOperationInProgress<Mutation> mb = invocation.getArgument(0); 3409 mb.addOperationsFromCP(0, new Mutation[] { addPut }); 3410 return null; 3411 } 3412 }).when(mockedCPHost).preBatchMutate(isA(MiniBatchOperationInProgress.class)); 3413 ColumnFamilyDescriptorBuilder builder = 3414 ColumnFamilyDescriptorBuilder.newBuilder(COLUMN_FAMILY_BYTES); 3415 ScanInfo info = new ScanInfo(CONF, builder.build(), Long.MAX_VALUE, Long.MAX_VALUE, 3416 region.getCellComparator()); 3417 when(mockedCPHost.preFlushScannerOpen(any(HStore.class), any())).thenReturn(info); 3418 3419 when(mockedCPHost.preFlush(any(), any(StoreScanner.class), any())) 3420 .thenAnswer(i -> i.getArgument(1)); 3421 region.setCoprocessorHost(mockedCPHost); 3422 3423 region.put(originalPut); 3424 region.setCoprocessorHost(normalCPHost); 3425 final long finalSize = region.getDataInMemoryWithoutWAL(); 3426 assertEquals("finalSize:" + finalSize + ", initSize:" + initSize + ", delta:" + delta, 3427 finalSize, initSize + delta); 3428 } 3429 3430 @Test 3431 public void testDeleteColumns_PostInsert() throws IOException, InterruptedException { 3432 Delete delete = new Delete(row); 3433 delete.addColumns(fam1, qual1); 3434 doTestDelete_AndPostInsert(delete); 3435 } 3436 3437 @Test 3438 public void testaddFamily_PostInsert() throws IOException, InterruptedException { 3439 Delete delete = new Delete(row); 3440 delete.addFamily(fam1); 3441 doTestDelete_AndPostInsert(delete); 3442 } 3443 3444 public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException { 3445 this.region = initHRegion(tableName, method, CONF, fam1); 3446 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge()); 3447 Put put = new Put(row); 3448 put.addColumn(fam1, qual1, value1); 3449 region.put(put); 3450 3451 // now delete the value: 3452 region.delete(delete); 3453 3454 // ok put data: 3455 put = new Put(row); 3456 put.addColumn(fam1, qual1, value2); 3457 region.put(put); 3458 3459 // ok get: 3460 Get get = new Get(row); 3461 get.addColumn(fam1, qual1); 3462 3463 Result r = region.get(get); 3464 assertEquals(1, r.size()); 3465 assertArrayEquals(value2, r.getValue(fam1, qual1)); 3466 3467 // next: 3468 Scan scan = new Scan().withStartRow(row); 3469 scan.addColumn(fam1, qual1); 3470 try (InternalScanner s = region.getScanner(scan)) { 3471 List<Cell> results = new ArrayList<>(); 3472 assertEquals(false, s.next(results)); 3473 assertEquals(1, results.size()); 3474 Cell kv = results.get(0); 3475 3476 assertArrayEquals(value2, CellUtil.cloneValue(kv)); 3477 assertArrayEquals(fam1, CellUtil.cloneFamily(kv)); 3478 assertArrayEquals(qual1, CellUtil.cloneQualifier(kv)); 3479 assertArrayEquals(row, CellUtil.cloneRow(kv)); 3480 } 3481 } 3482 3483 @Test 3484 public void testDelete_CheckTimestampUpdated() throws IOException { 3485 byte[] row1 = Bytes.toBytes("row1"); 3486 byte[] col1 = Bytes.toBytes("col1"); 3487 byte[] col2 = Bytes.toBytes("col2"); 3488 byte[] col3 = Bytes.toBytes("col3"); 3489 3490 byte[] forUnitTestsOnly = Bytes.toBytes("ForUnitTestsOnly"); 3491 3492 // Setting up region 3493 this.region = initHRegion(tableName, method, CONF, fam1); 3494 // Building checkerList 3495 List<Cell> kvs = new ArrayList<>(); 3496 kvs.add(new KeyValue(row1, fam1, col1, null)); 3497 kvs.add(new KeyValue(row1, fam1, col2, null)); 3498 kvs.add(new KeyValue(row1, fam1, col3, null)); 3499 3500 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<>(Bytes.BYTES_COMPARATOR); 3501 deleteMap.put(fam1, kvs); 3502 region.delete(new Delete(forUnitTestsOnly, HConstants.LATEST_TIMESTAMP, deleteMap)); 3503 3504 // extract the key values out the memstore: 3505 // This is kinda hacky, but better than nothing... 3506 long now = EnvironmentEdgeManager.currentTime(); 3507 AbstractMemStore memstore = (AbstractMemStore) region.getStore(fam1).memstore; 3508 Cell firstCell = memstore.getActive().first(); 3509 assertTrue(firstCell.getTimestamp() <= now); 3510 now = firstCell.getTimestamp(); 3511 for (Cell cell : memstore.getActive().getCellSet()) { 3512 assertTrue(cell.getTimestamp() <= now); 3513 now = cell.getTimestamp(); 3514 } 3515 } 3516 3517 // //////////////////////////////////////////////////////////////////////////// 3518 // Get tests 3519 // //////////////////////////////////////////////////////////////////////////// 3520 @Test 3521 public void testGet_FamilyChecker() throws IOException { 3522 byte[] row1 = Bytes.toBytes("row1"); 3523 byte[] fam1 = Bytes.toBytes("fam1"); 3524 byte[] fam2 = Bytes.toBytes("False"); 3525 byte[] col1 = Bytes.toBytes("col1"); 3526 3527 // Setting up region 3528 this.region = initHRegion(tableName, method, CONF, fam1); 3529 Get get = new Get(row1); 3530 get.addColumn(fam2, col1); 3531 3532 // Test 3533 try { 3534 region.get(get); 3535 fail("Expecting DoNotRetryIOException in get but did not get any"); 3536 } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) { 3537 LOG.info("Got expected DoNotRetryIOException successfully"); 3538 } 3539 } 3540 3541 @Test 3542 public void testGet_Basic() throws IOException { 3543 byte[] row1 = Bytes.toBytes("row1"); 3544 byte[] fam1 = Bytes.toBytes("fam1"); 3545 byte[] col1 = Bytes.toBytes("col1"); 3546 byte[] col2 = Bytes.toBytes("col2"); 3547 byte[] col3 = Bytes.toBytes("col3"); 3548 byte[] col4 = Bytes.toBytes("col4"); 3549 byte[] col5 = Bytes.toBytes("col5"); 3550 3551 // Setting up region 3552 this.region = initHRegion(tableName, method, CONF, fam1); 3553 // Add to memstore 3554 Put put = new Put(row1); 3555 put.addColumn(fam1, col1, null); 3556 put.addColumn(fam1, col2, null); 3557 put.addColumn(fam1, col3, null); 3558 put.addColumn(fam1, col4, null); 3559 put.addColumn(fam1, col5, null); 3560 region.put(put); 3561 3562 Get get = new Get(row1); 3563 get.addColumn(fam1, col2); 3564 get.addColumn(fam1, col4); 3565 // Expected result 3566 KeyValue kv1 = new KeyValue(row1, fam1, col2); 3567 KeyValue kv2 = new KeyValue(row1, fam1, col4); 3568 KeyValue[] expected = { kv1, kv2 }; 3569 3570 // Test 3571 Result res = region.get(get); 3572 assertEquals(expected.length, res.size()); 3573 for (int i = 0; i < res.size(); i++) { 3574 assertTrue(CellUtil.matchingRows(expected[i], res.rawCells()[i])); 3575 assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i])); 3576 assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i])); 3577 } 3578 3579 // Test using a filter on a Get 3580 Get g = new Get(row1); 3581 final int count = 2; 3582 g.setFilter(new ColumnCountGetFilter(count)); 3583 res = region.get(g); 3584 assertEquals(count, res.size()); 3585 } 3586 3587 @Test 3588 public void testGet_Empty() throws IOException { 3589 byte[] row = Bytes.toBytes("row"); 3590 byte[] fam = Bytes.toBytes("fam"); 3591 3592 this.region = initHRegion(tableName, method, CONF, fam); 3593 Get get = new Get(row); 3594 get.addFamily(fam); 3595 Result r = region.get(get); 3596 3597 assertTrue(r.isEmpty()); 3598 } 3599 3600 @Test 3601 public void testGetWithFilter() throws IOException, InterruptedException { 3602 byte[] row1 = Bytes.toBytes("row1"); 3603 byte[] fam1 = Bytes.toBytes("fam1"); 3604 byte[] col1 = Bytes.toBytes("col1"); 3605 byte[] value1 = Bytes.toBytes("value1"); 3606 byte[] value2 = Bytes.toBytes("value2"); 3607 3608 final int maxVersions = 3; 3609 TableDescriptor tableDescriptor = 3610 TableDescriptorBuilder.newBuilder(TableName.valueOf("testFilterAndColumnTracker")) 3611 .setColumnFamily( 3612 ColumnFamilyDescriptorBuilder.newBuilder(fam1).setMaxVersions(maxVersions).build()) 3613 .build(); 3614 ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null, 3615 MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT); 3616 RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 3617 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + ".log"); 3618 final WAL wal = HBaseTestingUtil.createWal(TEST_UTIL.getConfiguration(), logDir, info); 3619 this.region = TEST_UTIL.createLocalHRegion(info, CONF, tableDescriptor, wal); 3620 3621 // Put 4 version to memstore 3622 long ts = 0; 3623 Put put = new Put(row1, ts); 3624 put.addColumn(fam1, col1, value1); 3625 region.put(put); 3626 put = new Put(row1, ts + 1); 3627 put.addColumn(fam1, col1, Bytes.toBytes("filter1")); 3628 region.put(put); 3629 put = new Put(row1, ts + 2); 3630 put.addColumn(fam1, col1, Bytes.toBytes("filter2")); 3631 region.put(put); 3632 put = new Put(row1, ts + 3); 3633 put.addColumn(fam1, col1, value2); 3634 region.put(put); 3635 3636 Get get = new Get(row1); 3637 get.readAllVersions(); 3638 Result res = region.get(get); 3639 // Get 3 versions, the oldest version has gone from user view 3640 assertEquals(maxVersions, res.size()); 3641 3642 get.setFilter(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("value"))); 3643 res = region.get(get); 3644 // When use value filter, the oldest version should still gone from user view and it 3645 // should only return one key vaule 3646 assertEquals(1, res.size()); 3647 assertTrue(CellUtil.matchingValue(new KeyValue(row1, fam1, col1, value2), res.rawCells()[0])); 3648 assertEquals(ts + 3, res.rawCells()[0].getTimestamp()); 3649 3650 region.flush(true); 3651 region.compact(true); 3652 Thread.sleep(1000); 3653 res = region.get(get); 3654 // After flush and compact, the result should be consistent with previous result 3655 assertEquals(1, res.size()); 3656 assertTrue(CellUtil.matchingValue(new KeyValue(row1, fam1, col1, value2), res.rawCells()[0])); 3657 } 3658 3659 // //////////////////////////////////////////////////////////////////////////// 3660 // Scanner tests 3661 // //////////////////////////////////////////////////////////////////////////// 3662 @Test 3663 public void testGetScanner_WithOkFamilies() throws IOException { 3664 byte[] fam1 = Bytes.toBytes("fam1"); 3665 byte[] fam2 = Bytes.toBytes("fam2"); 3666 3667 byte[][] families = { fam1, fam2 }; 3668 3669 // Setting up region 3670 this.region = initHRegion(tableName, method, CONF, families); 3671 Scan scan = new Scan(); 3672 scan.addFamily(fam1); 3673 scan.addFamily(fam2); 3674 region.getScanner(scan).close(); 3675 } 3676 3677 @Test 3678 public void testGetScanner_WithNotOkFamilies() throws IOException { 3679 byte[] fam1 = Bytes.toBytes("fam1"); 3680 byte[] fam2 = Bytes.toBytes("fam2"); 3681 3682 byte[][] families = { fam1 }; 3683 3684 // Setting up region 3685 this.region = initHRegion(tableName, method, CONF, families); 3686 Scan scan = new Scan(); 3687 scan.addFamily(fam2); 3688 assertThrows(NoSuchColumnFamilyException.class, () -> region.getScanner(scan)); 3689 } 3690 3691 @Test 3692 public void testGetScanner_WithNoFamilies() throws IOException { 3693 byte[] row1 = Bytes.toBytes("row1"); 3694 byte[] fam1 = Bytes.toBytes("fam1"); 3695 byte[] fam2 = Bytes.toBytes("fam2"); 3696 byte[] fam3 = Bytes.toBytes("fam3"); 3697 byte[] fam4 = Bytes.toBytes("fam4"); 3698 3699 byte[][] families = { fam1, fam2, fam3, fam4 }; 3700 3701 // Setting up region 3702 this.region = initHRegion(tableName, method, CONF, families); 3703 // Putting data in Region 3704 Put put = new Put(row1); 3705 put.addColumn(fam1, null, null); 3706 put.addColumn(fam2, null, null); 3707 put.addColumn(fam3, null, null); 3708 put.addColumn(fam4, null, null); 3709 region.put(put); 3710 3711 Scan scan = null; 3712 3713 // Testing to see how many scanners that is produced by getScanner, starting with known number, 3714 // 2 - current = 1 3715 scan = new Scan(); 3716 scan.addFamily(fam2); 3717 scan.addFamily(fam4); 3718 try (RegionScannerImpl is = region.getScanner(scan)) { 3719 assertEquals(1, is.storeHeap.getHeap().size()); 3720 } 3721 3722 scan = new Scan(); 3723 try (RegionScannerImpl is = region.getScanner(scan)) { 3724 assertEquals(families.length - 1, is.storeHeap.getHeap().size()); 3725 } 3726 } 3727 3728 /** 3729 * This method tests https://issues.apache.org/jira/browse/HBASE-2516. 3730 */ 3731 @Test 3732 public void testGetScanner_WithRegionClosed() throws IOException { 3733 byte[] fam1 = Bytes.toBytes("fam1"); 3734 byte[] fam2 = Bytes.toBytes("fam2"); 3735 3736 byte[][] families = { fam1, fam2 }; 3737 3738 // Setting up region 3739 region = initHRegion(tableName, method, CONF, families); 3740 region.closed.set(true); 3741 try { 3742 assertThrows(NotServingRegionException.class, () -> region.getScanner(null)); 3743 } finally { 3744 // so we can close the region in tearDown 3745 region.closed.set(false); 3746 } 3747 } 3748 3749 @Test 3750 public void testRegionScanner_Next() throws IOException { 3751 byte[] row1 = Bytes.toBytes("row1"); 3752 byte[] row2 = Bytes.toBytes("row2"); 3753 byte[] fam1 = Bytes.toBytes("fam1"); 3754 byte[] fam2 = Bytes.toBytes("fam2"); 3755 byte[] fam3 = Bytes.toBytes("fam3"); 3756 byte[] fam4 = Bytes.toBytes("fam4"); 3757 3758 byte[][] families = { fam1, fam2, fam3, fam4 }; 3759 long ts = EnvironmentEdgeManager.currentTime(); 3760 3761 // Setting up region 3762 this.region = initHRegion(tableName, method, CONF, families); 3763 // Putting data in Region 3764 Put put = null; 3765 put = new Put(row1); 3766 put.addColumn(fam1, (byte[]) null, ts, null); 3767 put.addColumn(fam2, (byte[]) null, ts, null); 3768 put.addColumn(fam3, (byte[]) null, ts, null); 3769 put.addColumn(fam4, (byte[]) null, ts, null); 3770 region.put(put); 3771 3772 put = new Put(row2); 3773 put.addColumn(fam1, (byte[]) null, ts, null); 3774 put.addColumn(fam2, (byte[]) null, ts, null); 3775 put.addColumn(fam3, (byte[]) null, ts, null); 3776 put.addColumn(fam4, (byte[]) null, ts, null); 3777 region.put(put); 3778 3779 Scan scan = new Scan(); 3780 scan.addFamily(fam2); 3781 scan.addFamily(fam4); 3782 try (InternalScanner is = region.getScanner(scan)) { 3783 List<Cell> res = null; 3784 3785 // Result 1 3786 List<Cell> expected1 = new ArrayList<>(); 3787 expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null)); 3788 expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null)); 3789 3790 res = new ArrayList<>(); 3791 is.next(res); 3792 for (int i = 0; i < res.size(); i++) { 3793 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected1.get(i), res.get(i))); 3794 } 3795 3796 // Result 2 3797 List<Cell> expected2 = new ArrayList<>(); 3798 expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null)); 3799 expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null)); 3800 3801 res = new ArrayList<>(); 3802 is.next(res); 3803 for (int i = 0; i < res.size(); i++) { 3804 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected2.get(i), res.get(i))); 3805 } 3806 } 3807 } 3808 3809 @Test 3810 public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException { 3811 byte[] row1 = Bytes.toBytes("row1"); 3812 byte[] qf1 = Bytes.toBytes("qualifier1"); 3813 byte[] qf2 = Bytes.toBytes("qualifier2"); 3814 byte[] fam1 = Bytes.toBytes("fam1"); 3815 byte[][] families = { fam1 }; 3816 3817 long ts1 = EnvironmentEdgeManager.currentTime(); 3818 long ts2 = ts1 + 1; 3819 long ts3 = ts1 + 2; 3820 3821 // Setting up region 3822 this.region = initHRegion(tableName, method, CONF, families); 3823 // Putting data in Region 3824 Put put = null; 3825 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 3826 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 3827 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 3828 3829 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 3830 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 3831 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 3832 3833 put = new Put(row1); 3834 put.add(kv13); 3835 put.add(kv12); 3836 put.add(kv11); 3837 put.add(kv23); 3838 put.add(kv22); 3839 put.add(kv21); 3840 region.put(put); 3841 3842 // Expected 3843 List<Cell> expected = new ArrayList<>(); 3844 expected.add(kv13); 3845 expected.add(kv12); 3846 3847 Scan scan = new Scan().withStartRow(row1); 3848 scan.addColumn(fam1, qf1); 3849 scan.readVersions(MAX_VERSIONS); 3850 List<Cell> actual = new ArrayList<>(); 3851 try (InternalScanner scanner = region.getScanner(scan)) { 3852 boolean hasNext = scanner.next(actual); 3853 assertEquals(false, hasNext); 3854 3855 // Verify result 3856 for (int i = 0; i < expected.size(); i++) { 3857 assertEquals(expected.get(i), actual.get(i)); 3858 } 3859 } 3860 } 3861 3862 @Test 3863 public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException { 3864 byte[] row1 = Bytes.toBytes("row1"); 3865 byte[] qf1 = Bytes.toBytes("qualifier1"); 3866 byte[] qf2 = Bytes.toBytes("qualifier2"); 3867 byte[] fam1 = Bytes.toBytes("fam1"); 3868 byte[][] families = { fam1 }; 3869 3870 long ts1 = 1; 3871 long ts2 = ts1 + 1; 3872 long ts3 = ts1 + 2; 3873 3874 // Setting up region 3875 this.region = initHRegion(tableName, method, CONF, families); 3876 // Putting data in Region 3877 Put put = null; 3878 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 3879 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 3880 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 3881 3882 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 3883 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 3884 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 3885 3886 put = new Put(row1); 3887 put.add(kv13); 3888 put.add(kv12); 3889 put.add(kv11); 3890 put.add(kv23); 3891 put.add(kv22); 3892 put.add(kv21); 3893 region.put(put); 3894 region.flush(true); 3895 3896 // Expected 3897 List<Cell> expected = new ArrayList<>(); 3898 expected.add(kv13); 3899 expected.add(kv12); 3900 expected.add(kv23); 3901 expected.add(kv22); 3902 3903 Scan scan = new Scan().withStartRow(row1); 3904 scan.addColumn(fam1, qf1); 3905 scan.addColumn(fam1, qf2); 3906 scan.readVersions(MAX_VERSIONS); 3907 List<Cell> actual = new ArrayList<>(); 3908 try (InternalScanner scanner = region.getScanner(scan)) { 3909 boolean hasNext = scanner.next(actual); 3910 assertEquals(false, hasNext); 3911 3912 // Verify result 3913 for (int i = 0; i < expected.size(); i++) { 3914 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); 3915 } 3916 } 3917 } 3918 3919 @Test 3920 public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() 3921 throws IOException { 3922 byte[] row1 = Bytes.toBytes("row1"); 3923 byte[] fam1 = Bytes.toBytes("fam1"); 3924 byte[][] families = { fam1 }; 3925 byte[] qf1 = Bytes.toBytes("qualifier1"); 3926 byte[] qf2 = Bytes.toBytes("qualifier2"); 3927 3928 long ts1 = 1; 3929 long ts2 = ts1 + 1; 3930 long ts3 = ts1 + 2; 3931 long ts4 = ts1 + 3; 3932 3933 // Setting up region 3934 this.region = initHRegion(tableName, method, CONF, families); 3935 // Putting data in Region 3936 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null); 3937 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 3938 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 3939 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 3940 3941 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null); 3942 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 3943 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 3944 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 3945 3946 Put put = null; 3947 put = new Put(row1); 3948 put.add(kv14); 3949 put.add(kv24); 3950 region.put(put); 3951 region.flush(true); 3952 3953 put = new Put(row1); 3954 put.add(kv23); 3955 put.add(kv13); 3956 region.put(put); 3957 region.flush(true); 3958 3959 put = new Put(row1); 3960 put.add(kv22); 3961 put.add(kv12); 3962 region.put(put); 3963 region.flush(true); 3964 3965 put = new Put(row1); 3966 put.add(kv21); 3967 put.add(kv11); 3968 region.put(put); 3969 3970 // Expected 3971 List<Cell> expected = new ArrayList<>(); 3972 expected.add(kv14); 3973 expected.add(kv13); 3974 expected.add(kv12); 3975 expected.add(kv24); 3976 expected.add(kv23); 3977 expected.add(kv22); 3978 3979 Scan scan = new Scan().withStartRow(row1); 3980 scan.addColumn(fam1, qf1); 3981 scan.addColumn(fam1, qf2); 3982 int versions = 3; 3983 scan.readVersions(versions); 3984 List<Cell> actual = new ArrayList<>(); 3985 try (InternalScanner scanner = region.getScanner(scan)) { 3986 boolean hasNext = scanner.next(actual); 3987 assertEquals(false, hasNext); 3988 3989 // Verify result 3990 for (int i = 0; i < expected.size(); i++) { 3991 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); 3992 } 3993 } 3994 } 3995 3996 @Test 3997 public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException { 3998 byte[] row1 = Bytes.toBytes("row1"); 3999 byte[] qf1 = Bytes.toBytes("qualifier1"); 4000 byte[] qf2 = Bytes.toBytes("qualifier2"); 4001 byte[] fam1 = Bytes.toBytes("fam1"); 4002 byte[][] families = { fam1 }; 4003 4004 long ts1 = EnvironmentEdgeManager.currentTime(); 4005 long ts2 = ts1 + 1; 4006 long ts3 = ts1 + 2; 4007 4008 // Setting up region 4009 this.region = initHRegion(tableName, method, CONF, families); 4010 // Putting data in Region 4011 Put put = null; 4012 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 4013 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 4014 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 4015 4016 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 4017 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 4018 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 4019 4020 put = new Put(row1); 4021 put.add(kv13); 4022 put.add(kv12); 4023 put.add(kv11); 4024 put.add(kv23); 4025 put.add(kv22); 4026 put.add(kv21); 4027 region.put(put); 4028 4029 // Expected 4030 List<Cell> expected = new ArrayList<>(); 4031 expected.add(kv13); 4032 expected.add(kv12); 4033 expected.add(kv23); 4034 expected.add(kv22); 4035 4036 Scan scan = new Scan().withStartRow(row1); 4037 scan.addFamily(fam1); 4038 scan.readVersions(MAX_VERSIONS); 4039 List<Cell> actual = new ArrayList<>(); 4040 try (InternalScanner scanner = region.getScanner(scan)) { 4041 boolean hasNext = scanner.next(actual); 4042 assertEquals(false, hasNext); 4043 4044 // Verify result 4045 for (int i = 0; i < expected.size(); i++) { 4046 assertEquals(expected.get(i), actual.get(i)); 4047 } 4048 } 4049 } 4050 4051 @Test 4052 public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException { 4053 byte[] row1 = Bytes.toBytes("row1"); 4054 byte[] qf1 = Bytes.toBytes("qualifier1"); 4055 byte[] qf2 = Bytes.toBytes("qualifier2"); 4056 byte[] fam1 = Bytes.toBytes("fam1"); 4057 4058 long ts1 = 1; 4059 long ts2 = ts1 + 1; 4060 long ts3 = ts1 + 2; 4061 4062 // Setting up region 4063 this.region = initHRegion(tableName, method, CONF, fam1); 4064 // Putting data in Region 4065 Put put = null; 4066 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 4067 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 4068 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 4069 4070 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 4071 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 4072 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 4073 4074 put = new Put(row1); 4075 put.add(kv13); 4076 put.add(kv12); 4077 put.add(kv11); 4078 put.add(kv23); 4079 put.add(kv22); 4080 put.add(kv21); 4081 region.put(put); 4082 region.flush(true); 4083 4084 // Expected 4085 List<Cell> expected = new ArrayList<>(); 4086 expected.add(kv13); 4087 expected.add(kv12); 4088 expected.add(kv23); 4089 expected.add(kv22); 4090 4091 Scan scan = new Scan().withStartRow(row1); 4092 scan.addFamily(fam1); 4093 scan.readVersions(MAX_VERSIONS); 4094 List<Cell> actual = new ArrayList<>(); 4095 try (InternalScanner scanner = region.getScanner(scan)) { 4096 boolean hasNext = scanner.next(actual); 4097 assertEquals(false, hasNext); 4098 4099 // Verify result 4100 for (int i = 0; i < expected.size(); i++) { 4101 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); 4102 } 4103 } 4104 } 4105 4106 @Test 4107 public void testScanner_StopRow1542() throws IOException { 4108 byte[] family = Bytes.toBytes("testFamily"); 4109 this.region = initHRegion(tableName, method, CONF, family); 4110 byte[] row1 = Bytes.toBytes("row111"); 4111 byte[] row2 = Bytes.toBytes("row222"); 4112 byte[] row3 = Bytes.toBytes("row333"); 4113 byte[] row4 = Bytes.toBytes("row444"); 4114 byte[] row5 = Bytes.toBytes("row555"); 4115 4116 byte[] col1 = Bytes.toBytes("Pub111"); 4117 byte[] col2 = Bytes.toBytes("Pub222"); 4118 4119 Put put = new Put(row1); 4120 put.addColumn(family, col1, Bytes.toBytes(10L)); 4121 region.put(put); 4122 4123 put = new Put(row2); 4124 put.addColumn(family, col1, Bytes.toBytes(15L)); 4125 region.put(put); 4126 4127 put = new Put(row3); 4128 put.addColumn(family, col2, Bytes.toBytes(20L)); 4129 region.put(put); 4130 4131 put = new Put(row4); 4132 put.addColumn(family, col2, Bytes.toBytes(30L)); 4133 region.put(put); 4134 4135 put = new Put(row5); 4136 put.addColumn(family, col1, Bytes.toBytes(40L)); 4137 region.put(put); 4138 4139 Scan scan = new Scan().withStartRow(row3).withStopRow(row4); 4140 scan.readAllVersions(); 4141 scan.addColumn(family, col1); 4142 try (InternalScanner s = region.getScanner(scan)) { 4143 List<Cell> results = new ArrayList<>(); 4144 assertEquals(false, s.next(results)); 4145 assertEquals(0, results.size()); 4146 } 4147 } 4148 4149 @Test 4150 public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException { 4151 byte[] row1 = Bytes.toBytes("row1"); 4152 byte[] fam1 = Bytes.toBytes("fam1"); 4153 byte[] qf1 = Bytes.toBytes("qualifier1"); 4154 byte[] qf2 = Bytes.toBytes("quateslifier2"); 4155 4156 long ts1 = 1; 4157 long ts2 = ts1 + 1; 4158 long ts3 = ts1 + 2; 4159 long ts4 = ts1 + 3; 4160 4161 // Setting up region 4162 this.region = initHRegion(tableName, method, CONF, fam1); 4163 // Putting data in Region 4164 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null); 4165 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 4166 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 4167 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 4168 4169 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null); 4170 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 4171 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 4172 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 4173 4174 Put put = null; 4175 put = new Put(row1); 4176 put.add(kv14); 4177 put.add(kv24); 4178 region.put(put); 4179 region.flush(true); 4180 4181 put = new Put(row1); 4182 put.add(kv23); 4183 put.add(kv13); 4184 region.put(put); 4185 region.flush(true); 4186 4187 put = new Put(row1); 4188 put.add(kv22); 4189 put.add(kv12); 4190 region.put(put); 4191 region.flush(true); 4192 4193 put = new Put(row1); 4194 put.add(kv21); 4195 put.add(kv11); 4196 region.put(put); 4197 4198 // Expected 4199 List<KeyValue> expected = new ArrayList<>(); 4200 expected.add(kv14); 4201 expected.add(kv13); 4202 expected.add(kv12); 4203 expected.add(kv24); 4204 expected.add(kv23); 4205 expected.add(kv22); 4206 4207 Scan scan = new Scan().withStartRow(row1); 4208 int versions = 3; 4209 scan.readVersions(versions); 4210 List<Cell> actual = new ArrayList<>(); 4211 try (InternalScanner scanner = region.getScanner(scan)) { 4212 boolean hasNext = scanner.next(actual); 4213 assertEquals(false, hasNext); 4214 4215 // Verify result 4216 for (int i = 0; i < expected.size(); i++) { 4217 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); 4218 } 4219 } 4220 } 4221 4222 /** 4223 * Added for HBASE-5416 Here we test scan optimization when only subset of CFs are used in filter 4224 * conditions. 4225 */ 4226 @Test 4227 public void testScanner_JoinedScanners() throws IOException { 4228 byte[] cf_essential = Bytes.toBytes("essential"); 4229 byte[] cf_joined = Bytes.toBytes("joined"); 4230 byte[] cf_alpha = Bytes.toBytes("alpha"); 4231 this.region = initHRegion(tableName, method, CONF, cf_essential, cf_joined, cf_alpha); 4232 byte[] row1 = Bytes.toBytes("row1"); 4233 byte[] row2 = Bytes.toBytes("row2"); 4234 byte[] row3 = Bytes.toBytes("row3"); 4235 4236 byte[] col_normal = Bytes.toBytes("d"); 4237 byte[] col_alpha = Bytes.toBytes("a"); 4238 4239 byte[] filtered_val = Bytes.toBytes(3); 4240 4241 Put put = new Put(row1); 4242 put.addColumn(cf_essential, col_normal, Bytes.toBytes(1)); 4243 put.addColumn(cf_joined, col_alpha, Bytes.toBytes(1)); 4244 region.put(put); 4245 4246 put = new Put(row2); 4247 put.addColumn(cf_essential, col_alpha, Bytes.toBytes(2)); 4248 put.addColumn(cf_joined, col_normal, Bytes.toBytes(2)); 4249 put.addColumn(cf_alpha, col_alpha, Bytes.toBytes(2)); 4250 region.put(put); 4251 4252 put = new Put(row3); 4253 put.addColumn(cf_essential, col_normal, filtered_val); 4254 put.addColumn(cf_joined, col_normal, filtered_val); 4255 region.put(put); 4256 4257 // Check two things: 4258 // 1. result list contains expected values 4259 // 2. result list is sorted properly 4260 4261 Scan scan = new Scan(); 4262 Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal, 4263 CompareOperator.NOT_EQUAL, filtered_val); 4264 scan.setFilter(filter); 4265 scan.setLoadColumnFamiliesOnDemand(true); 4266 try (InternalScanner s = region.getScanner(scan)) { 4267 List<Cell> results = new ArrayList<>(); 4268 assertTrue(s.next(results)); 4269 assertEquals(1, results.size()); 4270 results.clear(); 4271 4272 assertTrue(s.next(results)); 4273 assertEquals(3, results.size()); 4274 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha)); 4275 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential)); 4276 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined)); 4277 results.clear(); 4278 4279 assertFalse(s.next(results)); 4280 assertEquals(0, results.size()); 4281 } 4282 } 4283 4284 /** 4285 * HBASE-5416 Test case when scan limits amount of KVs returned on each next() call. 4286 */ 4287 @Test 4288 public void testScanner_JoinedScannersWithLimits() throws IOException { 4289 final byte[] cf_first = Bytes.toBytes("first"); 4290 final byte[] cf_second = Bytes.toBytes("second"); 4291 4292 this.region = initHRegion(tableName, method, CONF, cf_first, cf_second); 4293 final byte[] col_a = Bytes.toBytes("a"); 4294 final byte[] col_b = Bytes.toBytes("b"); 4295 4296 Put put; 4297 4298 for (int i = 0; i < 10; i++) { 4299 put = new Put(Bytes.toBytes("r" + Integer.toString(i))); 4300 put.addColumn(cf_first, col_a, Bytes.toBytes(i)); 4301 if (i < 5) { 4302 put.addColumn(cf_first, col_b, Bytes.toBytes(i)); 4303 put.addColumn(cf_second, col_a, Bytes.toBytes(i)); 4304 put.addColumn(cf_second, col_b, Bytes.toBytes(i)); 4305 } 4306 region.put(put); 4307 } 4308 4309 Scan scan = new Scan(); 4310 scan.setLoadColumnFamiliesOnDemand(true); 4311 Filter bogusFilter = new FilterBase() { 4312 @Override 4313 public ReturnCode filterCell(final Cell ignored) throws IOException { 4314 return ReturnCode.INCLUDE; 4315 } 4316 4317 @Override 4318 public boolean isFamilyEssential(byte[] name) { 4319 return Bytes.equals(name, cf_first); 4320 } 4321 }; 4322 4323 scan.setFilter(bogusFilter); 4324 try (InternalScanner s = region.getScanner(scan)) { 4325 // Our data looks like this: 4326 // r0: first:a, first:b, second:a, second:b 4327 // r1: first:a, first:b, second:a, second:b 4328 // r2: first:a, first:b, second:a, second:b 4329 // r3: first:a, first:b, second:a, second:b 4330 // r4: first:a, first:b, second:a, second:b 4331 // r5: first:a 4332 // r6: first:a 4333 // r7: first:a 4334 // r8: first:a 4335 // r9: first:a 4336 4337 // But due to next's limit set to 3, we should get this: 4338 // r0: first:a, first:b, second:a 4339 // r0: second:b 4340 // r1: first:a, first:b, second:a 4341 // r1: second:b 4342 // r2: first:a, first:b, second:a 4343 // r2: second:b 4344 // r3: first:a, first:b, second:a 4345 // r3: second:b 4346 // r4: first:a, first:b, second:a 4347 // r4: second:b 4348 // r5: first:a 4349 // r6: first:a 4350 // r7: first:a 4351 // r8: first:a 4352 // r9: first:a 4353 4354 List<Cell> results = new ArrayList<>(); 4355 int index = 0; 4356 ScannerContext scannerContext = ScannerContext.newBuilder().setBatchLimit(3).build(); 4357 while (true) { 4358 boolean more = s.next(results, scannerContext); 4359 if ((index >> 1) < 5) { 4360 if (index % 2 == 0) { 4361 assertEquals(3, results.size()); 4362 } else { 4363 assertEquals(1, results.size()); 4364 } 4365 } else { 4366 assertEquals(1, results.size()); 4367 } 4368 results.clear(); 4369 index++; 4370 if (!more) { 4371 break; 4372 } 4373 } 4374 } 4375 } 4376 4377 @Test 4378 public void testScannerOperationId() throws IOException { 4379 region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES); 4380 Scan scan = new Scan(); 4381 try (RegionScanner scanner = region.getScanner(scan)) { 4382 assertNull(scanner.getOperationId()); 4383 } 4384 4385 String operationId = "test_operation_id_0101"; 4386 scan = new Scan().setId(operationId); 4387 try (RegionScanner scanner = region.getScanner(scan)) { 4388 assertEquals(operationId, scanner.getOperationId()); 4389 } 4390 4391 HBaseTestingUtil.closeRegionAndWAL(this.region); 4392 } 4393 4394 /** 4395 * Write an HFile block full with Cells whose qualifier that are identical between 0 and 4396 * Short.MAX_VALUE. See HBASE-13329. 4397 */ 4398 @Test 4399 public void testLongQualifier() throws Exception { 4400 byte[] family = Bytes.toBytes("family"); 4401 this.region = initHRegion(tableName, method, CONF, family); 4402 byte[] q = new byte[Short.MAX_VALUE + 2]; 4403 Arrays.fill(q, 0, q.length - 1, (byte) 42); 4404 for (byte i = 0; i < 10; i++) { 4405 Put p = new Put(Bytes.toBytes("row")); 4406 // qualifiers that differ past Short.MAX_VALUE 4407 q[q.length - 1] = i; 4408 p.addColumn(family, q, q); 4409 region.put(p); 4410 } 4411 region.flush(false); 4412 } 4413 4414 /** 4415 * Flushes the cache in a thread while scanning. The tests verify that the scan is coherent - e.g. 4416 * the returned results are always of the same or later update as the previous results. scan / 4417 * compact thread join 4418 */ 4419 @Test 4420 public void testFlushCacheWhileScanning() throws IOException, InterruptedException { 4421 byte[] family = Bytes.toBytes("family"); 4422 int numRows = 1000; 4423 int flushAndScanInterval = 10; 4424 int compactInterval = 10 * flushAndScanInterval; 4425 4426 this.region = initHRegion(tableName, method, CONF, family); 4427 FlushThread flushThread = new FlushThread(); 4428 try { 4429 flushThread.start(); 4430 4431 Scan scan = new Scan(); 4432 scan.addFamily(family); 4433 scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOperator.EQUAL, 4434 new BinaryComparator(Bytes.toBytes(5L)))); 4435 4436 int expectedCount = 0; 4437 List<Cell> res = new ArrayList<>(); 4438 4439 boolean toggle = true; 4440 for (long i = 0; i < numRows; i++) { 4441 Put put = new Put(Bytes.toBytes(i)); 4442 put.setDurability(Durability.SKIP_WAL); 4443 put.addColumn(family, qual1, Bytes.toBytes(i % 10)); 4444 region.put(put); 4445 4446 if (i != 0 && i % compactInterval == 0) { 4447 LOG.debug("iteration = " + i + " ts=" + EnvironmentEdgeManager.currentTime()); 4448 region.compact(true); 4449 } 4450 4451 if (i % 10 == 5L) { 4452 expectedCount++; 4453 } 4454 4455 if (i != 0 && i % flushAndScanInterval == 0) { 4456 res.clear(); 4457 try (InternalScanner scanner = region.getScanner(scan)) { 4458 if (toggle) { 4459 flushThread.flush(); 4460 } 4461 while (scanner.next(res)) { 4462 // ignore 4463 } 4464 } 4465 if (!toggle) { 4466 flushThread.flush(); 4467 } 4468 assertEquals( 4469 "toggle=" + toggle + "i=" + i + " ts=" + EnvironmentEdgeManager.currentTime(), 4470 expectedCount, res.size()); 4471 toggle = !toggle; 4472 } 4473 } 4474 4475 } finally { 4476 try { 4477 flushThread.done(); 4478 flushThread.join(); 4479 flushThread.checkNoError(); 4480 } catch (InterruptedException ie) { 4481 LOG.warn("Caught exception when joining with flushThread", ie); 4482 } 4483 HBaseTestingUtil.closeRegionAndWAL(this.region); 4484 this.region = null; 4485 } 4486 } 4487 4488 protected class FlushThread extends Thread { 4489 private volatile boolean done; 4490 private Throwable error = null; 4491 4492 FlushThread() { 4493 super("FlushThread"); 4494 } 4495 4496 public void done() { 4497 done = true; 4498 synchronized (this) { 4499 interrupt(); 4500 } 4501 } 4502 4503 public void checkNoError() { 4504 if (error != null) { 4505 assertNull(error); 4506 } 4507 } 4508 4509 @Override 4510 public void run() { 4511 done = false; 4512 while (!done) { 4513 synchronized (this) { 4514 try { 4515 wait(); 4516 } catch (InterruptedException ignored) { 4517 if (done) { 4518 break; 4519 } 4520 } 4521 } 4522 try { 4523 region.flush(true); 4524 } catch (IOException e) { 4525 if (!done) { 4526 LOG.error("Error while flushing cache", e); 4527 error = e; 4528 } 4529 break; 4530 } catch (Throwable t) { 4531 LOG.error("Uncaught exception", t); 4532 throw t; 4533 } 4534 } 4535 } 4536 4537 public void flush() { 4538 synchronized (this) { 4539 notify(); 4540 } 4541 } 4542 } 4543 4544 /** 4545 * So can be overridden in subclasses. 4546 */ 4547 protected int getNumQualifiersForTestWritesWhileScanning() { 4548 return 100; 4549 } 4550 4551 /** 4552 * So can be overridden in subclasses. 4553 */ 4554 protected int getTestCountForTestWritesWhileScanning() { 4555 return 100; 4556 } 4557 4558 /** 4559 * Writes very wide records and scans for the latest every time.. Flushes and compacts the region 4560 * every now and then to keep things realistic. by flush / scan / compaction when joining threads 4561 */ 4562 @Test 4563 public void testWritesWhileScanning() throws IOException, InterruptedException { 4564 int testCount = getTestCountForTestWritesWhileScanning(); 4565 int numRows = 1; 4566 int numFamilies = 10; 4567 int numQualifiers = getNumQualifiersForTestWritesWhileScanning(); 4568 int flushInterval = 7; 4569 int compactInterval = 5 * flushInterval; 4570 byte[][] families = new byte[numFamilies][]; 4571 for (int i = 0; i < numFamilies; i++) { 4572 families[i] = Bytes.toBytes("family" + i); 4573 } 4574 byte[][] qualifiers = new byte[numQualifiers][]; 4575 for (int i = 0; i < numQualifiers; i++) { 4576 qualifiers[i] = Bytes.toBytes("qual" + i); 4577 } 4578 4579 this.region = initHRegion(tableName, method, CONF, families); 4580 FlushThread flushThread = new FlushThread(); 4581 PutThread putThread = new PutThread(numRows, families, qualifiers); 4582 try { 4583 putThread.start(); 4584 putThread.waitForFirstPut(); 4585 4586 flushThread.start(); 4587 4588 Scan scan = new Scan().withStartRow(Bytes.toBytes("row0")).withStopRow(Bytes.toBytes("row1")); 4589 4590 int expectedCount = numFamilies * numQualifiers; 4591 List<Cell> res = new ArrayList<>(); 4592 4593 long prevTimestamp = 0L; 4594 for (int i = 0; i < testCount; i++) { 4595 4596 if (i != 0 && i % compactInterval == 0) { 4597 region.compact(true); 4598 for (HStore store : region.getStores()) { 4599 store.closeAndArchiveCompactedFiles(); 4600 } 4601 } 4602 4603 if (i != 0 && i % flushInterval == 0) { 4604 flushThread.flush(); 4605 } 4606 4607 boolean previousEmpty = res.isEmpty(); 4608 res.clear(); 4609 try (InternalScanner scanner = region.getScanner(scan)) { 4610 boolean moreRows; 4611 do { 4612 moreRows = scanner.next(res); 4613 } while (moreRows); 4614 } 4615 if (!res.isEmpty() || !previousEmpty || i > compactInterval) { 4616 assertEquals("i=" + i, expectedCount, res.size()); 4617 long timestamp = res.get(0).getTimestamp(); 4618 assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp, 4619 timestamp >= prevTimestamp); 4620 prevTimestamp = timestamp; 4621 } 4622 } 4623 4624 putThread.done(); 4625 4626 region.flush(true); 4627 4628 } finally { 4629 try { 4630 flushThread.done(); 4631 flushThread.join(); 4632 flushThread.checkNoError(); 4633 4634 putThread.join(); 4635 putThread.checkNoError(); 4636 } catch (InterruptedException ie) { 4637 LOG.warn("Caught exception when joining with flushThread", ie); 4638 } 4639 4640 try { 4641 HBaseTestingUtil.closeRegionAndWAL(this.region); 4642 } catch (DroppedSnapshotException dse) { 4643 // We could get this on way out because we interrupt the background flusher and it could 4644 // fail anywhere causing a DSE over in the background flusher... only it is not properly 4645 // dealt with so could still be memory hanging out when we get to here -- memory we can't 4646 // flush because the accounting is 'off' since original DSE. 4647 } 4648 this.region = null; 4649 } 4650 } 4651 4652 @Test 4653 public void testCloseAndArchiveCompactedFiles() throws IOException { 4654 byte[] CF1 = Bytes.toBytes("CF1"); 4655 byte[] CF2 = Bytes.toBytes("CF2"); 4656 this.region = initHRegion(tableName, method, CONF, CF1, CF2); 4657 for (int i = 0; i < 2; i++) { 4658 int index = i; 4659 Put put = 4660 new Put(Bytes.toBytes(index)).addColumn(CF1, Bytes.toBytes("q"), Bytes.toBytes(index)); 4661 region.put(put); 4662 region.flush(true); 4663 } 4664 4665 region.compact(true); 4666 4667 HStore store1 = region.getStore(CF1); 4668 HStore store2 = region.getStore(CF2); 4669 store1.closeAndArchiveCompactedFiles(); 4670 store2.closeAndArchiveCompactedFiles(); 4671 4672 int storefilesCount = region.getStores().stream().mapToInt(Store::getStorefilesCount).sum(); 4673 assertTrue(storefilesCount == 1); 4674 4675 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 4676 Configuration conf = region.getReadOnlyConfiguration(); 4677 RegionInfo regionInfo = region.getRegionInfo(); 4678 Path store1ArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, CF1); 4679 assertTrue(fs.exists(store1ArchiveDir)); 4680 // The archived dir of CF2 does not exist because this column family has no data at all 4681 Path store2ArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, CF2); 4682 assertFalse(fs.exists(store2ArchiveDir)); 4683 } 4684 4685 protected class PutThread extends Thread { 4686 private volatile boolean done; 4687 private volatile int numPutsFinished = 0; 4688 4689 private Throwable error = null; 4690 private int numRows; 4691 private byte[][] families; 4692 private byte[][] qualifiers; 4693 4694 private PutThread(int numRows, byte[][] families, byte[][] qualifiers) { 4695 super("PutThread"); 4696 this.numRows = numRows; 4697 this.families = families; 4698 this.qualifiers = qualifiers; 4699 } 4700 4701 /** 4702 * Block calling thread until this instance of PutThread has put at least one row. 4703 */ 4704 public void waitForFirstPut() throws InterruptedException { 4705 // wait until put thread actually puts some data 4706 while (isAlive() && numPutsFinished == 0) { 4707 checkNoError(); 4708 Thread.sleep(50); 4709 } 4710 } 4711 4712 public void done() { 4713 done = true; 4714 synchronized (this) { 4715 interrupt(); 4716 } 4717 } 4718 4719 public void checkNoError() { 4720 if (error != null) { 4721 assertNull(error); 4722 } 4723 } 4724 4725 @Override 4726 public void run() { 4727 done = false; 4728 while (!done) { 4729 try { 4730 for (int r = 0; r < numRows; r++) { 4731 byte[] row = Bytes.toBytes("row" + r); 4732 Put put = new Put(row); 4733 put.setDurability(Durability.SKIP_WAL); 4734 byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished)); 4735 for (byte[] family : families) { 4736 for (byte[] qualifier : qualifiers) { 4737 put.addColumn(family, qualifier, numPutsFinished, value); 4738 } 4739 } 4740 region.put(put); 4741 numPutsFinished++; 4742 if (numPutsFinished > 0 && numPutsFinished % 47 == 0) { 4743 LOG.debug("put iteration = {}", numPutsFinished); 4744 Delete delete = new Delete(row, (long) numPutsFinished - 30); 4745 region.delete(delete); 4746 } 4747 numPutsFinished++; 4748 } 4749 } catch (InterruptedIOException e) { 4750 // This is fine. It means we are done, or didn't get the lock on time 4751 LOG.info("Interrupted", e); 4752 } catch (IOException e) { 4753 LOG.error("Error while putting records", e); 4754 error = e; 4755 break; 4756 } 4757 } 4758 4759 } 4760 4761 } 4762 4763 /** 4764 * Writes very wide records and gets the latest row every time.. Flushes and compacts the region 4765 * aggressivly to catch issues. by flush / scan / compaction when joining threads 4766 */ 4767 @Test 4768 public void testWritesWhileGetting() throws Exception { 4769 int testCount = 50; 4770 int numRows = 1; 4771 int numFamilies = 10; 4772 int numQualifiers = 100; 4773 int compactInterval = 100; 4774 byte[][] families = new byte[numFamilies][]; 4775 for (int i = 0; i < numFamilies; i++) { 4776 families[i] = Bytes.toBytes("family" + i); 4777 } 4778 byte[][] qualifiers = new byte[numQualifiers][]; 4779 for (int i = 0; i < numQualifiers; i++) { 4780 qualifiers[i] = Bytes.toBytes("qual" + i); 4781 } 4782 4783 // This test flushes constantly and can cause many files to be created, 4784 // possibly 4785 // extending over the ulimit. Make sure compactions are aggressive in 4786 // reducing 4787 // the number of HFiles created. 4788 Configuration conf = HBaseConfiguration.create(CONF); 4789 conf.setInt("hbase.hstore.compaction.min", 1); 4790 conf.setInt("hbase.hstore.compaction.max", 1000); 4791 this.region = initHRegion(tableName, method, conf, families); 4792 PutThread putThread = null; 4793 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf); 4794 try { 4795 putThread = new PutThread(numRows, families, qualifiers); 4796 putThread.start(); 4797 putThread.waitForFirstPut(); 4798 4799 // Add a thread that flushes as fast as possible 4800 ctx.addThread(new RepeatingTestThread(ctx) { 4801 4802 @Override 4803 public void doAnAction() throws Exception { 4804 region.flush(true); 4805 // Compact regularly to avoid creating too many files and exceeding 4806 // the ulimit. 4807 region.compact(false); 4808 for (HStore store : region.getStores()) { 4809 store.closeAndArchiveCompactedFiles(); 4810 } 4811 } 4812 }); 4813 ctx.startThreads(); 4814 4815 Get get = new Get(Bytes.toBytes("row0")); 4816 Result result = null; 4817 4818 int expectedCount = numFamilies * numQualifiers; 4819 4820 long prevTimestamp = 0L; 4821 for (int i = 0; i < testCount; i++) { 4822 LOG.info("testWritesWhileGetting verify turn " + i); 4823 boolean previousEmpty = result == null || result.isEmpty(); 4824 result = region.get(get); 4825 if (!result.isEmpty() || !previousEmpty || i > compactInterval) { 4826 assertEquals("i=" + i, expectedCount, result.size()); 4827 // TODO this was removed, now what dangit?! 4828 // search looking for the qualifier in question? 4829 long timestamp = 0; 4830 for (Cell kv : result.rawCells()) { 4831 if ( 4832 CellUtil.matchingFamily(kv, families[0]) 4833 && CellUtil.matchingQualifier(kv, qualifiers[0]) 4834 ) { 4835 timestamp = kv.getTimestamp(); 4836 } 4837 } 4838 assertTrue(timestamp >= prevTimestamp); 4839 prevTimestamp = timestamp; 4840 Cell previousKV = null; 4841 4842 for (Cell kv : result.rawCells()) { 4843 byte[] thisValue = CellUtil.cloneValue(kv); 4844 if (previousKV != null) { 4845 if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) { 4846 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV 4847 + "(memStoreTS:" + previousKV.getSequenceId() + ")" + ", New KV: " + kv 4848 + "(memStoreTS:" + kv.getSequenceId() + ")"); 4849 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue)); 4850 } 4851 } 4852 previousKV = kv; 4853 } 4854 } 4855 } 4856 } finally { 4857 if (putThread != null) putThread.done(); 4858 4859 region.flush(true); 4860 4861 if (putThread != null) { 4862 putThread.join(); 4863 putThread.checkNoError(); 4864 } 4865 4866 ctx.stop(); 4867 HBaseTestingUtil.closeRegionAndWAL(this.region); 4868 this.region = null; 4869 } 4870 } 4871 4872 @Test 4873 public void testHolesInMeta() throws Exception { 4874 byte[] family = Bytes.toBytes("family"); 4875 this.region = 4876 initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF, false, family); 4877 byte[] rowNotServed = Bytes.toBytes("a"); 4878 Get g = new Get(rowNotServed); 4879 try { 4880 region.get(g); 4881 fail(); 4882 } catch (WrongRegionException x) { 4883 // OK 4884 } 4885 byte[] row = Bytes.toBytes("y"); 4886 g = new Get(row); 4887 region.get(g); 4888 } 4889 4890 @Test 4891 public void testIndexesScanWithOneDeletedRow() throws IOException { 4892 byte[] family = Bytes.toBytes("family"); 4893 4894 // Setting up region 4895 this.region = initHRegion(tableName, method, CONF, family); 4896 Put put = new Put(Bytes.toBytes(1L)); 4897 put.addColumn(family, qual1, 1L, Bytes.toBytes(1L)); 4898 region.put(put); 4899 4900 region.flush(true); 4901 4902 Delete delete = new Delete(Bytes.toBytes(1L), 1L); 4903 region.delete(delete); 4904 4905 put = new Put(Bytes.toBytes(2L)); 4906 put.addColumn(family, qual1, 2L, Bytes.toBytes(2L)); 4907 region.put(put); 4908 4909 Scan idxScan = new Scan(); 4910 idxScan.addFamily(family); 4911 idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, 4912 Arrays.<Filter> asList( 4913 new SingleColumnValueFilter(family, qual1, CompareOperator.GREATER_OR_EQUAL, 4914 new BinaryComparator(Bytes.toBytes(0L))), 4915 new SingleColumnValueFilter(family, qual1, CompareOperator.LESS_OR_EQUAL, 4916 new BinaryComparator(Bytes.toBytes(3L)))))); 4917 try (InternalScanner scanner = region.getScanner(idxScan)) { 4918 List<Cell> res = new ArrayList<>(); 4919 4920 while (scanner.next(res)) { 4921 // Ignore res value. 4922 } 4923 assertEquals(1L, res.size()); 4924 } 4925 } 4926 4927 // //////////////////////////////////////////////////////////////////////////// 4928 // Bloom filter test 4929 // //////////////////////////////////////////////////////////////////////////// 4930 @Test 4931 public void testBloomFilterSize() throws IOException { 4932 byte[] fam1 = Bytes.toBytes("fam1"); 4933 byte[] qf1 = Bytes.toBytes("col"); 4934 byte[] val1 = Bytes.toBytes("value1"); 4935 // Create Table 4936 TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName) 4937 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam1) 4938 .setMaxVersions(Integer.MAX_VALUE).setBloomFilterType(BloomType.ROWCOL).build()) 4939 .build(); 4940 RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 4941 this.region = TEST_UTIL.createLocalHRegion(info, tableDescriptor); 4942 int num_unique_rows = 10; 4943 int duplicate_multiplier = 2; 4944 int num_storefiles = 4; 4945 4946 int version = 0; 4947 for (int f = 0; f < num_storefiles; f++) { 4948 for (int i = 0; i < duplicate_multiplier; i++) { 4949 for (int j = 0; j < num_unique_rows; j++) { 4950 Put put = new Put(Bytes.toBytes("row" + j)); 4951 put.setDurability(Durability.SKIP_WAL); 4952 long ts = version++; 4953 put.addColumn(fam1, qf1, ts, val1); 4954 region.put(put); 4955 } 4956 } 4957 region.flush(true); 4958 } 4959 // before compaction 4960 HStore store = region.getStore(fam1); 4961 Collection<HStoreFile> storeFiles = store.getStorefiles(); 4962 for (HStoreFile storefile : storeFiles) { 4963 StoreFileReader reader = storefile.getReader(); 4964 reader.loadFileInfo(); 4965 reader.loadBloomfilter(); 4966 assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries()); 4967 assertEquals(num_unique_rows, reader.getFilterEntries()); 4968 } 4969 4970 region.compact(true); 4971 4972 // after compaction 4973 storeFiles = store.getStorefiles(); 4974 for (HStoreFile storefile : storeFiles) { 4975 StoreFileReader reader = storefile.getReader(); 4976 reader.loadFileInfo(); 4977 reader.loadBloomfilter(); 4978 assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries()); 4979 assertEquals(num_unique_rows, reader.getFilterEntries()); 4980 } 4981 } 4982 4983 @Test 4984 public void testAllColumnsWithBloomFilter() throws IOException { 4985 byte[] TABLE = Bytes.toBytes(name.getMethodName()); 4986 byte[] FAMILY = Bytes.toBytes("family"); 4987 4988 // Create table 4989 TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(TableName.valueOf(TABLE)) 4990 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY) 4991 .setMaxVersions(Integer.MAX_VALUE).setBloomFilterType(BloomType.ROWCOL).build()) 4992 .build(); 4993 RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 4994 this.region = TEST_UTIL.createLocalHRegion(info, tableDescriptor); 4995 // For row:0, col:0: insert versions 1 through 5. 4996 byte[] row = Bytes.toBytes("row:" + 0); 4997 byte[] column = Bytes.toBytes("column:" + 0); 4998 Put put = new Put(row); 4999 put.setDurability(Durability.SKIP_WAL); 5000 for (long idx = 1; idx <= 4; idx++) { 5001 put.addColumn(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx)); 5002 } 5003 region.put(put); 5004 5005 // Flush 5006 region.flush(true); 5007 5008 // Get rows 5009 Get get = new Get(row); 5010 get.readAllVersions(); 5011 Cell[] kvs = region.get(get).rawCells(); 5012 5013 // Check if rows are correct 5014 assertEquals(4, kvs.length); 5015 checkOneCell(kvs[0], FAMILY, 0, 0, 4); 5016 checkOneCell(kvs[1], FAMILY, 0, 0, 3); 5017 checkOneCell(kvs[2], FAMILY, 0, 0, 2); 5018 checkOneCell(kvs[3], FAMILY, 0, 0, 1); 5019 } 5020 5021 /** 5022 * Testcase to cover bug-fix for HBASE-2823 Ensures correct delete when issuing delete row on 5023 * columns with bloom filter set to row+col (BloomType.ROWCOL) 5024 */ 5025 @Test 5026 public void testDeleteRowWithBloomFilter() throws IOException { 5027 byte[] familyName = Bytes.toBytes("familyName"); 5028 5029 // Create Table 5030 TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName) 5031 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(familyName) 5032 .setMaxVersions(Integer.MAX_VALUE).setBloomFilterType(BloomType.ROWCOL).build()) 5033 .build(); 5034 RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 5035 this.region = TEST_UTIL.createLocalHRegion(info, tableDescriptor); 5036 // Insert some data 5037 byte[] row = Bytes.toBytes("row1"); 5038 byte[] col = Bytes.toBytes("col1"); 5039 5040 Put put = new Put(row); 5041 put.addColumn(familyName, col, 1, Bytes.toBytes("SomeRandomValue")); 5042 region.put(put); 5043 region.flush(true); 5044 5045 Delete del = new Delete(row); 5046 region.delete(del); 5047 region.flush(true); 5048 5049 // Get remaining rows (should have none) 5050 Get get = new Get(row); 5051 get.addColumn(familyName, col); 5052 5053 Cell[] keyValues = region.get(get).rawCells(); 5054 assertEquals(0, keyValues.length); 5055 } 5056 5057 @Test 5058 public void testgetHDFSBlocksDistribution() throws Exception { 5059 HBaseTestingUtil htu = new HBaseTestingUtil(); 5060 // Why do we set the block size in this test? If we set it smaller than the kvs, then we'll 5061 // break up the file in to more pieces that can be distributed across the three nodes and we 5062 // won't be able to have the condition this test asserts; that at least one node has 5063 // a copy of all replicas -- if small block size, then blocks are spread evenly across the 5064 // the three nodes. hfilev3 with tags seems to put us over the block size. St.Ack. 5065 // final int DEFAULT_BLOCK_SIZE = 1024; 5066 // htu.getConfiguration().setLong("dfs.blocksize", DEFAULT_BLOCK_SIZE); 5067 htu.getConfiguration().setInt("dfs.replication", 2); 5068 5069 // set up a cluster with 3 nodes 5070 SingleProcessHBaseCluster cluster = null; 5071 String dataNodeHosts[] = new String[] { "host1", "host2", "host3" }; 5072 int regionServersCount = 3; 5073 5074 try { 5075 StartTestingClusterOption option = StartTestingClusterOption.builder() 5076 .numRegionServers(regionServersCount).dataNodeHosts(dataNodeHosts).build(); 5077 cluster = htu.startMiniCluster(option); 5078 byte[][] families = { fam1, fam2 }; 5079 Table ht = htu.createTable(tableName, families); 5080 5081 // Setting up region 5082 byte row[] = Bytes.toBytes("row1"); 5083 byte col[] = Bytes.toBytes("col1"); 5084 5085 Put put = new Put(row); 5086 put.addColumn(fam1, col, 1, Bytes.toBytes("test1")); 5087 put.addColumn(fam2, col, 1, Bytes.toBytes("test2")); 5088 ht.put(put); 5089 5090 HRegion firstRegion = htu.getHBaseCluster().getRegions(tableName).get(0); 5091 firstRegion.flush(true); 5092 HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution(); 5093 5094 // Given the default replication factor is 2 and we have 2 HFiles, 5095 // we will have total of 4 replica of blocks on 3 datanodes; thus there 5096 // must be at least one host that have replica for 2 HFiles. That host's 5097 // weight will be equal to the unique block weight. 5098 long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight(); 5099 StringBuilder sb = new StringBuilder(); 5100 for (String host : blocksDistribution1.getTopHosts()) { 5101 if (sb.length() > 0) sb.append(", "); 5102 sb.append(host); 5103 sb.append("="); 5104 sb.append(blocksDistribution1.getWeight(host)); 5105 } 5106 5107 String topHost = blocksDistribution1.getTopHosts().get(0); 5108 long topHostWeight = blocksDistribution1.getWeight(topHost); 5109 String msg = "uniqueBlocksWeight=" + uniqueBlocksWeight1 + ", topHostWeight=" + topHostWeight 5110 + ", topHost=" + topHost + "; " + sb.toString(); 5111 LOG.info(msg); 5112 assertTrue(msg, uniqueBlocksWeight1 == topHostWeight); 5113 5114 // use the static method to compute the value, it should be the same. 5115 // static method is used by load balancer or other components 5116 HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution( 5117 htu.getConfiguration(), firstRegion.getTableDescriptor(), firstRegion.getRegionInfo()); 5118 long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight(); 5119 5120 assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2); 5121 5122 ht.close(); 5123 } finally { 5124 if (cluster != null) { 5125 htu.shutdownMiniCluster(); 5126 } 5127 } 5128 } 5129 5130 /** 5131 * Testcase to check state of region initialization task set to ABORTED or not if any exceptions 5132 * during initialization 5133 */ 5134 @Test 5135 public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception { 5136 RegionInfo info; 5137 try { 5138 FileSystem fs = mock(FileSystem.class); 5139 when(fs.exists(any())).thenThrow(new IOException()); 5140 TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(tableName); 5141 ColumnFamilyDescriptor columnFamilyDescriptor = 5142 ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf")).build(); 5143 tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor); 5144 info = RegionInfoBuilder.newBuilder(tableName).build(); 5145 Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization"); 5146 region = HRegion.newHRegion(path, null, fs, CONF, info, tableDescriptorBuilder.build(), null); 5147 // region initialization throws IOException and set task state to ABORTED. 5148 region.initialize(); 5149 fail("Region initialization should fail due to IOException"); 5150 } catch (IOException io) { 5151 List<MonitoredTask> tasks = TaskMonitor.get().getTasks(); 5152 for (MonitoredTask monitoredTask : tasks) { 5153 if ( 5154 !(monitoredTask instanceof MonitoredRPCHandler) 5155 && monitoredTask.getDescription().contains(region.toString()) 5156 ) { 5157 assertTrue("Region state should be ABORTED.", 5158 monitoredTask.getState().equals(MonitoredTask.State.ABORTED)); 5159 break; 5160 } 5161 } 5162 } 5163 } 5164 5165 /** 5166 * Verifies that the .regioninfo file is written on region creation and that is recreated if 5167 * missing during region opening. 5168 */ 5169 @Test 5170 public void testRegionInfoFileCreation() throws IOException { 5171 Path rootDir = new Path(dir + "testRegionInfoFileCreation"); 5172 5173 TableDescriptorBuilder tableDescriptorBuilder = 5174 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())); 5175 ColumnFamilyDescriptor columnFamilyDescriptor = 5176 ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf")).build(); 5177 tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor); 5178 TableDescriptor tableDescriptor = tableDescriptorBuilder.build(); 5179 5180 RegionInfo hri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 5181 5182 // Create a region and skip the initialization (like CreateTableHandler) 5183 region = HBaseTestingUtil.createRegionAndWAL(hri, rootDir, CONF, tableDescriptor, false); 5184 Path regionDir = region.getRegionFileSystem().getRegionDir(); 5185 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 5186 HBaseTestingUtil.closeRegionAndWAL(region); 5187 5188 Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE); 5189 5190 // Verify that the .regioninfo file is present 5191 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir", 5192 fs.exists(regionInfoFile)); 5193 5194 // Try to open the region 5195 region = HRegion.openHRegion(rootDir, hri, tableDescriptor, null, CONF); 5196 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir()); 5197 HBaseTestingUtil.closeRegionAndWAL(region); 5198 5199 // Verify that the .regioninfo file is still there 5200 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir", 5201 fs.exists(regionInfoFile)); 5202 5203 // Remove the .regioninfo file and verify is recreated on region open 5204 fs.delete(regionInfoFile, true); 5205 assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir", 5206 fs.exists(regionInfoFile)); 5207 5208 region = HRegion.openHRegion(rootDir, hri, tableDescriptor, null, CONF); 5209 // region = TEST_UTIL.openHRegion(hri, htd); 5210 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir()); 5211 HBaseTestingUtil.closeRegionAndWAL(region); 5212 5213 // Verify that the .regioninfo file is still there 5214 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir", 5215 fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE))); 5216 5217 region = null; 5218 } 5219 5220 /** 5221 * TestCase for increment 5222 */ 5223 private static class Incrementer implements Runnable { 5224 private HRegion region; 5225 private final static byte[] incRow = Bytes.toBytes("incRow"); 5226 private final static byte[] family = Bytes.toBytes("family"); 5227 private final static byte[] qualifier = Bytes.toBytes("qualifier"); 5228 private final static long ONE = 1L; 5229 private int incCounter; 5230 5231 public Incrementer(HRegion region, int incCounter) { 5232 this.region = region; 5233 this.incCounter = incCounter; 5234 } 5235 5236 @Override 5237 public void run() { 5238 int count = 0; 5239 while (count < incCounter) { 5240 Increment inc = new Increment(incRow); 5241 inc.addColumn(family, qualifier, ONE); 5242 count++; 5243 try { 5244 region.increment(inc); 5245 } catch (IOException e) { 5246 LOG.info("Count=" + count + ", " + e); 5247 break; 5248 } 5249 } 5250 } 5251 } 5252 5253 /** 5254 * Test case to check increment function with memstore flushing 5255 */ 5256 @Test 5257 public void testParallelIncrementWithMemStoreFlush() throws Exception { 5258 byte[] family = Incrementer.family; 5259 this.region = initHRegion(tableName, method, CONF, family); 5260 final HRegion region = this.region; 5261 final AtomicBoolean incrementDone = new AtomicBoolean(false); 5262 Runnable flusher = new Runnable() { 5263 @Override 5264 public void run() { 5265 while (!incrementDone.get()) { 5266 try { 5267 region.flush(true); 5268 } catch (Exception e) { 5269 e.printStackTrace(); 5270 } 5271 } 5272 } 5273 }; 5274 5275 // after all increment finished, the row will increment to 20*100 = 2000 5276 int threadNum = 20; 5277 int incCounter = 100; 5278 long expected = (long) threadNum * incCounter; 5279 Thread[] incrementers = new Thread[threadNum]; 5280 Thread flushThread = new Thread(flusher); 5281 for (int i = 0; i < threadNum; i++) { 5282 incrementers[i] = new Thread(new Incrementer(this.region, incCounter)); 5283 incrementers[i].start(); 5284 } 5285 flushThread.start(); 5286 for (int i = 0; i < threadNum; i++) { 5287 incrementers[i].join(); 5288 } 5289 5290 incrementDone.set(true); 5291 flushThread.join(); 5292 5293 Get get = new Get(Incrementer.incRow); 5294 get.addColumn(Incrementer.family, Incrementer.qualifier); 5295 get.readVersions(1); 5296 Result res = this.region.get(get); 5297 List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier); 5298 5299 // we just got the latest version 5300 assertEquals(1, kvs.size()); 5301 Cell kv = kvs.get(0); 5302 assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset())); 5303 } 5304 5305 /** 5306 * TestCase for append 5307 */ 5308 private static class Appender implements Runnable { 5309 private HRegion region; 5310 private final static byte[] appendRow = Bytes.toBytes("appendRow"); 5311 private final static byte[] family = Bytes.toBytes("family"); 5312 private final static byte[] qualifier = Bytes.toBytes("qualifier"); 5313 private final static byte[] CHAR = Bytes.toBytes("a"); 5314 private int appendCounter; 5315 5316 public Appender(HRegion region, int appendCounter) { 5317 this.region = region; 5318 this.appendCounter = appendCounter; 5319 } 5320 5321 @Override 5322 public void run() { 5323 int count = 0; 5324 while (count < appendCounter) { 5325 Append app = new Append(appendRow); 5326 app.addColumn(family, qualifier, CHAR); 5327 count++; 5328 try { 5329 region.append(app); 5330 } catch (IOException e) { 5331 LOG.info("Count=" + count + ", max=" + appendCounter + ", " + e); 5332 break; 5333 } 5334 } 5335 } 5336 } 5337 5338 /** 5339 * Test case to check append function with memstore flushing 5340 */ 5341 @Test 5342 public void testParallelAppendWithMemStoreFlush() throws Exception { 5343 byte[] family = Appender.family; 5344 this.region = initHRegion(tableName, method, CONF, family); 5345 final HRegion region = this.region; 5346 final AtomicBoolean appendDone = new AtomicBoolean(false); 5347 Runnable flusher = new Runnable() { 5348 @Override 5349 public void run() { 5350 while (!appendDone.get()) { 5351 try { 5352 region.flush(true); 5353 } catch (Exception e) { 5354 e.printStackTrace(); 5355 } 5356 } 5357 } 5358 }; 5359 5360 // After all append finished, the value will append to threadNum * 5361 // appendCounter Appender.CHAR 5362 int threadNum = 20; 5363 int appendCounter = 100; 5364 byte[] expected = new byte[threadNum * appendCounter]; 5365 for (int i = 0; i < threadNum * appendCounter; i++) { 5366 System.arraycopy(Appender.CHAR, 0, expected, i, 1); 5367 } 5368 Thread[] appenders = new Thread[threadNum]; 5369 Thread flushThread = new Thread(flusher); 5370 for (int i = 0; i < threadNum; i++) { 5371 appenders[i] = new Thread(new Appender(this.region, appendCounter)); 5372 appenders[i].start(); 5373 } 5374 flushThread.start(); 5375 for (int i = 0; i < threadNum; i++) { 5376 appenders[i].join(); 5377 } 5378 5379 appendDone.set(true); 5380 flushThread.join(); 5381 5382 Get get = new Get(Appender.appendRow); 5383 get.addColumn(Appender.family, Appender.qualifier); 5384 get.readVersions(1); 5385 Result res = this.region.get(get); 5386 List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier); 5387 5388 // we just got the latest version 5389 assertEquals(1, kvs.size()); 5390 Cell kv = kvs.get(0); 5391 byte[] appendResult = new byte[kv.getValueLength()]; 5392 System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength()); 5393 assertArrayEquals(expected, appendResult); 5394 } 5395 5396 /** 5397 * Test case to check put function with memstore flushing for same row, same ts 5398 */ 5399 @Test 5400 public void testPutWithMemStoreFlush() throws Exception { 5401 byte[] family = Bytes.toBytes("family"); 5402 byte[] qualifier = Bytes.toBytes("qualifier"); 5403 byte[] row = Bytes.toBytes("putRow"); 5404 byte[] value = null; 5405 this.region = initHRegion(tableName, method, CONF, family); 5406 Put put = null; 5407 Get get = null; 5408 List<Cell> kvs = null; 5409 Result res = null; 5410 5411 put = new Put(row); 5412 value = Bytes.toBytes("value0"); 5413 put.addColumn(family, qualifier, 1234567L, value); 5414 region.put(put); 5415 get = new Get(row); 5416 get.addColumn(family, qualifier); 5417 get.readAllVersions(); 5418 res = this.region.get(get); 5419 kvs = res.getColumnCells(family, qualifier); 5420 assertEquals(1, kvs.size()); 5421 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0))); 5422 5423 region.flush(true); 5424 get = new Get(row); 5425 get.addColumn(family, qualifier); 5426 get.readAllVersions(); 5427 res = this.region.get(get); 5428 kvs = res.getColumnCells(family, qualifier); 5429 assertEquals(1, kvs.size()); 5430 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0))); 5431 5432 put = new Put(row); 5433 value = Bytes.toBytes("value1"); 5434 put.addColumn(family, qualifier, 1234567L, value); 5435 region.put(put); 5436 get = new Get(row); 5437 get.addColumn(family, qualifier); 5438 get.readAllVersions(); 5439 res = this.region.get(get); 5440 kvs = res.getColumnCells(family, qualifier); 5441 assertEquals(1, kvs.size()); 5442 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0))); 5443 5444 region.flush(true); 5445 get = new Get(row); 5446 get.addColumn(family, qualifier); 5447 get.readAllVersions(); 5448 res = this.region.get(get); 5449 kvs = res.getColumnCells(family, qualifier); 5450 assertEquals(1, kvs.size()); 5451 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0))); 5452 } 5453 5454 /** 5455 * For this test,the spied {@link AsyncFSWAL} can not work properly because of a Mockito defect 5456 * that can not deal with classes which have a field of an inner class. See discussions in 5457 * HBASE-15536.When we reuse the code of {@link AsyncFSWAL} for {@link FSHLog}, this test could 5458 * not work for {@link FSHLog} also. 5459 */ 5460 @Test 5461 @Ignore 5462 public void testDurability() throws Exception { 5463 // there are 5 x 5 cases: 5464 // table durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT) x mutation 5465 // durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT) 5466 5467 // expected cases for append and sync wal 5468 durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false); 5469 durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false); 5470 durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false); 5471 5472 durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false); 5473 durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false); 5474 durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false); 5475 5476 durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false); 5477 durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false); 5478 5479 durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false); 5480 durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false); 5481 5482 durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false); 5483 durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false); 5484 durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false); 5485 5486 // expected cases for async wal 5487 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false); 5488 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false); 5489 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false); 5490 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false); 5491 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false); 5492 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false); 5493 5494 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true); 5495 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true); 5496 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true); 5497 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true); 5498 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true); 5499 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true); 5500 5501 // expect skip wal cases 5502 durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false); 5503 durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false); 5504 durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false); 5505 durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false); 5506 durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false); 5507 durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false); 5508 5509 } 5510 5511 private void durabilityTest(String method, Durability tableDurability, 5512 Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync, 5513 final boolean expectSyncFromLogSyncer) throws Exception { 5514 Configuration conf = HBaseConfiguration.create(CONF); 5515 conf.setLong(AbstractFSWAL.WAL_SHUTDOWN_WAIT_TIMEOUT_MS, 60 * 60 * 1000); 5516 method = method + "_" + tableDurability.name() + "_" + mutationDurability.name(); 5517 byte[] family = Bytes.toBytes("family"); 5518 Path logDir = new Path(new Path(dir + method), "log"); 5519 final Configuration walConf = new Configuration(conf); 5520 CommonFSUtils.setRootDir(walConf, logDir); 5521 // XXX: The spied AsyncFSWAL can not work properly because of a Mockito defect that can not 5522 // deal with classes which have a field of an inner class. See discussions in HBASE-15536. 5523 walConf.set(WALFactory.WAL_PROVIDER, "filesystem"); 5524 final WALFactory wals = new WALFactory(walConf, HBaseTestingUtil.getRandomUUID().toString()); 5525 final WAL wal = spy(wals.getWAL(RegionInfoBuilder.newBuilder(tableName).build())); 5526 this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF, 5527 false, tableDurability, wal, new byte[][] { family }); 5528 5529 Put put = new Put(Bytes.toBytes("r1")); 5530 put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); 5531 put.setDurability(mutationDurability); 5532 region.put(put); 5533 5534 // verify append called or not 5535 verify(wal, expectAppend ? times(1) : never()).appendData((RegionInfo) any(), 5536 (WALKeyImpl) any(), (WALEdit) any()); 5537 5538 // verify sync called or not 5539 if (expectSync || expectSyncFromLogSyncer) { 5540 TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() { 5541 @Override 5542 public boolean evaluate() throws Exception { 5543 try { 5544 if (expectSync) { 5545 verify(wal, times(1)).sync(anyLong()); // Hregion calls this one 5546 } else if (expectSyncFromLogSyncer) { 5547 verify(wal, times(1)).sync(); // wal syncer calls this one 5548 } 5549 } catch (Throwable ignore) { 5550 } 5551 return true; 5552 } 5553 }); 5554 } else { 5555 // verify(wal, never()).sync(anyLong()); 5556 verify(wal, never()).sync(); 5557 } 5558 5559 HBaseTestingUtil.closeRegionAndWAL(this.region); 5560 wals.close(); 5561 this.region = null; 5562 } 5563 5564 @Test 5565 public void testRegionReplicaSecondary() throws IOException { 5566 // create a primary region, load some data and flush 5567 // create a secondary region, and do a get against that 5568 Path rootDir = new Path(dir + name.getMethodName()); 5569 CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir); 5570 5571 byte[][] families = 5572 new byte[][] { Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3") }; 5573 byte[] cq = Bytes.toBytes("cq"); 5574 TableDescriptorBuilder builder = 5575 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())); 5576 for (byte[] family : families) { 5577 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)); 5578 } 5579 TableDescriptor tableDescriptor = builder.build(); 5580 long time = EnvironmentEdgeManager.currentTime(); 5581 RegionInfo primaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5582 .setRegionId(time).setReplicaId(0).build(); 5583 RegionInfo secondaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5584 .setRegionId(time).setReplicaId(1).build(); 5585 5586 HRegion primaryRegion = null, secondaryRegion = null; 5587 5588 try { 5589 primaryRegion = HBaseTestingUtil.createRegionAndWAL(primaryHri, rootDir, 5590 TEST_UTIL.getConfiguration(), tableDescriptor); 5591 5592 // load some data 5593 putData(primaryRegion, 0, 1000, cq, families); 5594 5595 // flush region 5596 primaryRegion.flush(true); 5597 5598 // open secondary region 5599 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, tableDescriptor, null, CONF); 5600 5601 verifyData(secondaryRegion, 0, 1000, cq, families); 5602 } finally { 5603 if (primaryRegion != null) { 5604 HBaseTestingUtil.closeRegionAndWAL(primaryRegion); 5605 } 5606 if (secondaryRegion != null) { 5607 HBaseTestingUtil.closeRegionAndWAL(secondaryRegion); 5608 } 5609 } 5610 } 5611 5612 @Test 5613 public void testRegionReplicaSecondaryIsReadOnly() throws IOException { 5614 // create a primary region, load some data and flush 5615 // create a secondary region, and do a put against that 5616 Path rootDir = new Path(dir + name.getMethodName()); 5617 CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir); 5618 5619 byte[][] families = 5620 new byte[][] { Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3") }; 5621 byte[] cq = Bytes.toBytes("cq"); 5622 TableDescriptorBuilder builder = 5623 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())); 5624 for (byte[] family : families) { 5625 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)); 5626 } 5627 TableDescriptor tableDescriptor = builder.build(); 5628 long time = EnvironmentEdgeManager.currentTime(); 5629 RegionInfo primaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5630 .setRegionId(time).setReplicaId(0).build(); 5631 RegionInfo secondaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5632 .setRegionId(time).setReplicaId(1).build(); 5633 5634 HRegion primaryRegion = null, secondaryRegion = null; 5635 5636 try { 5637 primaryRegion = HBaseTestingUtil.createRegionAndWAL(primaryHri, rootDir, 5638 TEST_UTIL.getConfiguration(), tableDescriptor); 5639 5640 // load some data 5641 putData(primaryRegion, 0, 1000, cq, families); 5642 5643 // flush region 5644 primaryRegion.flush(true); 5645 5646 // open secondary region 5647 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, tableDescriptor, null, CONF); 5648 5649 try { 5650 putData(secondaryRegion, 0, 1000, cq, families); 5651 fail("Should have thrown exception"); 5652 } catch (IOException ex) { 5653 // expected 5654 } 5655 } finally { 5656 if (primaryRegion != null) { 5657 HBaseTestingUtil.closeRegionAndWAL(primaryRegion); 5658 } 5659 if (secondaryRegion != null) { 5660 HBaseTestingUtil.closeRegionAndWAL(secondaryRegion); 5661 } 5662 } 5663 } 5664 5665 static WALFactory createWALFactory(Configuration conf, Path rootDir) throws IOException { 5666 Configuration confForWAL = new Configuration(conf); 5667 confForWAL.set(HConstants.HBASE_DIR, rootDir.toString()); 5668 return new WALFactory(confForWAL, "hregion-" + RandomStringUtils.randomNumeric(8)); 5669 } 5670 5671 @Test 5672 public void testCompactionFromPrimary() throws IOException { 5673 Path rootDir = new Path(dir + name.getMethodName()); 5674 CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir); 5675 5676 byte[][] families = 5677 new byte[][] { Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3") }; 5678 byte[] cq = Bytes.toBytes("cq"); 5679 TableDescriptorBuilder builder = 5680 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())); 5681 for (byte[] family : families) { 5682 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)); 5683 } 5684 TableDescriptor tableDescriptor = builder.build(); 5685 long time = EnvironmentEdgeManager.currentTime(); 5686 RegionInfo primaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5687 .setRegionId(time).setReplicaId(0).build(); 5688 RegionInfo secondaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5689 .setRegionId(time).setReplicaId(1).build(); 5690 5691 HRegion primaryRegion = null, secondaryRegion = null; 5692 5693 try { 5694 primaryRegion = HBaseTestingUtil.createRegionAndWAL(primaryHri, rootDir, 5695 TEST_UTIL.getConfiguration(), tableDescriptor); 5696 5697 // load some data 5698 putData(primaryRegion, 0, 1000, cq, families); 5699 5700 // flush region 5701 primaryRegion.flush(true); 5702 5703 // open secondary region 5704 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, tableDescriptor, null, CONF); 5705 5706 // move the file of the primary region to the archive, simulating a compaction 5707 Collection<HStoreFile> storeFiles = primaryRegion.getStore(families[0]).getStorefiles(); 5708 primaryRegion.getRegionFileSystem().removeStoreFiles(Bytes.toString(families[0]), storeFiles); 5709 Collection<StoreFileInfo> storeFileInfos = 5710 primaryRegion.getRegionFileSystem().getStoreFiles(Bytes.toString(families[0])); 5711 Assert.assertTrue(storeFileInfos == null || storeFileInfos.isEmpty()); 5712 5713 verifyData(secondaryRegion, 0, 1000, cq, families); 5714 } finally { 5715 if (primaryRegion != null) { 5716 HBaseTestingUtil.closeRegionAndWAL(primaryRegion); 5717 } 5718 if (secondaryRegion != null) { 5719 HBaseTestingUtil.closeRegionAndWAL(secondaryRegion); 5720 } 5721 } 5722 } 5723 5724 private void putData(int startRow, int numRows, byte[] qf, byte[]... families) 5725 throws IOException { 5726 putData(this.region, startRow, numRows, qf, families); 5727 } 5728 5729 private void putData(HRegion region, int startRow, int numRows, byte[] qf, byte[]... families) 5730 throws IOException { 5731 putData(region, Durability.SKIP_WAL, startRow, numRows, qf, families); 5732 } 5733 5734 static void putData(HRegion region, Durability durability, int startRow, int numRows, byte[] qf, 5735 byte[]... families) throws IOException { 5736 for (int i = startRow; i < startRow + numRows; i++) { 5737 Put put = new Put(Bytes.toBytes("" + i)); 5738 put.setDurability(durability); 5739 for (byte[] family : families) { 5740 put.addColumn(family, qf, null); 5741 } 5742 region.put(put); 5743 LOG.info(put.toString()); 5744 } 5745 } 5746 5747 static void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families) 5748 throws IOException { 5749 for (int i = startRow; i < startRow + numRows; i++) { 5750 byte[] row = Bytes.toBytes("" + i); 5751 Get get = new Get(row); 5752 for (byte[] family : families) { 5753 get.addColumn(family, qf); 5754 } 5755 Result result = newReg.get(get); 5756 Cell[] raw = result.rawCells(); 5757 assertEquals(families.length, result.size()); 5758 for (int j = 0; j < families.length; j++) { 5759 assertTrue(CellUtil.matchingRows(raw[j], row)); 5760 assertTrue(CellUtil.matchingFamily(raw[j], families[j])); 5761 assertTrue(CellUtil.matchingQualifier(raw[j], qf)); 5762 } 5763 } 5764 } 5765 5766 static void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException { 5767 // Now I have k, get values out and assert they are as expected. 5768 Get get = new Get(k).addFamily(family).readAllVersions(); 5769 Cell[] results = r.get(get).rawCells(); 5770 for (int j = 0; j < results.length; j++) { 5771 byte[] tmp = CellUtil.cloneValue(results[j]); 5772 // Row should be equal to value every time. 5773 assertTrue(Bytes.equals(k, tmp)); 5774 } 5775 } 5776 5777 /* 5778 * Assert first value in the passed region is <code>firstValue</code>. 5779 */ 5780 protected void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue) 5781 throws IOException { 5782 byte[][] families = { fs }; 5783 Scan scan = new Scan(); 5784 for (int i = 0; i < families.length; i++) 5785 scan.addFamily(families[i]); 5786 try (InternalScanner s = r.getScanner(scan)) { 5787 List<Cell> curVals = new ArrayList<>(); 5788 boolean first = true; 5789 OUTER_LOOP: while (s.next(curVals)) { 5790 for (Cell kv : curVals) { 5791 byte[] val = CellUtil.cloneValue(kv); 5792 byte[] curval = val; 5793 if (first) { 5794 first = false; 5795 assertTrue(Bytes.compareTo(curval, firstValue) == 0); 5796 } else { 5797 // Not asserting anything. Might as well break. 5798 break OUTER_LOOP; 5799 } 5800 } 5801 } 5802 } 5803 } 5804 5805 /** 5806 * Test that we get the expected flush results back 5807 */ 5808 @Test 5809 public void testFlushResult() throws IOException { 5810 byte[] family = Bytes.toBytes("family"); 5811 5812 this.region = initHRegion(tableName, method, family); 5813 5814 // empty memstore, flush doesn't run 5815 HRegion.FlushResult fr = region.flush(true); 5816 assertFalse(fr.isFlushSucceeded()); 5817 assertFalse(fr.isCompactionNeeded()); 5818 5819 // Flush enough files to get up to the threshold, doesn't need compactions 5820 for (int i = 0; i < 2; i++) { 5821 Put put = new Put(tableName.toBytes()).addColumn(family, family, tableName.toBytes()); 5822 region.put(put); 5823 fr = region.flush(true); 5824 assertTrue(fr.isFlushSucceeded()); 5825 assertFalse(fr.isCompactionNeeded()); 5826 } 5827 5828 // Two flushes after the threshold, compactions are needed 5829 for (int i = 0; i < 2; i++) { 5830 Put put = new Put(tableName.toBytes()).addColumn(family, family, tableName.toBytes()); 5831 region.put(put); 5832 fr = region.flush(true); 5833 assertTrue(fr.isFlushSucceeded()); 5834 assertTrue(fr.isCompactionNeeded()); 5835 } 5836 } 5837 5838 protected Configuration initSplit() { 5839 // Always compact if there is more than one store file. 5840 CONF.setInt("hbase.hstore.compactionThreshold", 2); 5841 5842 CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000); 5843 5844 // Increase the amount of time between client retries 5845 CONF.setLong("hbase.client.pause", 15 * 1000); 5846 5847 // This size should make it so we always split using the addContent 5848 // below. After adding all data, the first region is 1.3M 5849 CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128); 5850 return CONF; 5851 } 5852 5853 /** 5854 * @return A region on which you must call {@link HBaseTestingUtil#closeRegionAndWAL(HRegion)} 5855 * when done. 5856 */ 5857 protected HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf, 5858 byte[]... families) throws IOException { 5859 return initHRegion(tableName, callingMethod, conf, false, families); 5860 } 5861 5862 /** 5863 * @return A region on which you must call {@link HBaseTestingUtil#closeRegionAndWAL(HRegion)} 5864 * when done. 5865 */ 5866 private HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf, 5867 boolean isReadOnly, byte[]... families) throws IOException { 5868 return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families); 5869 } 5870 5871 private HRegion initHRegion(TableName tableName, byte[] startKey, byte[] stopKey, 5872 String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families) 5873 throws IOException { 5874 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(callingMethod + ".log"); 5875 RegionInfo hri = 5876 RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey).setEndKey(stopKey).build(); 5877 final WAL wal = HBaseTestingUtil.createWal(conf, logDir, hri); 5878 return initHRegion(tableName, startKey, stopKey, conf, isReadOnly, Durability.SYNC_WAL, wal, 5879 families); 5880 } 5881 5882 /** 5883 * @return A region on which you must call {@link HBaseTestingUtil#closeRegionAndWAL(HRegion)} 5884 * when done. 5885 */ 5886 protected HRegion initHRegion(TableName tableName, byte[] startKey, byte[] stopKey, 5887 Configuration conf, boolean isReadOnly, Durability durability, WAL wal, byte[]... families) 5888 throws IOException { 5889 ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null, 5890 MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT); 5891 return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, conf, isReadOnly, durability, 5892 wal, families); 5893 } 5894 5895 /** 5896 * Assert that the passed in Cell has expected contents for the specified row, column & timestamp. 5897 */ 5898 private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) { 5899 String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts; 5900 assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx, 5901 Bytes.toString(CellUtil.cloneRow(kv))); 5902 assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf), 5903 Bytes.toString(CellUtil.cloneFamily(kv))); 5904 assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx, 5905 Bytes.toString(CellUtil.cloneQualifier(kv))); 5906 assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp()); 5907 assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts, 5908 Bytes.toString(CellUtil.cloneValue(kv))); 5909 } 5910 5911 @Test 5912 public void testReverseScanner_FromMemStore_SingleCF_Normal() throws IOException { 5913 byte[] rowC = Bytes.toBytes("rowC"); 5914 byte[] rowA = Bytes.toBytes("rowA"); 5915 byte[] rowB = Bytes.toBytes("rowB"); 5916 byte[] cf = Bytes.toBytes("CF"); 5917 byte[][] families = { cf }; 5918 byte[] col = Bytes.toBytes("C"); 5919 long ts = 1; 5920 this.region = initHRegion(tableName, method, families); 5921 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null); 5922 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, null); 5923 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null); 5924 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null); 5925 Put put = null; 5926 put = new Put(rowC); 5927 put.add(kv1); 5928 put.add(kv11); 5929 region.put(put); 5930 put = new Put(rowA); 5931 put.add(kv2); 5932 region.put(put); 5933 put = new Put(rowB); 5934 put.add(kv3); 5935 region.put(put); 5936 5937 Scan scan = new Scan().withStartRow(rowC); 5938 scan.readVersions(5); 5939 scan.setReversed(true); 5940 try (InternalScanner scanner = region.getScanner(scan)) { 5941 List<Cell> currRow = new ArrayList<>(); 5942 boolean hasNext = scanner.next(currRow); 5943 assertEquals(2, currRow.size()); 5944 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5945 currRow.get(0).getRowLength(), rowC, 0, rowC.length)); 5946 assertTrue(hasNext); 5947 currRow.clear(); 5948 hasNext = scanner.next(currRow); 5949 assertEquals(1, currRow.size()); 5950 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5951 currRow.get(0).getRowLength(), rowB, 0, rowB.length)); 5952 assertTrue(hasNext); 5953 currRow.clear(); 5954 hasNext = scanner.next(currRow); 5955 assertEquals(1, currRow.size()); 5956 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5957 currRow.get(0).getRowLength(), rowA, 0, rowA.length)); 5958 assertFalse(hasNext); 5959 } 5960 } 5961 5962 @Test 5963 public void testReverseScanner_FromMemStore_SingleCF_LargerKey() throws IOException { 5964 byte[] rowC = Bytes.toBytes("rowC"); 5965 byte[] rowA = Bytes.toBytes("rowA"); 5966 byte[] rowB = Bytes.toBytes("rowB"); 5967 byte[] rowD = Bytes.toBytes("rowD"); 5968 byte[] cf = Bytes.toBytes("CF"); 5969 byte[][] families = { cf }; 5970 byte[] col = Bytes.toBytes("C"); 5971 long ts = 1; 5972 this.region = initHRegion(tableName, method, families); 5973 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null); 5974 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, null); 5975 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null); 5976 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null); 5977 Put put = null; 5978 put = new Put(rowC); 5979 put.add(kv1); 5980 put.add(kv11); 5981 region.put(put); 5982 put = new Put(rowA); 5983 put.add(kv2); 5984 region.put(put); 5985 put = new Put(rowB); 5986 put.add(kv3); 5987 region.put(put); 5988 5989 Scan scan = new Scan().withStartRow(rowD); 5990 List<Cell> currRow = new ArrayList<>(); 5991 scan.setReversed(true); 5992 scan.readVersions(5); 5993 try (InternalScanner scanner = region.getScanner(scan)) { 5994 boolean hasNext = scanner.next(currRow); 5995 assertEquals(2, currRow.size()); 5996 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5997 currRow.get(0).getRowLength(), rowC, 0, rowC.length)); 5998 assertTrue(hasNext); 5999 currRow.clear(); 6000 hasNext = scanner.next(currRow); 6001 assertEquals(1, currRow.size()); 6002 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6003 currRow.get(0).getRowLength(), rowB, 0, rowB.length)); 6004 assertTrue(hasNext); 6005 currRow.clear(); 6006 hasNext = scanner.next(currRow); 6007 assertEquals(1, currRow.size()); 6008 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6009 currRow.get(0).getRowLength(), rowA, 0, rowA.length)); 6010 assertFalse(hasNext); 6011 } 6012 } 6013 6014 @Test 6015 public void testReverseScanner_FromMemStore_SingleCF_FullScan() throws IOException { 6016 byte[] rowC = Bytes.toBytes("rowC"); 6017 byte[] rowA = Bytes.toBytes("rowA"); 6018 byte[] rowB = Bytes.toBytes("rowB"); 6019 byte[] cf = Bytes.toBytes("CF"); 6020 byte[][] families = { cf }; 6021 byte[] col = Bytes.toBytes("C"); 6022 long ts = 1; 6023 this.region = initHRegion(tableName, method, families); 6024 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null); 6025 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, null); 6026 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null); 6027 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null); 6028 Put put = null; 6029 put = new Put(rowC); 6030 put.add(kv1); 6031 put.add(kv11); 6032 region.put(put); 6033 put = new Put(rowA); 6034 put.add(kv2); 6035 region.put(put); 6036 put = new Put(rowB); 6037 put.add(kv3); 6038 region.put(put); 6039 Scan scan = new Scan(); 6040 List<Cell> currRow = new ArrayList<>(); 6041 scan.setReversed(true); 6042 try (InternalScanner scanner = region.getScanner(scan)) { 6043 boolean hasNext = scanner.next(currRow); 6044 assertEquals(1, currRow.size()); 6045 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6046 currRow.get(0).getRowLength(), rowC, 0, rowC.length)); 6047 assertTrue(hasNext); 6048 currRow.clear(); 6049 hasNext = scanner.next(currRow); 6050 assertEquals(1, currRow.size()); 6051 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6052 currRow.get(0).getRowLength(), rowB, 0, rowB.length)); 6053 assertTrue(hasNext); 6054 currRow.clear(); 6055 hasNext = scanner.next(currRow); 6056 assertEquals(1, currRow.size()); 6057 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6058 currRow.get(0).getRowLength(), rowA, 0, rowA.length)); 6059 assertFalse(hasNext); 6060 } 6061 } 6062 6063 @Test 6064 public void testReverseScanner_moreRowsMayExistAfter() throws IOException { 6065 // case for "INCLUDE_AND_SEEK_NEXT_ROW & SEEK_NEXT_ROW" endless loop 6066 byte[] rowA = Bytes.toBytes("rowA"); 6067 byte[] rowB = Bytes.toBytes("rowB"); 6068 byte[] rowC = Bytes.toBytes("rowC"); 6069 byte[] rowD = Bytes.toBytes("rowD"); 6070 byte[] rowE = Bytes.toBytes("rowE"); 6071 byte[] cf = Bytes.toBytes("CF"); 6072 byte[][] families = { cf }; 6073 byte[] col1 = Bytes.toBytes("col1"); 6074 byte[] col2 = Bytes.toBytes("col2"); 6075 long ts = 1; 6076 this.region = initHRegion(tableName, method, families); 6077 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null); 6078 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null); 6079 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null); 6080 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null); 6081 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null); 6082 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null); 6083 Put put = null; 6084 put = new Put(rowA); 6085 put.add(kv1); 6086 region.put(put); 6087 put = new Put(rowB); 6088 put.add(kv2); 6089 region.put(put); 6090 put = new Put(rowC); 6091 put.add(kv3); 6092 region.put(put); 6093 put = new Put(rowD); 6094 put.add(kv4_1); 6095 region.put(put); 6096 put = new Put(rowD); 6097 put.add(kv4_2); 6098 region.put(put); 6099 put = new Put(rowE); 6100 put.add(kv5); 6101 region.put(put); 6102 region.flush(true); 6103 Scan scan = new Scan().withStartRow(rowD).withStopRow(rowA); 6104 scan.addColumn(families[0], col1); 6105 scan.setReversed(true); 6106 List<Cell> currRow = new ArrayList<>(); 6107 try (InternalScanner scanner = region.getScanner(scan)) { 6108 boolean hasNext = scanner.next(currRow); 6109 assertEquals(1, currRow.size()); 6110 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6111 currRow.get(0).getRowLength(), rowD, 0, rowD.length)); 6112 assertTrue(hasNext); 6113 currRow.clear(); 6114 hasNext = scanner.next(currRow); 6115 assertEquals(1, currRow.size()); 6116 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6117 currRow.get(0).getRowLength(), rowC, 0, rowC.length)); 6118 assertTrue(hasNext); 6119 currRow.clear(); 6120 hasNext = scanner.next(currRow); 6121 assertEquals(1, currRow.size()); 6122 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6123 currRow.get(0).getRowLength(), rowB, 0, rowB.length)); 6124 assertFalse(hasNext); 6125 } 6126 6127 scan = new Scan().withStartRow(rowD).withStopRow(rowA); 6128 scan.addColumn(families[0], col2); 6129 scan.setReversed(true); 6130 currRow.clear(); 6131 try (InternalScanner scanner = region.getScanner(scan)) { 6132 boolean hasNext = scanner.next(currRow); 6133 assertEquals(1, currRow.size()); 6134 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6135 currRow.get(0).getRowLength(), rowD, 0, rowD.length)); 6136 assertTrue(hasNext); 6137 } 6138 } 6139 6140 @Test 6141 public void testReverseScanner_smaller_blocksize() throws IOException { 6142 // case to ensure no conflict with HFile index optimization 6143 byte[] rowA = Bytes.toBytes("rowA"); 6144 byte[] rowB = Bytes.toBytes("rowB"); 6145 byte[] rowC = Bytes.toBytes("rowC"); 6146 byte[] rowD = Bytes.toBytes("rowD"); 6147 byte[] rowE = Bytes.toBytes("rowE"); 6148 byte[] cf = Bytes.toBytes("CF"); 6149 byte[][] families = { cf }; 6150 byte[] col1 = Bytes.toBytes("col1"); 6151 byte[] col2 = Bytes.toBytes("col2"); 6152 long ts = 1; 6153 Configuration conf = new Configuration(CONF); 6154 conf.setInt("test.block.size", 1); 6155 this.region = initHRegion(tableName, method, conf, families); 6156 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null); 6157 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null); 6158 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null); 6159 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null); 6160 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null); 6161 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null); 6162 Put put = null; 6163 put = new Put(rowA); 6164 put.add(kv1); 6165 region.put(put); 6166 put = new Put(rowB); 6167 put.add(kv2); 6168 region.put(put); 6169 put = new Put(rowC); 6170 put.add(kv3); 6171 region.put(put); 6172 put = new Put(rowD); 6173 put.add(kv4_1); 6174 region.put(put); 6175 put = new Put(rowD); 6176 put.add(kv4_2); 6177 region.put(put); 6178 put = new Put(rowE); 6179 put.add(kv5); 6180 region.put(put); 6181 region.flush(true); 6182 Scan scan = new Scan().withStartRow(rowD).withStopRow(rowA); 6183 scan.addColumn(families[0], col1); 6184 scan.setReversed(true); 6185 List<Cell> currRow = new ArrayList<>(); 6186 try (InternalScanner scanner = region.getScanner(scan)) { 6187 boolean hasNext = scanner.next(currRow); 6188 assertEquals(1, currRow.size()); 6189 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6190 currRow.get(0).getRowLength(), rowD, 0, rowD.length)); 6191 assertTrue(hasNext); 6192 currRow.clear(); 6193 hasNext = scanner.next(currRow); 6194 assertEquals(1, currRow.size()); 6195 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6196 currRow.get(0).getRowLength(), rowC, 0, rowC.length)); 6197 assertTrue(hasNext); 6198 currRow.clear(); 6199 hasNext = scanner.next(currRow); 6200 assertEquals(1, currRow.size()); 6201 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6202 currRow.get(0).getRowLength(), rowB, 0, rowB.length)); 6203 assertFalse(hasNext); 6204 } 6205 6206 scan = new Scan().withStartRow(rowD).withStopRow(rowA); 6207 scan.addColumn(families[0], col2); 6208 scan.setReversed(true); 6209 currRow.clear(); 6210 try (InternalScanner scanner = region.getScanner(scan)) { 6211 boolean hasNext = scanner.next(currRow); 6212 assertEquals(1, currRow.size()); 6213 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6214 currRow.get(0).getRowLength(), rowD, 0, rowD.length)); 6215 assertTrue(hasNext); 6216 } 6217 } 6218 6219 @Test 6220 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1() throws IOException { 6221 byte[] row0 = Bytes.toBytes("row0"); // 1 kv 6222 byte[] row1 = Bytes.toBytes("row1"); // 2 kv 6223 byte[] row2 = Bytes.toBytes("row2"); // 4 kv 6224 byte[] row3 = Bytes.toBytes("row3"); // 2 kv 6225 byte[] row4 = Bytes.toBytes("row4"); // 5 kv 6226 byte[] row5 = Bytes.toBytes("row5"); // 2 kv 6227 byte[] cf1 = Bytes.toBytes("CF1"); 6228 byte[] cf2 = Bytes.toBytes("CF2"); 6229 byte[] cf3 = Bytes.toBytes("CF3"); 6230 byte[][] families = { cf1, cf2, cf3 }; 6231 byte[] col = Bytes.toBytes("C"); 6232 long ts = 1; 6233 Configuration conf = new Configuration(CONF); 6234 // disable compactions in this test. 6235 conf.setInt("hbase.hstore.compactionThreshold", 10000); 6236 this.region = initHRegion(tableName, method, conf, families); 6237 // kv naming style: kv(row number) totalKvCountInThisRow seq no 6238 KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put, null); 6239 KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put, null); 6240 KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1, KeyValue.Type.Put, null); 6241 KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null); 6242 KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put, null); 6243 KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put, null); 6244 KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4, KeyValue.Type.Put, null); 6245 KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put, null); 6246 KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4, KeyValue.Type.Put, null); 6247 KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put, null); 6248 KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put, null); 6249 KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5, KeyValue.Type.Put, null); 6250 KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put, null); 6251 KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3, KeyValue.Type.Put, null); 6252 KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put, null); 6253 KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put, null); 6254 // hfiles(cf1/cf2) :"row1"(1 kv) / "row2"(1 kv) / "row4"(2 kv) 6255 Put put = null; 6256 put = new Put(row1); 6257 put.add(kv1_2_1); 6258 region.put(put); 6259 put = new Put(row2); 6260 put.add(kv2_4_1); 6261 region.put(put); 6262 put = new Put(row4); 6263 put.add(kv4_5_4); 6264 put.add(kv4_5_5); 6265 region.put(put); 6266 region.flush(true); 6267 // hfiles(cf1/cf3) : "row1" (1 kvs) / "row2" (1 kv) / "row4" (2 kv) 6268 put = new Put(row4); 6269 put.add(kv4_5_1); 6270 put.add(kv4_5_3); 6271 region.put(put); 6272 put = new Put(row1); 6273 put.add(kv1_2_2); 6274 region.put(put); 6275 put = new Put(row2); 6276 put.add(kv2_4_4); 6277 region.put(put); 6278 region.flush(true); 6279 // hfiles(cf1/cf3) : "row2"(2 kv) / "row3"(1 kvs) / "row4" (1 kv) 6280 put = new Put(row4); 6281 put.add(kv4_5_2); 6282 region.put(put); 6283 put = new Put(row2); 6284 put.add(kv2_4_2); 6285 put.add(kv2_4_3); 6286 region.put(put); 6287 put = new Put(row3); 6288 put.add(kv3_2_2); 6289 region.put(put); 6290 region.flush(true); 6291 // memstore(cf1/cf2/cf3) : "row0" (1 kvs) / "row3" ( 1 kv) / "row5" (max) 6292 // ( 2 kv) 6293 put = new Put(row0); 6294 put.add(kv0_1_1); 6295 region.put(put); 6296 put = new Put(row3); 6297 put.add(kv3_2_1); 6298 region.put(put); 6299 put = new Put(row5); 6300 put.add(kv5_2_1); 6301 put.add(kv5_2_2); 6302 region.put(put); 6303 // scan range = ["row4", min), skip the max "row5" 6304 Scan scan = new Scan().withStartRow(row4); 6305 scan.readVersions(5); 6306 scan.setBatch(3); 6307 scan.setReversed(true); 6308 try (InternalScanner scanner = region.getScanner(scan)) { 6309 List<Cell> currRow = new ArrayList<>(); 6310 boolean hasNext = false; 6311 // 1. scan out "row4" (5 kvs), "row5" can't be scanned out since not 6312 // included in scan range 6313 // "row4" takes 2 next() calls since batch=3 6314 hasNext = scanner.next(currRow); 6315 assertEquals(3, currRow.size()); 6316 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6317 currRow.get(0).getRowLength(), row4, 0, row4.length)); 6318 assertTrue(hasNext); 6319 currRow.clear(); 6320 hasNext = scanner.next(currRow); 6321 assertEquals(2, currRow.size()); 6322 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6323 currRow.get(0).getRowLength(), row4, 0, row4.length)); 6324 assertTrue(hasNext); 6325 // 2. scan out "row3" (2 kv) 6326 currRow.clear(); 6327 hasNext = scanner.next(currRow); 6328 assertEquals(2, currRow.size()); 6329 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6330 currRow.get(0).getRowLength(), row3, 0, row3.length)); 6331 assertTrue(hasNext); 6332 // 3. scan out "row2" (4 kvs) 6333 // "row2" takes 2 next() calls since batch=3 6334 currRow.clear(); 6335 hasNext = scanner.next(currRow); 6336 assertEquals(3, currRow.size()); 6337 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6338 currRow.get(0).getRowLength(), row2, 0, row2.length)); 6339 assertTrue(hasNext); 6340 currRow.clear(); 6341 hasNext = scanner.next(currRow); 6342 assertEquals(1, currRow.size()); 6343 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6344 currRow.get(0).getRowLength(), row2, 0, row2.length)); 6345 assertTrue(hasNext); 6346 // 4. scan out "row1" (2 kv) 6347 currRow.clear(); 6348 hasNext = scanner.next(currRow); 6349 assertEquals(2, currRow.size()); 6350 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6351 currRow.get(0).getRowLength(), row1, 0, row1.length)); 6352 assertTrue(hasNext); 6353 // 5. scan out "row0" (1 kv) 6354 currRow.clear(); 6355 hasNext = scanner.next(currRow); 6356 assertEquals(1, currRow.size()); 6357 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6358 currRow.get(0).getRowLength(), row0, 0, row0.length)); 6359 assertFalse(hasNext); 6360 } 6361 } 6362 6363 @Test 6364 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2() throws IOException { 6365 byte[] row1 = Bytes.toBytes("row1"); 6366 byte[] row2 = Bytes.toBytes("row2"); 6367 byte[] row3 = Bytes.toBytes("row3"); 6368 byte[] row4 = Bytes.toBytes("row4"); 6369 byte[] cf1 = Bytes.toBytes("CF1"); 6370 byte[] cf2 = Bytes.toBytes("CF2"); 6371 byte[] cf3 = Bytes.toBytes("CF3"); 6372 byte[] cf4 = Bytes.toBytes("CF4"); 6373 byte[][] families = { cf1, cf2, cf3, cf4 }; 6374 byte[] col = Bytes.toBytes("C"); 6375 long ts = 1; 6376 Configuration conf = new Configuration(CONF); 6377 // disable compactions in this test. 6378 conf.setInt("hbase.hstore.compactionThreshold", 10000); 6379 this.region = initHRegion(tableName, method, conf, families); 6380 KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null); 6381 KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null); 6382 KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null); 6383 KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null); 6384 // storefile1 6385 Put put = new Put(row1); 6386 put.add(kv1); 6387 region.put(put); 6388 region.flush(true); 6389 // storefile2 6390 put = new Put(row2); 6391 put.add(kv2); 6392 region.put(put); 6393 region.flush(true); 6394 // storefile3 6395 put = new Put(row3); 6396 put.add(kv3); 6397 region.put(put); 6398 region.flush(true); 6399 // memstore 6400 put = new Put(row4); 6401 put.add(kv4); 6402 region.put(put); 6403 // scan range = ["row4", min) 6404 Scan scan = new Scan().withStartRow(row4); 6405 scan.setReversed(true); 6406 scan.setBatch(10); 6407 try (InternalScanner scanner = region.getScanner(scan)) { 6408 List<Cell> currRow = new ArrayList<>(); 6409 boolean hasNext = scanner.next(currRow); 6410 assertEquals(1, currRow.size()); 6411 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6412 currRow.get(0).getRowLength(), row4, 0, row4.length)); 6413 assertTrue(hasNext); 6414 currRow.clear(); 6415 hasNext = scanner.next(currRow); 6416 assertEquals(1, currRow.size()); 6417 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6418 currRow.get(0).getRowLength(), row3, 0, row3.length)); 6419 assertTrue(hasNext); 6420 currRow.clear(); 6421 hasNext = scanner.next(currRow); 6422 assertEquals(1, currRow.size()); 6423 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6424 currRow.get(0).getRowLength(), row2, 0, row2.length)); 6425 assertTrue(hasNext); 6426 currRow.clear(); 6427 hasNext = scanner.next(currRow); 6428 assertEquals(1, currRow.size()); 6429 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6430 currRow.get(0).getRowLength(), row1, 0, row1.length)); 6431 assertFalse(hasNext); 6432 } 6433 } 6434 6435 /** 6436 * Test for HBASE-14497: Reverse Scan threw StackOverflow caused by readPt checking 6437 */ 6438 @Test 6439 public void testReverseScanner_StackOverflow() throws IOException { 6440 byte[] cf1 = Bytes.toBytes("CF1"); 6441 byte[][] families = { cf1 }; 6442 byte[] col = Bytes.toBytes("C"); 6443 Configuration conf = new Configuration(CONF); 6444 this.region = initHRegion(tableName, method, conf, families); 6445 // setup with one storefile and one memstore, to create scanner and get an earlier readPt 6446 Put put = new Put(Bytes.toBytes("19998")); 6447 put.addColumn(cf1, col, Bytes.toBytes("val")); 6448 region.put(put); 6449 region.flushcache(true, true, FlushLifeCycleTracker.DUMMY); 6450 Put put2 = new Put(Bytes.toBytes("19997")); 6451 put2.addColumn(cf1, col, Bytes.toBytes("val")); 6452 region.put(put2); 6453 6454 Scan scan = new Scan().withStartRow(Bytes.toBytes("19998")); 6455 scan.setReversed(true); 6456 try (InternalScanner scanner = region.getScanner(scan)) { 6457 // create one storefile contains many rows will be skipped 6458 // to check StoreFileScanner.seekToPreviousRow 6459 for (int i = 10000; i < 20000; i++) { 6460 Put p = new Put(Bytes.toBytes("" + i)); 6461 p.addColumn(cf1, col, Bytes.toBytes("" + i)); 6462 region.put(p); 6463 } 6464 region.flushcache(true, true, FlushLifeCycleTracker.DUMMY); 6465 6466 // create one memstore contains many rows will be skipped 6467 // to check MemStoreScanner.seekToPreviousRow 6468 for (int i = 10000; i < 20000; i++) { 6469 Put p = new Put(Bytes.toBytes("" + i)); 6470 p.addColumn(cf1, col, Bytes.toBytes("" + i)); 6471 region.put(p); 6472 } 6473 6474 List<Cell> currRow = new ArrayList<>(); 6475 boolean hasNext; 6476 do { 6477 hasNext = scanner.next(currRow); 6478 } while (hasNext); 6479 assertEquals(2, currRow.size()); 6480 assertEquals("19998", Bytes.toString(currRow.get(0).getRowArray(), 6481 currRow.get(0).getRowOffset(), currRow.get(0).getRowLength())); 6482 assertEquals("19997", Bytes.toString(currRow.get(1).getRowArray(), 6483 currRow.get(1).getRowOffset(), currRow.get(1).getRowLength())); 6484 } 6485 } 6486 6487 @Test 6488 public void testReverseScanShouldNotScanMemstoreIfReadPtLesser() throws Exception { 6489 byte[] cf1 = Bytes.toBytes("CF1"); 6490 byte[][] families = { cf1 }; 6491 byte[] col = Bytes.toBytes("C"); 6492 this.region = initHRegion(tableName, method, CONF, families); 6493 // setup with one storefile and one memstore, to create scanner and get an earlier readPt 6494 Put put = new Put(Bytes.toBytes("19996")); 6495 put.addColumn(cf1, col, Bytes.toBytes("val")); 6496 region.put(put); 6497 Put put2 = new Put(Bytes.toBytes("19995")); 6498 put2.addColumn(cf1, col, Bytes.toBytes("val")); 6499 region.put(put2); 6500 // create a reverse scan 6501 Scan scan = new Scan().withStartRow(Bytes.toBytes("19996")); 6502 scan.setReversed(true); 6503 try (RegionScannerImpl scanner = region.getScanner(scan)) { 6504 // flush the cache. This will reset the store scanner 6505 region.flushcache(true, true, FlushLifeCycleTracker.DUMMY); 6506 6507 // create one memstore contains many rows will be skipped 6508 // to check MemStoreScanner.seekToPreviousRow 6509 for (int i = 10000; i < 20000; i++) { 6510 Put p = new Put(Bytes.toBytes("" + i)); 6511 p.addColumn(cf1, col, Bytes.toBytes("" + i)); 6512 region.put(p); 6513 } 6514 List<Cell> currRow = new ArrayList<>(); 6515 boolean hasNext; 6516 boolean assertDone = false; 6517 do { 6518 hasNext = scanner.next(currRow); 6519 // With HBASE-15871, after the scanner is reset the memstore scanner should not be 6520 // added here 6521 if (!assertDone) { 6522 StoreScanner current = (StoreScanner) (scanner.storeHeap).getCurrentForTesting(); 6523 List<KeyValueScanner> scanners = current.getAllScannersForTesting(); 6524 assertEquals("There should be only one scanner the store file scanner", 1, 6525 scanners.size()); 6526 assertDone = true; 6527 } 6528 } while (hasNext); 6529 assertEquals(2, currRow.size()); 6530 assertEquals("19996", Bytes.toString(currRow.get(0).getRowArray(), 6531 currRow.get(0).getRowOffset(), currRow.get(0).getRowLength())); 6532 assertEquals("19995", Bytes.toString(currRow.get(1).getRowArray(), 6533 currRow.get(1).getRowOffset(), currRow.get(1).getRowLength())); 6534 } 6535 } 6536 6537 @Test 6538 public void testReverseScanWhenPutCellsAfterOpenReverseScan() throws Exception { 6539 byte[] cf1 = Bytes.toBytes("CF1"); 6540 byte[][] families = { cf1 }; 6541 byte[] col = Bytes.toBytes("C"); 6542 6543 this.region = initHRegion(tableName, method, CONF, families); 6544 6545 Put put = new Put(Bytes.toBytes("199996")); 6546 put.addColumn(cf1, col, Bytes.toBytes("val")); 6547 region.put(put); 6548 Put put2 = new Put(Bytes.toBytes("199995")); 6549 put2.addColumn(cf1, col, Bytes.toBytes("val")); 6550 region.put(put2); 6551 6552 // Create a reverse scan 6553 Scan scan = new Scan().withStartRow(Bytes.toBytes("199996")); 6554 scan.setReversed(true); 6555 try (RegionScannerImpl scanner = region.getScanner(scan)) { 6556 // Put a lot of cells that have sequenceIDs grater than the readPt of the reverse scan 6557 for (int i = 100000; i < 200000; i++) { 6558 Put p = new Put(Bytes.toBytes("" + i)); 6559 p.addColumn(cf1, col, Bytes.toBytes("" + i)); 6560 region.put(p); 6561 } 6562 List<Cell> currRow = new ArrayList<>(); 6563 boolean hasNext; 6564 do { 6565 hasNext = scanner.next(currRow); 6566 } while (hasNext); 6567 6568 assertEquals(2, currRow.size()); 6569 assertEquals("199996", Bytes.toString(currRow.get(0).getRowArray(), 6570 currRow.get(0).getRowOffset(), currRow.get(0).getRowLength())); 6571 assertEquals("199995", Bytes.toString(currRow.get(1).getRowArray(), 6572 currRow.get(1).getRowOffset(), currRow.get(1).getRowLength())); 6573 } 6574 } 6575 6576 @Test 6577 public void testWriteRequestsCounter() throws IOException { 6578 byte[] fam = Bytes.toBytes("info"); 6579 byte[][] families = { fam }; 6580 this.region = initHRegion(tableName, method, CONF, families); 6581 6582 Assert.assertEquals(0L, region.getWriteRequestsCount()); 6583 6584 Put put = new Put(row); 6585 put.addColumn(fam, fam, fam); 6586 6587 Assert.assertEquals(0L, region.getWriteRequestsCount()); 6588 region.put(put); 6589 Assert.assertEquals(1L, region.getWriteRequestsCount()); 6590 region.put(put); 6591 Assert.assertEquals(2L, region.getWriteRequestsCount()); 6592 region.put(put); 6593 Assert.assertEquals(3L, region.getWriteRequestsCount()); 6594 6595 region.delete(new Delete(row)); 6596 Assert.assertEquals(4L, region.getWriteRequestsCount()); 6597 } 6598 6599 @Test 6600 public void testOpenRegionWrittenToWAL() throws Exception { 6601 final ServerName serverName = ServerName.valueOf(name.getMethodName(), 100, 42); 6602 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName)); 6603 6604 TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) 6605 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)) 6606 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam2)).build(); 6607 RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build(); 6608 6609 // open the region w/o rss and wal and flush some files 6610 region = HBaseTestingUtil.createRegionAndWAL(hri, TEST_UTIL.getDataTestDir(), 6611 TEST_UTIL.getConfiguration(), htd); 6612 assertNotNull(region); 6613 6614 // create a file in fam1 for the region before opening in OpenRegionHandler 6615 region.put(new Put(Bytes.toBytes("a")).addColumn(fam1, fam1, fam1)); 6616 region.flush(true); 6617 HBaseTestingUtil.closeRegionAndWAL(region); 6618 6619 ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class); 6620 6621 // capture append() calls 6622 WAL wal = mockWAL(); 6623 when(rss.getWAL(any(RegionInfo.class))).thenReturn(wal); 6624 6625 region = 6626 HRegion.openHRegion(hri, htd, rss.getWAL(hri), TEST_UTIL.getConfiguration(), rss, null); 6627 6628 verify(wal, times(1)).appendMarker(any(RegionInfo.class), any(WALKeyImpl.class), 6629 editCaptor.capture()); 6630 6631 WALEdit edit = editCaptor.getValue(); 6632 assertNotNull(edit); 6633 assertNotNull(edit.getCells()); 6634 assertEquals(1, edit.getCells().size()); 6635 RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0)); 6636 assertNotNull(desc); 6637 6638 LOG.info("RegionEventDescriptor from WAL: " + desc); 6639 6640 assertEquals(RegionEventDescriptor.EventType.REGION_OPEN, desc.getEventType()); 6641 assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getTableName().toBytes())); 6642 assertTrue( 6643 Bytes.equals(desc.getEncodedRegionName().toByteArray(), hri.getEncodedNameAsBytes())); 6644 assertTrue(desc.getLogSequenceNumber() > 0); 6645 assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer())); 6646 assertEquals(2, desc.getStoresCount()); 6647 6648 StoreDescriptor store = desc.getStores(0); 6649 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1)); 6650 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1)); 6651 assertEquals(1, store.getStoreFileCount()); // 1store file 6652 assertFalse(store.getStoreFile(0).contains("/")); // ensure path is relative 6653 6654 store = desc.getStores(1); 6655 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2)); 6656 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2)); 6657 assertEquals(0, store.getStoreFileCount()); // no store files 6658 } 6659 6660 // Helper for test testOpenRegionWrittenToWALForLogReplay 6661 static class HRegionWithSeqId extends HRegion { 6662 public HRegionWithSeqId(final Path tableDir, final WAL wal, final FileSystem fs, 6663 final Configuration confParam, final RegionInfo regionInfo, final TableDescriptor htd, 6664 final RegionServerServices rsServices) { 6665 super(tableDir, wal, fs, confParam, regionInfo, htd, rsServices); 6666 } 6667 6668 @Override 6669 protected long getNextSequenceId(WAL wal) throws IOException { 6670 return 42; 6671 } 6672 } 6673 6674 @Test 6675 public void testFlushedFileWithNoTags() throws Exception { 6676 final TableName tableName = TableName.valueOf(name.getMethodName()); 6677 TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName) 6678 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)).build(); 6679 RegionInfo info = RegionInfoBuilder.newBuilder(tableName).build(); 6680 Path path = TEST_UTIL.getDataTestDir(getClass().getSimpleName()); 6681 region = HBaseTestingUtil.createRegionAndWAL(info, path, TEST_UTIL.getConfiguration(), 6682 tableDescriptor); 6683 Put put = new Put(Bytes.toBytes("a-b-0-0")); 6684 put.addColumn(fam1, qual1, Bytes.toBytes("c1-value")); 6685 region.put(put); 6686 region.flush(true); 6687 HStore store = region.getStore(fam1); 6688 Collection<HStoreFile> storefiles = store.getStorefiles(); 6689 for (HStoreFile sf : storefiles) { 6690 assertFalse("Tags should not be present ", 6691 sf.getReader().getHFileReader().getFileContext().isIncludesTags()); 6692 } 6693 } 6694 6695 /** 6696 * Utility method to setup a WAL mock. 6697 * <p/> 6698 * Needs to do the bit where we close latch on the WALKeyImpl on append else test hangs. 6699 * @return a mock WAL 6700 */ 6701 private WAL mockWAL() throws IOException { 6702 WAL wal = mock(WAL.class); 6703 when(wal.appendData(any(RegionInfo.class), any(WALKeyImpl.class), any(WALEdit.class))) 6704 .thenAnswer(new Answer<Long>() { 6705 @Override 6706 public Long answer(InvocationOnMock invocation) throws Throwable { 6707 WALKeyImpl key = invocation.getArgument(1); 6708 MultiVersionConcurrencyControl.WriteEntry we = key.getMvcc().begin(); 6709 key.setWriteEntry(we); 6710 return 1L; 6711 } 6712 }); 6713 when(wal.appendMarker(any(RegionInfo.class), any(WALKeyImpl.class), any(WALEdit.class))) 6714 .thenAnswer(new Answer<Long>() { 6715 @Override 6716 public Long answer(InvocationOnMock invocation) throws Throwable { 6717 WALKeyImpl key = invocation.getArgument(1); 6718 MultiVersionConcurrencyControl.WriteEntry we = key.getMvcc().begin(); 6719 key.setWriteEntry(we); 6720 return 1L; 6721 } 6722 }); 6723 return wal; 6724 } 6725 6726 @Test 6727 public void testCloseRegionWrittenToWAL() throws Exception { 6728 Path rootDir = new Path(dir + name.getMethodName()); 6729 CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir); 6730 6731 final ServerName serverName = ServerName.valueOf("testCloseRegionWrittenToWAL", 100, 42); 6732 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName)); 6733 6734 TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) 6735 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)) 6736 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam2)).build(); 6737 RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build(); 6738 6739 ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class); 6740 6741 // capture append() calls 6742 WAL wal = mockWAL(); 6743 when(rss.getWAL(any(RegionInfo.class))).thenReturn(wal); 6744 6745 // create the region 6746 region = HBaseTestingUtil.createRegionAndWAL(hri, rootDir, CONF, htd); 6747 HBaseTestingUtil.closeRegionAndWAL(region); 6748 region = null; 6749 // open the region first and then close it 6750 HRegion.openHRegion(hri, htd, rss.getWAL(hri), TEST_UTIL.getConfiguration(), rss, null).close(); 6751 6752 // 2 times, one for region open, the other close region 6753 verify(wal, times(2)).appendMarker(any(RegionInfo.class), (WALKeyImpl) any(WALKeyImpl.class), 6754 editCaptor.capture()); 6755 6756 WALEdit edit = editCaptor.getAllValues().get(1); 6757 assertNotNull(edit); 6758 assertNotNull(edit.getCells()); 6759 assertEquals(1, edit.getCells().size()); 6760 RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0)); 6761 assertNotNull(desc); 6762 6763 LOG.info("RegionEventDescriptor from WAL: " + desc); 6764 6765 assertEquals(RegionEventDescriptor.EventType.REGION_CLOSE, desc.getEventType()); 6766 assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getTableName().toBytes())); 6767 assertTrue( 6768 Bytes.equals(desc.getEncodedRegionName().toByteArray(), hri.getEncodedNameAsBytes())); 6769 assertTrue(desc.getLogSequenceNumber() > 0); 6770 assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer())); 6771 assertEquals(2, desc.getStoresCount()); 6772 6773 StoreDescriptor store = desc.getStores(0); 6774 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1)); 6775 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1)); 6776 assertEquals(0, store.getStoreFileCount()); // no store files 6777 6778 store = desc.getStores(1); 6779 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2)); 6780 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2)); 6781 assertEquals(0, store.getStoreFileCount()); // no store files 6782 } 6783 6784 /** 6785 * Test RegionTooBusyException thrown when region is busy 6786 */ 6787 @Test 6788 public void testRegionTooBusy() throws IOException { 6789 byte[] family = Bytes.toBytes("family"); 6790 long defaultBusyWaitDuration = 6791 CONF.getLong("hbase.busy.wait.duration", HRegion.DEFAULT_BUSY_WAIT_DURATION); 6792 CONF.setLong("hbase.busy.wait.duration", 1000); 6793 region = initHRegion(tableName, method, CONF, family); 6794 final AtomicBoolean stopped = new AtomicBoolean(true); 6795 Thread t = new Thread(new Runnable() { 6796 @Override 6797 public void run() { 6798 try { 6799 region.lock.writeLock().lock(); 6800 stopped.set(false); 6801 while (!stopped.get()) { 6802 Thread.sleep(100); 6803 } 6804 } catch (InterruptedException ie) { 6805 } finally { 6806 region.lock.writeLock().unlock(); 6807 } 6808 } 6809 }); 6810 t.start(); 6811 Get get = new Get(row); 6812 try { 6813 while (stopped.get()) { 6814 Thread.sleep(100); 6815 } 6816 region.get(get); 6817 fail("Should throw RegionTooBusyException"); 6818 } catch (InterruptedException ie) { 6819 fail("test interrupted"); 6820 } catch (RegionTooBusyException e) { 6821 // Good, expected 6822 } finally { 6823 stopped.set(true); 6824 try { 6825 t.join(); 6826 } catch (Throwable e) { 6827 } 6828 6829 HBaseTestingUtil.closeRegionAndWAL(region); 6830 region = null; 6831 CONF.setLong("hbase.busy.wait.duration", defaultBusyWaitDuration); 6832 } 6833 } 6834 6835 @Test 6836 public void testCellTTLs() throws IOException { 6837 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge(); 6838 EnvironmentEdgeManager.injectEdge(edge); 6839 6840 final byte[] row = Bytes.toBytes("testRow"); 6841 final byte[] q1 = Bytes.toBytes("q1"); 6842 final byte[] q2 = Bytes.toBytes("q2"); 6843 final byte[] q3 = Bytes.toBytes("q3"); 6844 final byte[] q4 = Bytes.toBytes("q4"); 6845 6846 // 10 seconds 6847 TableDescriptor tableDescriptor = 6848 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) 6849 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam1).setTimeToLive(10).build()) 6850 .build(); 6851 6852 Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); 6853 conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS); 6854 6855 region = HBaseTestingUtil.createRegionAndWAL( 6856 RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(), 6857 TEST_UTIL.getDataTestDir(), conf, tableDescriptor); 6858 assertNotNull(region); 6859 long now = EnvironmentEdgeManager.currentTime(); 6860 // Add a cell that will expire in 5 seconds via cell TTL 6861 region.put(new Put(row).add(new KeyValue(row, fam1, q1, now, HConstants.EMPTY_BYTE_ARRAY, 6862 new ArrayBackedTag[] { 6863 // TTL tags specify ts in milliseconds 6864 new ArrayBackedTag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) }))); 6865 // Add a cell that will expire after 10 seconds via family setting 6866 region.put(new Put(row).addColumn(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY)); 6867 // Add a cell that will expire in 15 seconds via cell TTL 6868 region.put(new Put(row).add(new KeyValue(row, fam1, q3, now + 10000 - 1, 6869 HConstants.EMPTY_BYTE_ARRAY, new ArrayBackedTag[] { 6870 // TTL tags specify ts in milliseconds 6871 new ArrayBackedTag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) }))); 6872 // Add a cell that will expire in 20 seconds via family setting 6873 region.put(new Put(row).addColumn(fam1, q4, now + 10000 - 1, HConstants.EMPTY_BYTE_ARRAY)); 6874 6875 // Flush so we are sure store scanning gets this right 6876 region.flush(true); 6877 6878 // A query at time T+0 should return all cells 6879 Result r = region.get(new Get(row)); 6880 assertNotNull(r.getValue(fam1, q1)); 6881 assertNotNull(r.getValue(fam1, q2)); 6882 assertNotNull(r.getValue(fam1, q3)); 6883 assertNotNull(r.getValue(fam1, q4)); 6884 6885 // Increment time to T+5 seconds 6886 edge.incrementTime(5000); 6887 6888 r = region.get(new Get(row)); 6889 assertNull(r.getValue(fam1, q1)); 6890 assertNotNull(r.getValue(fam1, q2)); 6891 assertNotNull(r.getValue(fam1, q3)); 6892 assertNotNull(r.getValue(fam1, q4)); 6893 6894 // Increment time to T+10 seconds 6895 edge.incrementTime(5000); 6896 6897 r = region.get(new Get(row)); 6898 assertNull(r.getValue(fam1, q1)); 6899 assertNull(r.getValue(fam1, q2)); 6900 assertNotNull(r.getValue(fam1, q3)); 6901 assertNotNull(r.getValue(fam1, q4)); 6902 6903 // Increment time to T+15 seconds 6904 edge.incrementTime(5000); 6905 6906 r = region.get(new Get(row)); 6907 assertNull(r.getValue(fam1, q1)); 6908 assertNull(r.getValue(fam1, q2)); 6909 assertNull(r.getValue(fam1, q3)); 6910 assertNotNull(r.getValue(fam1, q4)); 6911 6912 // Increment time to T+20 seconds 6913 edge.incrementTime(10000); 6914 6915 r = region.get(new Get(row)); 6916 assertNull(r.getValue(fam1, q1)); 6917 assertNull(r.getValue(fam1, q2)); 6918 assertNull(r.getValue(fam1, q3)); 6919 assertNull(r.getValue(fam1, q4)); 6920 6921 // Fun with disappearing increments 6922 6923 // Start at 1 6924 region.put(new Put(row).addColumn(fam1, q1, Bytes.toBytes(1L))); 6925 r = region.get(new Get(row)); 6926 byte[] val = r.getValue(fam1, q1); 6927 assertNotNull(val); 6928 assertEquals(1L, Bytes.toLong(val)); 6929 6930 // Increment with a TTL of 5 seconds 6931 Increment incr = new Increment(row).addColumn(fam1, q1, 1L); 6932 incr.setTTL(5000); 6933 region.increment(incr); // 2 6934 6935 // New value should be 2 6936 r = region.get(new Get(row)); 6937 val = r.getValue(fam1, q1); 6938 assertNotNull(val); 6939 assertEquals(2L, Bytes.toLong(val)); 6940 6941 // Increment time to T+25 seconds 6942 edge.incrementTime(5000); 6943 6944 // Value should be back to 1 6945 r = region.get(new Get(row)); 6946 val = r.getValue(fam1, q1); 6947 assertNotNull(val); 6948 assertEquals(1L, Bytes.toLong(val)); 6949 6950 // Increment time to T+30 seconds 6951 edge.incrementTime(5000); 6952 6953 // Original value written at T+20 should be gone now via family TTL 6954 r = region.get(new Get(row)); 6955 assertNull(r.getValue(fam1, q1)); 6956 } 6957 6958 @Test 6959 public void testTTLsUsingSmallHeartBeatCells() throws IOException { 6960 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge(); 6961 EnvironmentEdgeManager.injectEdge(edge); 6962 6963 final byte[] row = Bytes.toBytes("testRow"); 6964 final byte[] q1 = Bytes.toBytes("q1"); 6965 final byte[] q2 = Bytes.toBytes("q2"); 6966 final byte[] q3 = Bytes.toBytes("q3"); 6967 final byte[] q4 = Bytes.toBytes("q4"); 6968 final byte[] q5 = Bytes.toBytes("q5"); 6969 final byte[] q6 = Bytes.toBytes("q6"); 6970 final byte[] q7 = Bytes.toBytes("q7"); 6971 final byte[] q8 = Bytes.toBytes("q8"); 6972 6973 // 10 seconds 6974 int ttlSecs = 10; 6975 TableDescriptor tableDescriptor = 6976 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily( 6977 ColumnFamilyDescriptorBuilder.newBuilder(fam1).setTimeToLive(ttlSecs).build()).build(); 6978 6979 Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); 6980 conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS); 6981 // using small heart beat cells 6982 conf.setLong(StoreScanner.HBASE_CELLS_SCANNED_PER_HEARTBEAT_CHECK, 2); 6983 6984 region = HBaseTestingUtil.createRegionAndWAL( 6985 RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(), 6986 TEST_UTIL.getDataTestDir(), conf, tableDescriptor); 6987 assertNotNull(region); 6988 long now = EnvironmentEdgeManager.currentTime(); 6989 // Add a cell that will expire in 5 seconds via cell TTL 6990 region.put(new Put(row).addColumn(fam1, q1, now, HConstants.EMPTY_BYTE_ARRAY)); 6991 region.put(new Put(row).addColumn(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY)); 6992 region.put(new Put(row).addColumn(fam1, q3, now, HConstants.EMPTY_BYTE_ARRAY)); 6993 // Add a cell that will expire after 10 seconds via family setting 6994 region 6995 .put(new Put(row).addColumn(fam1, q4, now + ttlSecs * 1000 + 1, HConstants.EMPTY_BYTE_ARRAY)); 6996 region 6997 .put(new Put(row).addColumn(fam1, q5, now + ttlSecs * 1000 + 1, HConstants.EMPTY_BYTE_ARRAY)); 6998 6999 region.put(new Put(row).addColumn(fam1, q6, now, HConstants.EMPTY_BYTE_ARRAY)); 7000 region.put(new Put(row).addColumn(fam1, q7, now, HConstants.EMPTY_BYTE_ARRAY)); 7001 region 7002 .put(new Put(row).addColumn(fam1, q8, now + ttlSecs * 1000 + 1, HConstants.EMPTY_BYTE_ARRAY)); 7003 7004 // Flush so we are sure store scanning gets this right 7005 region.flush(true); 7006 7007 // A query at time T+0 should return all cells 7008 checkScan(8); 7009 region.delete(new Delete(row).addColumn(fam1, q8)); 7010 7011 // Increment time to T+ttlSecs seconds 7012 edge.incrementTime(ttlSecs * 1000); 7013 checkScan(2); 7014 } 7015 7016 private void checkScan(int expectCellSize) throws IOException { 7017 Scan s = new Scan().withStartRow(row); 7018 ScannerContext.Builder contextBuilder = ScannerContext.newBuilder(true); 7019 ScannerContext scannerContext = contextBuilder.build(); 7020 try (RegionScanner scanner = region.getScanner(s)) { 7021 List<Cell> kvs = new ArrayList<>(); 7022 scanner.next(kvs, scannerContext); 7023 assertEquals(expectCellSize, kvs.size()); 7024 } 7025 } 7026 7027 @Test 7028 public void testIncrementTimestampsAreMonotonic() throws IOException { 7029 region = initHRegion(tableName, method, CONF, fam1); 7030 ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); 7031 EnvironmentEdgeManager.injectEdge(edge); 7032 7033 edge.setValue(10); 7034 Increment inc = new Increment(row); 7035 inc.setDurability(Durability.SKIP_WAL); 7036 inc.addColumn(fam1, qual1, 1L); 7037 region.increment(inc); 7038 7039 Result result = region.get(new Get(row)); 7040 Cell c = result.getColumnLatestCell(fam1, qual1); 7041 assertNotNull(c); 7042 assertEquals(10L, c.getTimestamp()); 7043 7044 edge.setValue(1); // clock goes back 7045 region.increment(inc); 7046 result = region.get(new Get(row)); 7047 c = result.getColumnLatestCell(fam1, qual1); 7048 assertEquals(11L, c.getTimestamp()); 7049 assertEquals(2L, Bytes.toLong(c.getValueArray(), c.getValueOffset(), c.getValueLength())); 7050 } 7051 7052 @Test 7053 public void testAppendTimestampsAreMonotonic() throws IOException { 7054 region = initHRegion(tableName, method, CONF, fam1); 7055 ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); 7056 EnvironmentEdgeManager.injectEdge(edge); 7057 7058 edge.setValue(10); 7059 Append a = new Append(row); 7060 a.setDurability(Durability.SKIP_WAL); 7061 a.addColumn(fam1, qual1, qual1); 7062 region.append(a); 7063 7064 Result result = region.get(new Get(row)); 7065 Cell c = result.getColumnLatestCell(fam1, qual1); 7066 assertNotNull(c); 7067 assertEquals(10L, c.getTimestamp()); 7068 7069 edge.setValue(1); // clock goes back 7070 region.append(a); 7071 result = region.get(new Get(row)); 7072 c = result.getColumnLatestCell(fam1, qual1); 7073 assertEquals(11L, c.getTimestamp()); 7074 7075 byte[] expected = new byte[qual1.length * 2]; 7076 System.arraycopy(qual1, 0, expected, 0, qual1.length); 7077 System.arraycopy(qual1, 0, expected, qual1.length, qual1.length); 7078 7079 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(), expected, 0, 7080 expected.length)); 7081 } 7082 7083 @Test 7084 public void testCheckAndMutateTimestampsAreMonotonic() throws IOException { 7085 region = initHRegion(tableName, method, CONF, fam1); 7086 ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); 7087 EnvironmentEdgeManager.injectEdge(edge); 7088 7089 edge.setValue(10); 7090 Put p = new Put(row); 7091 p.setDurability(Durability.SKIP_WAL); 7092 p.addColumn(fam1, qual1, qual1); 7093 region.put(p); 7094 7095 Result result = region.get(new Get(row)); 7096 Cell c = result.getColumnLatestCell(fam1, qual1); 7097 assertNotNull(c); 7098 assertEquals(10L, c.getTimestamp()); 7099 7100 edge.setValue(1); // clock goes back 7101 p = new Put(row); 7102 p.setDurability(Durability.SKIP_WAL); 7103 p.addColumn(fam1, qual1, qual2); 7104 region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(qual1), p); 7105 result = region.get(new Get(row)); 7106 c = result.getColumnLatestCell(fam1, qual1); 7107 assertEquals(10L, c.getTimestamp()); 7108 7109 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(), qual2, 0, 7110 qual2.length)); 7111 } 7112 7113 @Test 7114 public void testBatchMutateWithWrongRegionException() throws Exception { 7115 final byte[] a = Bytes.toBytes("a"); 7116 final byte[] b = Bytes.toBytes("b"); 7117 final byte[] c = Bytes.toBytes("c"); // exclusive 7118 7119 int prevLockTimeout = CONF.getInt("hbase.rowlock.wait.duration", 30000); 7120 CONF.setInt("hbase.rowlock.wait.duration", 1000); 7121 region = initHRegion(tableName, a, c, method, CONF, false, fam1); 7122 7123 Mutation[] mutations = new Mutation[] { 7124 new Put(a).add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a) 7125 .setFamily(fam1).setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), 7126 // this is outside the region boundary 7127 new Put(c).add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(c) 7128 .setFamily(fam1).setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Type.Put).build()), 7129 new Put(b) 7130 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1) 7131 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()) }; 7132 7133 OperationStatus[] status = region.batchMutate(mutations); 7134 assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode()); 7135 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, status[1].getOperationStatusCode()); 7136 assertEquals(OperationStatusCode.SUCCESS, status[2].getOperationStatusCode()); 7137 7138 // test with a row lock held for a long time 7139 final CountDownLatch obtainedRowLock = new CountDownLatch(1); 7140 ExecutorService exec = Executors.newFixedThreadPool(2); 7141 Future<Void> f1 = exec.submit(new Callable<Void>() { 7142 @Override 7143 public Void call() throws Exception { 7144 LOG.info("Acquiring row lock"); 7145 RowLock rl = region.getRowLock(b); 7146 obtainedRowLock.countDown(); 7147 LOG.info("Waiting for 5 seconds before releasing lock"); 7148 Threads.sleep(5000); 7149 LOG.info("Releasing row lock"); 7150 rl.release(); 7151 return null; 7152 } 7153 }); 7154 obtainedRowLock.await(30, TimeUnit.SECONDS); 7155 7156 Future<Void> f2 = exec.submit(new Callable<Void>() { 7157 @Override 7158 public Void call() throws Exception { 7159 Mutation[] mutations = new Mutation[] { 7160 new Put(a) 7161 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a).setFamily(fam1) 7162 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), 7163 new Put(b) 7164 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1) 7165 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), }; 7166 7167 // this will wait for the row lock, and it will eventually succeed 7168 OperationStatus[] status = region.batchMutate(mutations); 7169 assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode()); 7170 assertEquals(OperationStatusCode.SUCCESS, status[1].getOperationStatusCode()); 7171 return null; 7172 } 7173 }); 7174 7175 f1.get(); 7176 f2.get(); 7177 7178 CONF.setInt("hbase.rowlock.wait.duration", prevLockTimeout); 7179 } 7180 7181 @Test 7182 public void testBatchMutateWithZeroRowLockWait() throws Exception { 7183 final byte[] a = Bytes.toBytes("a"); 7184 final byte[] b = Bytes.toBytes("b"); 7185 final byte[] c = Bytes.toBytes("c"); // exclusive 7186 7187 Configuration conf = new Configuration(CONF); 7188 conf.setInt("hbase.rowlock.wait.duration", 0); 7189 final RegionInfo hri = 7190 RegionInfoBuilder.newBuilder(tableName).setStartKey(a).setEndKey(c).build(); 7191 final TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName) 7192 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)).build(); 7193 region = HRegion.createHRegion(hri, TEST_UTIL.getDataTestDir(), conf, htd, 7194 HBaseTestingUtil.createWal(conf, TEST_UTIL.getDataTestDirOnTestFS(method + ".log"), hri)); 7195 7196 Mutation[] mutations = new Mutation[] { 7197 new Put(a).add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a) 7198 .setFamily(fam1).setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), 7199 new Put(b) 7200 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1) 7201 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()) }; 7202 7203 OperationStatus[] status = region.batchMutate(mutations); 7204 assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode()); 7205 assertEquals(OperationStatusCode.SUCCESS, status[1].getOperationStatusCode()); 7206 7207 // test with a row lock held for a long time 7208 final CountDownLatch obtainedRowLock = new CountDownLatch(1); 7209 ExecutorService exec = Executors.newFixedThreadPool(2); 7210 Future<Void> f1 = exec.submit(new Callable<Void>() { 7211 @Override 7212 public Void call() throws Exception { 7213 LOG.info("Acquiring row lock"); 7214 RowLock rl = region.getRowLock(b); 7215 obtainedRowLock.countDown(); 7216 LOG.info("Waiting for 5 seconds before releasing lock"); 7217 Threads.sleep(5000); 7218 LOG.info("Releasing row lock"); 7219 rl.release(); 7220 return null; 7221 } 7222 }); 7223 obtainedRowLock.await(30, TimeUnit.SECONDS); 7224 7225 Future<Void> f2 = exec.submit(new Callable<Void>() { 7226 @Override 7227 public Void call() throws Exception { 7228 Mutation[] mutations = new Mutation[] { 7229 new Put(a) 7230 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a).setFamily(fam1) 7231 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), 7232 new Put(b) 7233 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1) 7234 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), }; 7235 // when handling row b we are going to spin on the failure to get the row lock 7236 // until the lock above is released, but we will still succeed so long as that 7237 // takes less time then the test time out. 7238 OperationStatus[] status = region.batchMutate(mutations); 7239 assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode()); 7240 assertEquals(OperationStatusCode.SUCCESS, status[1].getOperationStatusCode()); 7241 return null; 7242 } 7243 }); 7244 7245 f1.get(); 7246 f2.get(); 7247 } 7248 7249 @Test 7250 public void testCheckAndRowMutateTimestampsAreMonotonic() throws IOException { 7251 region = initHRegion(tableName, method, CONF, fam1); 7252 ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); 7253 EnvironmentEdgeManager.injectEdge(edge); 7254 7255 edge.setValue(10); 7256 Put p = new Put(row); 7257 p.setDurability(Durability.SKIP_WAL); 7258 p.addColumn(fam1, qual1, qual1); 7259 region.put(p); 7260 7261 Result result = region.get(new Get(row)); 7262 Cell c = result.getColumnLatestCell(fam1, qual1); 7263 assertNotNull(c); 7264 assertEquals(10L, c.getTimestamp()); 7265 7266 edge.setValue(1); // clock goes back 7267 p = new Put(row); 7268 p.setDurability(Durability.SKIP_WAL); 7269 p.addColumn(fam1, qual1, qual2); 7270 RowMutations rm = new RowMutations(row); 7271 rm.add(p); 7272 assertTrue(region.checkAndRowMutate(row, fam1, qual1, CompareOperator.EQUAL, 7273 new BinaryComparator(qual1), rm)); 7274 result = region.get(new Get(row)); 7275 c = result.getColumnLatestCell(fam1, qual1); 7276 assertEquals(10L, c.getTimestamp()); 7277 LOG.info( 7278 "c value " + Bytes.toStringBinary(c.getValueArray(), c.getValueOffset(), c.getValueLength())); 7279 7280 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(), qual2, 0, 7281 qual2.length)); 7282 } 7283 7284 private HRegion initHRegion(TableName tableName, String callingMethod, byte[]... families) 7285 throws IOException { 7286 return initHRegion(tableName, callingMethod, HBaseConfiguration.create(), families); 7287 } 7288 7289 /** 7290 * HBASE-16429 Make sure no stuck if roll writer when ring buffer is filled with appends 7291 * @throws IOException if IO error occurred during test 7292 */ 7293 @Test 7294 public void testWritesWhileRollWriter() throws IOException { 7295 int testCount = 10; 7296 int numRows = 1024; 7297 int numFamilies = 2; 7298 int numQualifiers = 2; 7299 final byte[][] families = new byte[numFamilies][]; 7300 for (int i = 0; i < numFamilies; i++) { 7301 families[i] = Bytes.toBytes("family" + i); 7302 } 7303 final byte[][] qualifiers = new byte[numQualifiers][]; 7304 for (int i = 0; i < numQualifiers; i++) { 7305 qualifiers[i] = Bytes.toBytes("qual" + i); 7306 } 7307 7308 CONF.setInt("hbase.regionserver.wal.disruptor.event.count", 2); 7309 this.region = initHRegion(tableName, method, CONF, families); 7310 try { 7311 List<Thread> threads = new ArrayList<>(); 7312 for (int i = 0; i < numRows; i++) { 7313 final int count = i; 7314 Thread t = new Thread(new Runnable() { 7315 7316 @Override 7317 public void run() { 7318 byte[] row = Bytes.toBytes("row" + count); 7319 Put put = new Put(row); 7320 put.setDurability(Durability.SYNC_WAL); 7321 byte[] value = Bytes.toBytes(String.valueOf(count)); 7322 for (byte[] family : families) { 7323 for (byte[] qualifier : qualifiers) { 7324 put.addColumn(family, qualifier, count, value); 7325 } 7326 } 7327 try { 7328 region.put(put); 7329 } catch (IOException e) { 7330 throw new RuntimeException(e); 7331 } 7332 } 7333 }); 7334 threads.add(t); 7335 } 7336 for (Thread t : threads) { 7337 t.start(); 7338 } 7339 7340 for (int i = 0; i < testCount; i++) { 7341 region.getWAL().rollWriter(); 7342 Thread.yield(); 7343 } 7344 } finally { 7345 try { 7346 HBaseTestingUtil.closeRegionAndWAL(this.region); 7347 CONF.setInt("hbase.regionserver.wal.disruptor.event.count", 16 * 1024); 7348 } catch (DroppedSnapshotException dse) { 7349 // We could get this on way out because we interrupt the background flusher and it could 7350 // fail anywhere causing a DSE over in the background flusher... only it is not properly 7351 // dealt with so could still be memory hanging out when we get to here -- memory we can't 7352 // flush because the accounting is 'off' since original DSE. 7353 } 7354 this.region = null; 7355 } 7356 } 7357 7358 @Test 7359 public void testMutateRow() throws Exception { 7360 final byte[] row = Bytes.toBytes("row"); 7361 final byte[] q1 = Bytes.toBytes("q1"); 7362 final byte[] q2 = Bytes.toBytes("q2"); 7363 final byte[] q3 = Bytes.toBytes("q3"); 7364 final byte[] q4 = Bytes.toBytes("q4"); 7365 final String v1 = "v1"; 7366 7367 region = initHRegion(tableName, method, CONF, fam1); 7368 7369 // Initial values 7370 region 7371 .batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q2, Bytes.toBytes("toBeDeleted")), 7372 new Put(row).addColumn(fam1, q3, Bytes.toBytes(5L)), 7373 new Put(row).addColumn(fam1, q4, Bytes.toBytes("a")), }); 7374 7375 // Do mutateRow 7376 Result result = region.mutateRow( 7377 new RowMutations(row).add(Arrays.asList(new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1)), 7378 new Delete(row).addColumns(fam1, q2), new Increment(row).addColumn(fam1, q3, 1), 7379 new Append(row).addColumn(fam1, q4, Bytes.toBytes("b"))))); 7380 7381 assertNotNull(result); 7382 assertEquals(6L, Bytes.toLong(result.getValue(fam1, q3))); 7383 assertEquals("ab", Bytes.toString(result.getValue(fam1, q4))); 7384 7385 // Verify the value 7386 result = region.get(new Get(row)); 7387 assertEquals(v1, Bytes.toString(result.getValue(fam1, q1))); 7388 assertNull(result.getValue(fam1, q2)); 7389 assertEquals(6L, Bytes.toLong(result.getValue(fam1, q3))); 7390 assertEquals("ab", Bytes.toString(result.getValue(fam1, q4))); 7391 } 7392 7393 @Test 7394 public void testMutateRowInParallel() throws Exception { 7395 final int numReaderThreads = 100; 7396 final CountDownLatch latch = new CountDownLatch(numReaderThreads); 7397 7398 final byte[] row = Bytes.toBytes("row"); 7399 final byte[] q1 = Bytes.toBytes("q1"); 7400 final byte[] q2 = Bytes.toBytes("q2"); 7401 final byte[] q3 = Bytes.toBytes("q3"); 7402 final byte[] q4 = Bytes.toBytes("q4"); 7403 final String v1 = "v1"; 7404 final String v2 = "v2"; 7405 7406 // We need to ensure the timestamp of the delete operation is more than the previous one 7407 final AtomicLong deleteTimestamp = new AtomicLong(); 7408 7409 region = initHRegion(tableName, method, CONF, fam1); 7410 7411 // Initial values 7412 region.batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1)) 7413 .addColumn(fam1, q2, deleteTimestamp.getAndIncrement(), Bytes.toBytes(v2)) 7414 .addColumn(fam1, q3, Bytes.toBytes(1L)).addColumn(fam1, q4, Bytes.toBytes("a")) }); 7415 7416 final AtomicReference<AssertionError> assertionError = new AtomicReference<>(); 7417 7418 // Writer thread 7419 Thread writerThread = new Thread(() -> { 7420 try { 7421 while (true) { 7422 // If all the reader threads finish, then stop the writer thread 7423 if (latch.await(0, TimeUnit.MILLISECONDS)) { 7424 return; 7425 } 7426 7427 // Execute the mutations. This should be done atomically 7428 region.mutateRow(new RowMutations(row) 7429 .add(Arrays.asList(new Put(row).addColumn(fam1, q1, Bytes.toBytes(v2)), 7430 new Delete(row).addColumns(fam1, q2, deleteTimestamp.getAndIncrement()), 7431 new Increment(row).addColumn(fam1, q3, 1L), 7432 new Append(row).addColumn(fam1, q4, Bytes.toBytes("b"))))); 7433 7434 // We need to ensure the timestamps of the Increment/Append operations are more than the 7435 // previous ones 7436 Result result = region.get(new Get(row).addColumn(fam1, q3).addColumn(fam1, q4)); 7437 long tsIncrement = result.getColumnLatestCell(fam1, q3).getTimestamp(); 7438 long tsAppend = result.getColumnLatestCell(fam1, q4).getTimestamp(); 7439 7440 // Put the initial values 7441 region.batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1)) 7442 .addColumn(fam1, q2, deleteTimestamp.getAndIncrement(), Bytes.toBytes(v2)) 7443 .addColumn(fam1, q3, tsIncrement + 1, Bytes.toBytes(1L)) 7444 .addColumn(fam1, q4, tsAppend + 1, Bytes.toBytes("a")) }); 7445 } 7446 } catch (Exception e) { 7447 assertionError.set(new AssertionError(e)); 7448 } 7449 }); 7450 writerThread.start(); 7451 7452 // Reader threads 7453 for (int i = 0; i < numReaderThreads; i++) { 7454 new Thread(() -> { 7455 try { 7456 for (int j = 0; j < 10000; j++) { 7457 // Verify the values 7458 Result result = region.get(new Get(row)); 7459 7460 // The values should be equals to either the initial values or the values after 7461 // executing the mutations 7462 String q1Value = Bytes.toString(result.getValue(fam1, q1)); 7463 if (v1.equals(q1Value)) { 7464 assertEquals(v2, Bytes.toString(result.getValue(fam1, q2))); 7465 assertEquals(1L, Bytes.toLong(result.getValue(fam1, q3))); 7466 assertEquals("a", Bytes.toString(result.getValue(fam1, q4))); 7467 } else if (v2.equals(q1Value)) { 7468 assertNull(Bytes.toString(result.getValue(fam1, q2))); 7469 assertEquals(2L, Bytes.toLong(result.getValue(fam1, q3))); 7470 assertEquals("ab", Bytes.toString(result.getValue(fam1, q4))); 7471 } else { 7472 fail("the qualifier " + Bytes.toString(q1) + " should be " + v1 + " or " + v2 7473 + ", but " + q1Value); 7474 } 7475 } 7476 } catch (Exception e) { 7477 assertionError.set(new AssertionError(e)); 7478 } catch (AssertionError e) { 7479 assertionError.set(e); 7480 } 7481 7482 latch.countDown(); 7483 }).start(); 7484 } 7485 7486 writerThread.join(); 7487 7488 if (assertionError.get() != null) { 7489 throw assertionError.get(); 7490 } 7491 } 7492 7493 @Test 7494 public void testMutateRow_WriteRequestCount() throws Exception { 7495 byte[] row1 = Bytes.toBytes("row1"); 7496 byte[] fam1 = Bytes.toBytes("fam1"); 7497 byte[] qf1 = Bytes.toBytes("qualifier"); 7498 byte[] val1 = Bytes.toBytes("value1"); 7499 7500 RowMutations rm = new RowMutations(row1); 7501 Put put = new Put(row1); 7502 put.addColumn(fam1, qf1, val1); 7503 rm.add(put); 7504 7505 this.region = initHRegion(tableName, method, CONF, fam1); 7506 long wrcBeforeMutate = this.region.writeRequestsCount.longValue(); 7507 this.region.mutateRow(rm); 7508 long wrcAfterMutate = this.region.writeRequestsCount.longValue(); 7509 Assert.assertEquals(wrcBeforeMutate + rm.getMutations().size(), wrcAfterMutate); 7510 } 7511 7512 @Test 7513 public void testBulkLoadReplicationEnabled() throws IOException { 7514 TEST_UTIL.getConfiguration().setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true); 7515 final ServerName serverName = ServerName.valueOf(name.getMethodName(), 100, 42); 7516 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName)); 7517 7518 TableDescriptor tableDescriptor = 7519 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) 7520 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)).build(); 7521 RegionInfo hri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 7522 region = HRegion.openHRegion(hri, tableDescriptor, rss.getWAL(hri), 7523 TEST_UTIL.getConfiguration(), rss, null); 7524 7525 assertTrue(region.conf.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, false)); 7526 String plugins = region.conf.get(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, ""); 7527 String replicationCoprocessorClass = ReplicationObserver.class.getCanonicalName(); 7528 assertTrue(plugins.contains(replicationCoprocessorClass)); 7529 assertTrue(region.getCoprocessorHost().getCoprocessors() 7530 .contains(ReplicationObserver.class.getSimpleName())); 7531 } 7532 7533 /** 7534 * The same as HRegion class, the only difference is that instantiateHStore will create a 7535 * different HStore - HStoreForTesting. [HBASE-8518] 7536 */ 7537 public static class HRegionForTesting extends HRegion { 7538 7539 public HRegionForTesting(final Path tableDir, final WAL wal, final FileSystem fs, 7540 final Configuration confParam, final RegionInfo regionInfo, final TableDescriptor htd, 7541 final RegionServerServices rsServices) { 7542 this(new HRegionFileSystem(confParam, fs, tableDir, regionInfo), wal, confParam, htd, 7543 rsServices); 7544 } 7545 7546 public HRegionForTesting(HRegionFileSystem fs, WAL wal, Configuration confParam, 7547 TableDescriptor htd, RegionServerServices rsServices) { 7548 super(fs, wal, confParam, htd, rsServices); 7549 } 7550 7551 /** 7552 * Create HStore instance. 7553 * @return If Mob is enabled, return HMobStore, otherwise return HStoreForTesting. 7554 */ 7555 @Override 7556 protected HStore instantiateHStore(final ColumnFamilyDescriptor family, boolean warmup) 7557 throws IOException { 7558 if (family.isMobEnabled()) { 7559 if (HFile.getFormatVersion(this.conf) < HFile.MIN_FORMAT_VERSION_WITH_TAGS) { 7560 throw new IOException("A minimum HFile version of " + HFile.MIN_FORMAT_VERSION_WITH_TAGS 7561 + " is required for MOB feature. Consider setting " + HFile.FORMAT_VERSION_KEY 7562 + " accordingly."); 7563 } 7564 return new HMobStore(this, family, this.conf, warmup); 7565 } 7566 return new HStoreForTesting(this, family, this.conf, warmup); 7567 } 7568 } 7569 7570 /** 7571 * HStoreForTesting is merely the same as HStore, the difference is in the doCompaction method of 7572 * HStoreForTesting there is a checkpoint "hbase.hstore.compaction.complete" which doesn't let 7573 * hstore compaction complete. In the former edition, this config is set in HStore class inside 7574 * compact method, though this is just for testing, otherwise it doesn't do any help. In 7575 * HBASE-8518, we try to get rid of all "hbase.hstore.compaction.complete" config (except for 7576 * testing code). 7577 */ 7578 public static class HStoreForTesting extends HStore { 7579 7580 protected HStoreForTesting(final HRegion region, final ColumnFamilyDescriptor family, 7581 final Configuration confParam, boolean warmup) throws IOException { 7582 super(region, family, confParam, warmup); 7583 } 7584 7585 @Override 7586 protected List<HStoreFile> doCompaction(CompactionRequestImpl cr, 7587 Collection<HStoreFile> filesToCompact, User user, long compactionStartTime, 7588 List<Path> newFiles) throws IOException { 7589 // let compaction incomplete. 7590 if (!this.conf.getBoolean("hbase.hstore.compaction.complete", true)) { 7591 LOG.warn("hbase.hstore.compaction.complete is set to false"); 7592 List<HStoreFile> sfs = new ArrayList<>(newFiles.size()); 7593 final boolean evictOnClose = 7594 getCacheConfig() != null ? getCacheConfig().shouldEvictOnClose() : true; 7595 for (Path newFile : newFiles) { 7596 // Create storefile around what we wrote with a reader on it. 7597 HStoreFile sf = storeEngine.createStoreFileAndReader(newFile); 7598 sf.closeStoreFile(evictOnClose); 7599 sfs.add(sf); 7600 } 7601 return sfs; 7602 } 7603 return super.doCompaction(cr, filesToCompact, user, compactionStartTime, newFiles); 7604 } 7605 } 7606 7607 @Test 7608 public void testCloseNoInterrupt() throws Exception { 7609 byte[] cf1 = Bytes.toBytes("CF1"); 7610 byte[][] families = { cf1 }; 7611 final int SLEEP_TIME = 10 * 1000; 7612 7613 Configuration conf = new Configuration(CONF); 7614 // Disable close thread interrupt and server abort behavior 7615 conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, false); 7616 conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, 1000); 7617 region = initHRegion(tableName, method, conf, families); 7618 7619 final CountDownLatch latch = new CountDownLatch(1); 7620 final AtomicBoolean holderInterrupted = new AtomicBoolean(); 7621 Thread holder = new Thread(new Runnable() { 7622 @Override 7623 public void run() { 7624 try { 7625 LOG.info("Starting region operation holder"); 7626 region.startRegionOperation(Operation.SCAN); 7627 latch.countDown(); 7628 try { 7629 Thread.sleep(SLEEP_TIME); 7630 } catch (InterruptedException e) { 7631 LOG.info("Interrupted"); 7632 holderInterrupted.set(true); 7633 } 7634 } catch (Exception e) { 7635 throw new RuntimeException(e); 7636 } finally { 7637 try { 7638 region.closeRegionOperation(); 7639 } catch (IOException e) { 7640 } 7641 LOG.info("Stopped region operation holder"); 7642 } 7643 } 7644 }); 7645 7646 holder.start(); 7647 latch.await(); 7648 HBaseTestingUtil.closeRegionAndWAL(region); 7649 region = null; 7650 holder.join(); 7651 7652 assertFalse("Region lock holder should not have been interrupted", holderInterrupted.get()); 7653 } 7654 7655 @Test 7656 public void testCloseInterrupt() throws Exception { 7657 byte[] cf1 = Bytes.toBytes("CF1"); 7658 byte[][] families = { cf1 }; 7659 final int SLEEP_TIME = 10 * 1000; 7660 7661 Configuration conf = new Configuration(CONF); 7662 // Enable close thread interrupt and server abort behavior 7663 conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, true); 7664 // Speed up the unit test, no need to wait default 10 seconds. 7665 conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, 1000); 7666 region = initHRegion(tableName, method, conf, families); 7667 7668 final CountDownLatch latch = new CountDownLatch(1); 7669 final AtomicBoolean holderInterrupted = new AtomicBoolean(); 7670 Thread holder = new Thread(new Runnable() { 7671 @Override 7672 public void run() { 7673 try { 7674 LOG.info("Starting region operation holder"); 7675 region.startRegionOperation(Operation.SCAN); 7676 latch.countDown(); 7677 try { 7678 Thread.sleep(SLEEP_TIME); 7679 } catch (InterruptedException e) { 7680 LOG.info("Interrupted"); 7681 holderInterrupted.set(true); 7682 } 7683 } catch (Exception e) { 7684 throw new RuntimeException(e); 7685 } finally { 7686 try { 7687 region.closeRegionOperation(); 7688 } catch (IOException e) { 7689 } 7690 LOG.info("Stopped region operation holder"); 7691 } 7692 } 7693 }); 7694 7695 holder.start(); 7696 latch.await(); 7697 region.close(); 7698 region = null; 7699 holder.join(); 7700 7701 assertTrue("Region lock holder was not interrupted", holderInterrupted.get()); 7702 } 7703 7704 @Test 7705 public void testCloseAbort() throws Exception { 7706 byte[] cf1 = Bytes.toBytes("CF1"); 7707 byte[][] families = { cf1 }; 7708 final int SLEEP_TIME = 10 * 1000; 7709 7710 Configuration conf = new Configuration(CONF); 7711 // Enable close thread interrupt and server abort behavior. 7712 conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, true); 7713 // Set the abort interval to a fraction of sleep time so we are guaranteed to be aborted. 7714 conf.setInt(HRegion.CLOSE_WAIT_TIME, SLEEP_TIME / 2); 7715 // Set the wait interval to a fraction of sleep time so we are guaranteed to be interrupted. 7716 conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, SLEEP_TIME / 4); 7717 region = initHRegion(tableName, method, conf, families); 7718 RegionServerServices rsServices = mock(RegionServerServices.class); 7719 when(rsServices.getServerName()).thenReturn(ServerName.valueOf("localhost", 1000, 1000)); 7720 region.rsServices = rsServices; 7721 7722 final CountDownLatch latch = new CountDownLatch(1); 7723 Thread holder = new Thread(new Runnable() { 7724 @Override 7725 public void run() { 7726 try { 7727 LOG.info("Starting region operation holder"); 7728 region.startRegionOperation(Operation.SCAN); 7729 latch.countDown(); 7730 // Hold the lock for SLEEP_TIME seconds no matter how many times we are interrupted. 7731 int timeRemaining = SLEEP_TIME; 7732 while (timeRemaining > 0) { 7733 long start = EnvironmentEdgeManager.currentTime(); 7734 try { 7735 Thread.sleep(timeRemaining); 7736 } catch (InterruptedException e) { 7737 LOG.info("Interrupted"); 7738 } 7739 long end = EnvironmentEdgeManager.currentTime(); 7740 timeRemaining -= end - start; 7741 if (timeRemaining < 0) { 7742 timeRemaining = 0; 7743 } 7744 if (timeRemaining > 0) { 7745 LOG.info("Sleeping again, remaining time " + timeRemaining + " ms"); 7746 } 7747 } 7748 } catch (Exception e) { 7749 throw new RuntimeException(e); 7750 } finally { 7751 try { 7752 region.closeRegionOperation(); 7753 } catch (IOException e) { 7754 } 7755 LOG.info("Stopped region operation holder"); 7756 } 7757 } 7758 }); 7759 7760 holder.start(); 7761 latch.await(); 7762 assertThrows(IOException.class, () -> region.close()); 7763 holder.join(); 7764 7765 // Verify the region tried to abort the server 7766 verify(rsServices, atLeast(1)).abort(anyString(), any()); 7767 } 7768 7769 @Test 7770 public void testInterruptProtection() throws Exception { 7771 byte[] cf1 = Bytes.toBytes("CF1"); 7772 byte[][] families = { cf1 }; 7773 final int SLEEP_TIME = 10 * 1000; 7774 7775 Configuration conf = new Configuration(CONF); 7776 // Enable close thread interrupt and server abort behavior. 7777 conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, true); 7778 conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, 1000); 7779 region = initHRegion(tableName, method, conf, families); 7780 7781 final CountDownLatch latch = new CountDownLatch(1); 7782 final AtomicBoolean holderInterrupted = new AtomicBoolean(); 7783 Thread holder = new Thread(new Runnable() { 7784 @Override 7785 public void run() { 7786 try { 7787 LOG.info("Starting region operation holder"); 7788 region.startRegionOperation(Operation.SCAN); 7789 LOG.info("Protecting against interrupts"); 7790 region.disableInterrupts(); 7791 try { 7792 latch.countDown(); 7793 try { 7794 Thread.sleep(SLEEP_TIME); 7795 } catch (InterruptedException e) { 7796 LOG.info("Interrupted"); 7797 holderInterrupted.set(true); 7798 } 7799 } finally { 7800 region.enableInterrupts(); 7801 } 7802 } catch (Exception e) { 7803 throw new RuntimeException(e); 7804 } finally { 7805 try { 7806 region.closeRegionOperation(); 7807 } catch (IOException e) { 7808 } 7809 LOG.info("Stopped region operation holder"); 7810 } 7811 } 7812 }); 7813 7814 holder.start(); 7815 latch.await(); 7816 region.close(); 7817 region = null; 7818 holder.join(); 7819 7820 assertFalse("Region lock holder should not have been interrupted", holderInterrupted.get()); 7821 } 7822 7823 @Test 7824 public void testRegionOnCoprocessorsChange() throws IOException { 7825 byte[] cf1 = Bytes.toBytes("CF1"); 7826 byte[][] families = { cf1 }; 7827 7828 Configuration conf = new Configuration(CONF); 7829 region = initHRegion(tableName, method, conf, families); 7830 assertNull(region.getCoprocessorHost()); 7831 7832 // set and verify the system coprocessors for region and user region 7833 Configuration newConf = new Configuration(conf); 7834 newConf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, MetaTableMetrics.class.getName()); 7835 newConf.set(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, 7836 NoOpRegionCoprocessor.class.getName()); 7837 // trigger configuration change 7838 region.onConfigurationChange(newConf); 7839 assertTrue(region.getCoprocessorHost() != null); 7840 Set<String> coprocessors = region.getCoprocessorHost().getCoprocessors(); 7841 assertTrue(coprocessors.size() == 2); 7842 assertTrue(region.getCoprocessorHost().getCoprocessors() 7843 .contains(MetaTableMetrics.class.getSimpleName())); 7844 assertTrue(region.getCoprocessorHost().getCoprocessors() 7845 .contains(NoOpRegionCoprocessor.class.getSimpleName())); 7846 7847 // remove region coprocessor and keep only user region coprocessor 7848 newConf.unset(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY); 7849 region.onConfigurationChange(newConf); 7850 assertTrue(region.getCoprocessorHost() != null); 7851 coprocessors = region.getCoprocessorHost().getCoprocessors(); 7852 assertTrue(coprocessors.size() == 1); 7853 assertTrue(region.getCoprocessorHost().getCoprocessors() 7854 .contains(NoOpRegionCoprocessor.class.getSimpleName())); 7855 } 7856 7857 @Test 7858 public void testRegionOnCoprocessorsWithoutChange() throws IOException { 7859 byte[] cf1 = Bytes.toBytes("CF1"); 7860 byte[][] families = { cf1 }; 7861 7862 Configuration conf = new Configuration(CONF); 7863 conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, 7864 MetaTableMetrics.class.getCanonicalName()); 7865 region = initHRegion(tableName, method, conf, families); 7866 // region service is null in unit test, we need to load the coprocessor once 7867 region.setCoprocessorHost(new RegionCoprocessorHost(region, null, conf)); 7868 RegionCoprocessor regionCoprocessor = 7869 region.getCoprocessorHost().findCoprocessor(MetaTableMetrics.class.getName()); 7870 7871 // simulate when other configuration may have changed and onConfigurationChange execute once 7872 region.onConfigurationChange(conf); 7873 RegionCoprocessor regionCoprocessorAfterOnConfigurationChange = 7874 region.getCoprocessorHost().findCoprocessor(MetaTableMetrics.class.getName()); 7875 assertEquals(regionCoprocessor, regionCoprocessorAfterOnConfigurationChange); 7876 } 7877 7878 public static class NoOpRegionCoprocessor implements RegionCoprocessor, RegionObserver { 7879 // a empty region coprocessor class 7880 } 7881}