001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.apache.hadoop.hbase.HBaseTestingUtil.COLUMNS; 021import static org.apache.hadoop.hbase.HBaseTestingUtil.fam1; 022import static org.apache.hadoop.hbase.HBaseTestingUtil.fam2; 023import static org.apache.hadoop.hbase.HBaseTestingUtil.fam3; 024import static org.junit.Assert.assertArrayEquals; 025import static org.junit.Assert.assertEquals; 026import static org.junit.Assert.assertFalse; 027import static org.junit.Assert.assertNotNull; 028import static org.junit.Assert.assertNull; 029import static org.junit.Assert.assertThrows; 030import static org.junit.Assert.assertTrue; 031import static org.junit.Assert.fail; 032import static org.mockito.ArgumentMatchers.any; 033import static org.mockito.ArgumentMatchers.anyLong; 034import static org.mockito.ArgumentMatchers.anyString; 035import static org.mockito.Mockito.atLeast; 036import static org.mockito.Mockito.doThrow; 037import static org.mockito.Mockito.mock; 038import static org.mockito.Mockito.never; 039import static org.mockito.Mockito.spy; 040import static org.mockito.Mockito.times; 041import static org.mockito.Mockito.verify; 042import static org.mockito.Mockito.when; 043 044import java.io.IOException; 045import java.io.InterruptedIOException; 046import java.math.BigDecimal; 047import java.security.PrivilegedExceptionAction; 048import java.util.ArrayList; 049import java.util.Arrays; 050import java.util.Collection; 051import java.util.List; 052import java.util.Map; 053import java.util.NavigableMap; 054import java.util.Objects; 055import java.util.Set; 056import java.util.TreeMap; 057import java.util.concurrent.Callable; 058import java.util.concurrent.CountDownLatch; 059import java.util.concurrent.ExecutorService; 060import java.util.concurrent.Executors; 061import java.util.concurrent.Future; 062import java.util.concurrent.TimeUnit; 063import java.util.concurrent.atomic.AtomicBoolean; 064import java.util.concurrent.atomic.AtomicInteger; 065import java.util.concurrent.atomic.AtomicLong; 066import java.util.concurrent.atomic.AtomicReference; 067import org.apache.commons.lang3.RandomStringUtils; 068import org.apache.hadoop.conf.Configuration; 069import org.apache.hadoop.fs.FSDataOutputStream; 070import org.apache.hadoop.fs.FileStatus; 071import org.apache.hadoop.fs.FileSystem; 072import org.apache.hadoop.fs.Path; 073import org.apache.hadoop.hbase.ArrayBackedTag; 074import org.apache.hadoop.hbase.Cell; 075import org.apache.hadoop.hbase.Cell.Type; 076import org.apache.hadoop.hbase.CellBuilderFactory; 077import org.apache.hadoop.hbase.CellBuilderType; 078import org.apache.hadoop.hbase.CellUtil; 079import org.apache.hadoop.hbase.CompareOperator; 080import org.apache.hadoop.hbase.CompatibilitySingletonFactory; 081import org.apache.hadoop.hbase.DoNotRetryIOException; 082import org.apache.hadoop.hbase.DroppedSnapshotException; 083import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; 084import org.apache.hadoop.hbase.HBaseClassTestRule; 085import org.apache.hadoop.hbase.HBaseConfiguration; 086import org.apache.hadoop.hbase.HBaseTestingUtil; 087import org.apache.hadoop.hbase.HConstants; 088import org.apache.hadoop.hbase.HConstants.OperationStatusCode; 089import org.apache.hadoop.hbase.HDFSBlocksDistribution; 090import org.apache.hadoop.hbase.KeyValue; 091import org.apache.hadoop.hbase.MultithreadedTestUtil; 092import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread; 093import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread; 094import org.apache.hadoop.hbase.NotServingRegionException; 095import org.apache.hadoop.hbase.PrivateCellUtil; 096import org.apache.hadoop.hbase.RegionTooBusyException; 097import org.apache.hadoop.hbase.ServerName; 098import org.apache.hadoop.hbase.SingleProcessHBaseCluster; 099import org.apache.hadoop.hbase.StartTestingClusterOption; 100import org.apache.hadoop.hbase.TableName; 101import org.apache.hadoop.hbase.TagType; 102import org.apache.hadoop.hbase.Waiter; 103import org.apache.hadoop.hbase.client.Append; 104import org.apache.hadoop.hbase.client.CheckAndMutate; 105import org.apache.hadoop.hbase.client.CheckAndMutateResult; 106import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 107import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 108import org.apache.hadoop.hbase.client.Delete; 109import org.apache.hadoop.hbase.client.Durability; 110import org.apache.hadoop.hbase.client.Get; 111import org.apache.hadoop.hbase.client.Increment; 112import org.apache.hadoop.hbase.client.Mutation; 113import org.apache.hadoop.hbase.client.Put; 114import org.apache.hadoop.hbase.client.RegionInfo; 115import org.apache.hadoop.hbase.client.RegionInfoBuilder; 116import org.apache.hadoop.hbase.client.Result; 117import org.apache.hadoop.hbase.client.RowMutations; 118import org.apache.hadoop.hbase.client.Scan; 119import org.apache.hadoop.hbase.client.Table; 120import org.apache.hadoop.hbase.client.TableDescriptor; 121import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 122import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; 123import org.apache.hadoop.hbase.coprocessor.MetaTableMetrics; 124import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor; 125import org.apache.hadoop.hbase.coprocessor.RegionObserver; 126import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException; 127import org.apache.hadoop.hbase.filter.BigDecimalComparator; 128import org.apache.hadoop.hbase.filter.BinaryComparator; 129import org.apache.hadoop.hbase.filter.ColumnCountGetFilter; 130import org.apache.hadoop.hbase.filter.Filter; 131import org.apache.hadoop.hbase.filter.FilterBase; 132import org.apache.hadoop.hbase.filter.FilterList; 133import org.apache.hadoop.hbase.filter.NullComparator; 134import org.apache.hadoop.hbase.filter.PrefixFilter; 135import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter; 136import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; 137import org.apache.hadoop.hbase.filter.SubstringComparator; 138import org.apache.hadoop.hbase.filter.ValueFilter; 139import org.apache.hadoop.hbase.io.TimeRange; 140import org.apache.hadoop.hbase.io.hfile.HFile; 141import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler; 142import org.apache.hadoop.hbase.monitoring.MonitoredTask; 143import org.apache.hadoop.hbase.monitoring.TaskMonitor; 144import org.apache.hadoop.hbase.regionserver.Region.Operation; 145import org.apache.hadoop.hbase.regionserver.Region.RowLock; 146import org.apache.hadoop.hbase.regionserver.TestHStore.FaultyFileSystem; 147import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl; 148import org.apache.hadoop.hbase.regionserver.wal.FSHLog; 149import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource; 150import org.apache.hadoop.hbase.regionserver.wal.WALUtil; 151import org.apache.hadoop.hbase.replication.regionserver.ReplicationObserver; 152import org.apache.hadoop.hbase.security.User; 153import org.apache.hadoop.hbase.test.MetricsAssertHelper; 154import org.apache.hadoop.hbase.testclassification.LargeTests; 155import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; 156import org.apache.hadoop.hbase.util.Bytes; 157import org.apache.hadoop.hbase.util.CommonFSUtils; 158import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 159import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; 160import org.apache.hadoop.hbase.util.HFileArchiveUtil; 161import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge; 162import org.apache.hadoop.hbase.util.ManualEnvironmentEdge; 163import org.apache.hadoop.hbase.util.Threads; 164import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; 165import org.apache.hadoop.hbase.wal.FaultyFSLog; 166import org.apache.hadoop.hbase.wal.NettyAsyncFSWALConfigHelper; 167import org.apache.hadoop.hbase.wal.WAL; 168import org.apache.hadoop.hbase.wal.WALEdit; 169import org.apache.hadoop.hbase.wal.WALFactory; 170import org.apache.hadoop.hbase.wal.WALKeyImpl; 171import org.apache.hadoop.hbase.wal.WALProvider; 172import org.apache.hadoop.hbase.wal.WALProvider.Writer; 173import org.apache.hadoop.hbase.wal.WALSplitUtil; 174import org.junit.After; 175import org.junit.Assert; 176import org.junit.Before; 177import org.junit.ClassRule; 178import org.junit.Rule; 179import org.junit.Test; 180import org.junit.experimental.categories.Category; 181import org.junit.rules.ExpectedException; 182import org.junit.rules.TestName; 183import org.mockito.ArgumentCaptor; 184import org.mockito.ArgumentMatcher; 185import org.mockito.Mockito; 186import org.mockito.invocation.InvocationOnMock; 187import org.mockito.stubbing.Answer; 188import org.slf4j.Logger; 189import org.slf4j.LoggerFactory; 190 191import org.apache.hbase.thirdparty.com.google.common.collect.Lists; 192import org.apache.hbase.thirdparty.com.google.protobuf.ByteString; 193import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup; 194import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup; 195import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel; 196 197import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 198import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor; 199import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor; 200import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.FlushAction; 201import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor; 202import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor; 203import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.StoreDescriptor; 204 205/** 206 * Basic stand-alone testing of HRegion. No clusters! A lot of the meta information for an HRegion 207 * now lives inside other HRegions or in the HBaseMaster, so only basic testing is possible. 208 */ 209@Category({ VerySlowRegionServerTests.class, LargeTests.class }) 210@SuppressWarnings("deprecation") 211public class TestHRegion { 212 213 @ClassRule 214 public static final HBaseClassTestRule CLASS_RULE = 215 HBaseClassTestRule.forClass(TestHRegion.class); 216 217 // Do not spin up clusters in here. If you need to spin up a cluster, do it 218 // over in TestHRegionOnCluster. 219 private static final Logger LOG = LoggerFactory.getLogger(TestHRegion.class); 220 @Rule 221 public TestName name = new TestName(); 222 @Rule 223 public final ExpectedException thrown = ExpectedException.none(); 224 225 private static final String COLUMN_FAMILY = "MyCF"; 226 private static final byte[] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY); 227 private static final EventLoopGroup GROUP = new NioEventLoopGroup(); 228 229 HRegion region = null; 230 // Do not run unit tests in parallel (? Why not? It don't work? Why not? St.Ack) 231 protected static HBaseTestingUtil TEST_UTIL; 232 public static Configuration CONF; 233 private String dir; 234 private final int MAX_VERSIONS = 2; 235 236 // Test names 237 protected TableName tableName; 238 protected String method; 239 protected final byte[] qual = Bytes.toBytes("qual"); 240 protected final byte[] qual1 = Bytes.toBytes("qual1"); 241 protected final byte[] qual2 = Bytes.toBytes("qual2"); 242 protected final byte[] qual3 = Bytes.toBytes("qual3"); 243 protected final byte[] value = Bytes.toBytes("value"); 244 protected final byte[] value1 = Bytes.toBytes("value1"); 245 protected final byte[] value2 = Bytes.toBytes("value2"); 246 protected final byte[] row = Bytes.toBytes("rowA"); 247 protected final byte[] row2 = Bytes.toBytes("rowB"); 248 249 protected final MetricsAssertHelper metricsAssertHelper = 250 CompatibilitySingletonFactory.getInstance(MetricsAssertHelper.class); 251 252 @Before 253 public void setup() throws IOException { 254 TEST_UTIL = new HBaseTestingUtil(); 255 CONF = TEST_UTIL.getConfiguration(); 256 NettyAsyncFSWALConfigHelper.setEventLoopConfig(CONF, GROUP, NioSocketChannel.class); 257 dir = TEST_UTIL.getDataTestDir("TestHRegion").toString(); 258 method = name.getMethodName(); 259 tableName = TableName.valueOf(method); 260 CONF.set(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, String.valueOf(0.09)); 261 } 262 263 @After 264 public void tearDown() throws IOException { 265 // Region may have been closed, but it is still no harm if we close it again here using HTU. 266 HBaseTestingUtil.closeRegionAndWAL(region); 267 EnvironmentEdgeManagerTestHelper.reset(); 268 LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir()); 269 TEST_UTIL.cleanupTestDir(); 270 } 271 272 /** 273 * Test that I can use the max flushed sequence id after the close. n 274 */ 275 @Test 276 public void testSequenceId() throws IOException { 277 region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES); 278 assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId()); 279 // Weird. This returns 0 if no store files or no edits. Afraid to change it. 280 assertEquals(0, (long) region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES)); 281 HBaseTestingUtil.closeRegionAndWAL(this.region); 282 assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId()); 283 assertEquals(0, (long) region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES)); 284 // Open region again. 285 region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES); 286 byte[] value = Bytes.toBytes(method); 287 // Make a random put against our cf. 288 Put put = new Put(value); 289 put.addColumn(COLUMN_FAMILY_BYTES, null, value); 290 region.put(put); 291 // No flush yet so init numbers should still be in place. 292 assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId()); 293 assertEquals(0, (long) region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES)); 294 region.flush(true); 295 long max = region.getMaxFlushedSeqId(); 296 HBaseTestingUtil.closeRegionAndWAL(this.region); 297 assertEquals(max, region.getMaxFlushedSeqId()); 298 this.region = null; 299 } 300 301 /** 302 * Test for Bug 2 of HBASE-10466. "Bug 2: Conditions for the first flush of region close 303 * (so-called pre-flush) If memstoreSize is smaller than a certain value, or when region close 304 * starts a flush is ongoing, the first flush is skipped and only the second flush takes place. 305 * However, two flushes are required in case previous flush fails and leaves some data in 306 * snapshot. The bug could cause loss of data in current memstore. The fix is removing all 307 * conditions except abort check so we ensure 2 flushes for region close." n 308 */ 309 @Test 310 public void testCloseCarryingSnapshot() throws IOException { 311 region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES); 312 HStore store = region.getStore(COLUMN_FAMILY_BYTES); 313 // Get some random bytes. 314 byte[] value = Bytes.toBytes(method); 315 // Make a random put against our cf. 316 Put put = new Put(value); 317 put.addColumn(COLUMN_FAMILY_BYTES, null, value); 318 // First put something in current memstore, which will be in snapshot after flusher.prepare() 319 region.put(put); 320 StoreFlushContext storeFlushCtx = store.createFlushContext(12345, FlushLifeCycleTracker.DUMMY); 321 storeFlushCtx.prepare(); 322 // Second put something in current memstore 323 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value); 324 region.put(put); 325 // Close with something in memstore and something in the snapshot. Make sure all is cleared. 326 HBaseTestingUtil.closeRegionAndWAL(region); 327 assertEquals(0, region.getMemStoreDataSize()); 328 region = null; 329 } 330 331 /* 332 * This test is for verifying memstore snapshot size is correctly updated in case of rollback See 333 * HBASE-10845 334 */ 335 @Test 336 public void testMemstoreSnapshotSize() throws IOException { 337 class MyFaultyFSLog extends FaultyFSLog { 338 StoreFlushContext storeFlushCtx; 339 340 public MyFaultyFSLog(FileSystem fs, Path rootDir, String logName, Configuration conf) 341 throws IOException { 342 super(fs, rootDir, logName, conf); 343 } 344 345 void setStoreFlushCtx(StoreFlushContext storeFlushCtx) { 346 this.storeFlushCtx = storeFlushCtx; 347 } 348 349 @Override 350 protected void doSync(long txid, boolean forceSync) throws IOException { 351 storeFlushCtx.prepare(); 352 super.doSync(txid, forceSync); 353 } 354 } 355 356 FileSystem fs = FileSystem.get(CONF); 357 Path rootDir = new Path(dir + "testMemstoreSnapshotSize"); 358 MyFaultyFSLog faultyLog = new MyFaultyFSLog(fs, rootDir, "testMemstoreSnapshotSize", CONF); 359 faultyLog.init(); 360 region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, faultyLog, 361 COLUMN_FAMILY_BYTES); 362 363 HStore store = region.getStore(COLUMN_FAMILY_BYTES); 364 // Get some random bytes. 365 byte[] value = Bytes.toBytes(method); 366 faultyLog.setStoreFlushCtx(store.createFlushContext(12345, FlushLifeCycleTracker.DUMMY)); 367 368 Put put = new Put(value); 369 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value); 370 faultyLog.setFailureType(FaultyFSLog.FailureType.SYNC); 371 boolean threwIOE = false; 372 try { 373 region.put(put); 374 } catch (IOException ioe) { 375 threwIOE = true; 376 } finally { 377 assertTrue("The regionserver should have thrown an exception", threwIOE); 378 } 379 MemStoreSize mss = store.getFlushableSize(); 380 assertTrue("flushable size should be zero, but it is " + mss, mss.getDataSize() == 0); 381 } 382 383 /** 384 * Create a WAL outside of the usual helper in 385 * {@link HBaseTestingUtil#createWal(Configuration, Path, RegionInfo)} because that method doesn't 386 * play nicely with FaultyFileSystem. Call this method before overriding {@code fs.file.impl}. 387 * @param callingMethod a unique component for the path, probably the name of the test method. 388 */ 389 private static WAL createWALCompatibleWithFaultyFileSystem(String callingMethod, 390 Configuration conf, TableName tableName) throws IOException { 391 final Path logDir = TEST_UTIL.getDataTestDirOnTestFS(callingMethod + ".log"); 392 final Configuration walConf = new Configuration(conf); 393 CommonFSUtils.setRootDir(walConf, logDir); 394 return new WALFactory(walConf, callingMethod) 395 .getWAL(RegionInfoBuilder.newBuilder(tableName).build()); 396 } 397 398 @Test 399 public void testMemstoreSizeAccountingWithFailedPostBatchMutate() throws IOException { 400 String testName = "testMemstoreSizeAccountingWithFailedPostBatchMutate"; 401 FileSystem fs = FileSystem.get(CONF); 402 Path rootDir = new Path(dir + testName); 403 FSHLog hLog = new FSHLog(fs, rootDir, testName, CONF); 404 hLog.init(); 405 region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, hLog, 406 COLUMN_FAMILY_BYTES); 407 HStore store = region.getStore(COLUMN_FAMILY_BYTES); 408 assertEquals(0, region.getMemStoreDataSize()); 409 410 // Put one value 411 byte[] value = Bytes.toBytes(method); 412 Put put = new Put(value); 413 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value); 414 region.put(put); 415 long onePutSize = region.getMemStoreDataSize(); 416 assertTrue(onePutSize > 0); 417 418 RegionCoprocessorHost mockedCPHost = Mockito.mock(RegionCoprocessorHost.class); 419 doThrow(new IOException()).when(mockedCPHost) 420 .postBatchMutate(Mockito.<MiniBatchOperationInProgress<Mutation>> any()); 421 region.setCoprocessorHost(mockedCPHost); 422 423 put = new Put(value); 424 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("dfg"), value); 425 try { 426 region.put(put); 427 fail("Should have failed with IOException"); 428 } catch (IOException expected) { 429 } 430 long expectedSize = onePutSize * 2; 431 assertEquals("memstoreSize should be incremented", expectedSize, region.getMemStoreDataSize()); 432 assertEquals("flushable size should be incremented", expectedSize, 433 store.getFlushableSize().getDataSize()); 434 435 region.setCoprocessorHost(null); 436 } 437 438 /** 439 * A test case of HBASE-21041 440 * @throws Exception Exception 441 */ 442 @Test 443 public void testFlushAndMemstoreSizeCounting() throws Exception { 444 byte[] family = Bytes.toBytes("family"); 445 this.region = initHRegion(tableName, method, CONF, family); 446 final WALFactory wals = new WALFactory(CONF, method); 447 try { 448 for (byte[] row : HBaseTestingUtil.ROWS) { 449 Put put = new Put(row); 450 put.addColumn(family, family, row); 451 region.put(put); 452 } 453 region.flush(true); 454 // After flush, data size should be zero 455 assertEquals(0, region.getMemStoreDataSize()); 456 // After flush, a new active mutable segment is created, so the heap size 457 // should equal to MutableSegment.DEEP_OVERHEAD 458 assertEquals(MutableSegment.DEEP_OVERHEAD, region.getMemStoreHeapSize()); 459 // After flush, offheap should be zero 460 assertEquals(0, region.getMemStoreOffHeapSize()); 461 } finally { 462 HBaseTestingUtil.closeRegionAndWAL(this.region); 463 this.region = null; 464 wals.close(); 465 } 466 } 467 468 /** 469 * Test we do not lose data if we fail a flush and then close. Part of HBase-10466. Tests the 470 * following from the issue description: "Bug 1: Wrong calculation of HRegion.memstoreSize: When a 471 * flush fails, data to be flushed is kept in each MemStore's snapshot and wait for next flush 472 * attempt to continue on it. But when the next flush succeeds, the counter of total memstore size 473 * in HRegion is always deduced by the sum of current memstore sizes instead of snapshots left 474 * from previous failed flush. This calculation is problematic that almost every time there is 475 * failed flush, HRegion.memstoreSize gets reduced by a wrong value. If region flush could not 476 * proceed for a couple cycles, the size in current memstore could be much larger than the 477 * snapshot. It's likely to drift memstoreSize much smaller than expected. In extreme case, if the 478 * error accumulates to even bigger than HRegion's memstore size limit, any further flush is 479 * skipped because flush does not do anything if memstoreSize is not larger than 0." n 480 */ 481 @Test 482 public void testFlushSizeAccounting() throws Exception { 483 final Configuration conf = HBaseConfiguration.create(CONF); 484 final WAL wal = createWALCompatibleWithFaultyFileSystem(method, conf, tableName); 485 // Only retry once. 486 conf.setInt("hbase.hstore.flush.retries.number", 1); 487 final User user = User.createUserForTesting(conf, method, new String[] { "foo" }); 488 // Inject our faulty LocalFileSystem 489 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class); 490 user.runAs(new PrivilegedExceptionAction<Object>() { 491 @Override 492 public Object run() throws Exception { 493 // Make sure it worked (above is sensitive to caching details in hadoop core) 494 FileSystem fs = FileSystem.get(conf); 495 Assert.assertEquals(FaultyFileSystem.class, fs.getClass()); 496 FaultyFileSystem ffs = (FaultyFileSystem) fs; 497 HRegion region = null; 498 try { 499 // Initialize region 500 region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, wal, 501 COLUMN_FAMILY_BYTES); 502 long size = region.getMemStoreDataSize(); 503 Assert.assertEquals(0, size); 504 // Put one item into memstore. Measure the size of one item in memstore. 505 Put p1 = new Put(row); 506 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[]) null)); 507 region.put(p1); 508 final long sizeOfOnePut = region.getMemStoreDataSize(); 509 // Fail a flush which means the current memstore will hang out as memstore 'snapshot'. 510 try { 511 LOG.info("Flushing"); 512 region.flush(true); 513 Assert.fail("Didn't bubble up IOE!"); 514 } catch (DroppedSnapshotException dse) { 515 // What we are expecting 516 region.closing.set(false); // this is needed for the rest of the test to work 517 } 518 // Make it so all writes succeed from here on out 519 ffs.fault.set(false); 520 // Check sizes. Should still be the one entry. 521 Assert.assertEquals(sizeOfOnePut, region.getMemStoreDataSize()); 522 // Now add two entries so that on this next flush that fails, we can see if we 523 // subtract the right amount, the snapshot size only. 524 Put p2 = new Put(row); 525 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[]) null)); 526 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[]) null)); 527 region.put(p2); 528 long expectedSize = sizeOfOnePut * 3; 529 Assert.assertEquals(expectedSize, region.getMemStoreDataSize()); 530 // Do a successful flush. It will clear the snapshot only. Thats how flushes work. 531 // If already a snapshot, we clear it else we move the memstore to be snapshot and flush 532 // it 533 region.flush(true); 534 // Make sure our memory accounting is right. 535 Assert.assertEquals(sizeOfOnePut * 2, region.getMemStoreDataSize()); 536 } finally { 537 HBaseTestingUtil.closeRegionAndWAL(region); 538 } 539 return null; 540 } 541 }); 542 FileSystem.closeAllForUGI(user.getUGI()); 543 } 544 545 @Test 546 public void testCloseWithFailingFlush() throws Exception { 547 final Configuration conf = HBaseConfiguration.create(CONF); 548 final WAL wal = createWALCompatibleWithFaultyFileSystem(method, conf, tableName); 549 // Only retry once. 550 conf.setInt("hbase.hstore.flush.retries.number", 1); 551 final User user = User.createUserForTesting(conf, this.method, new String[] { "foo" }); 552 // Inject our faulty LocalFileSystem 553 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class); 554 user.runAs(new PrivilegedExceptionAction<Object>() { 555 @Override 556 public Object run() throws Exception { 557 // Make sure it worked (above is sensitive to caching details in hadoop core) 558 FileSystem fs = FileSystem.get(conf); 559 Assert.assertEquals(FaultyFileSystem.class, fs.getClass()); 560 FaultyFileSystem ffs = (FaultyFileSystem) fs; 561 HRegion region = null; 562 try { 563 // Initialize region 564 region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, wal, 565 COLUMN_FAMILY_BYTES); 566 long size = region.getMemStoreDataSize(); 567 Assert.assertEquals(0, size); 568 // Put one item into memstore. Measure the size of one item in memstore. 569 Put p1 = new Put(row); 570 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[]) null)); 571 region.put(p1); 572 // Manufacture an outstanding snapshot -- fake a failed flush by doing prepare step only. 573 HStore store = region.getStore(COLUMN_FAMILY_BYTES); 574 StoreFlushContext storeFlushCtx = 575 store.createFlushContext(12345, FlushLifeCycleTracker.DUMMY); 576 storeFlushCtx.prepare(); 577 // Now add two entries to the foreground memstore. 578 Put p2 = new Put(row); 579 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[]) null)); 580 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[]) null)); 581 region.put(p2); 582 // Now try close on top of a failing flush. 583 HBaseTestingUtil.closeRegionAndWAL(region); 584 region = null; 585 fail(); 586 } catch (DroppedSnapshotException dse) { 587 // Expected 588 LOG.info("Expected DroppedSnapshotException"); 589 } finally { 590 // Make it so all writes succeed from here on out so can close clean 591 ffs.fault.set(false); 592 HBaseTestingUtil.closeRegionAndWAL(region); 593 } 594 return null; 595 } 596 }); 597 FileSystem.closeAllForUGI(user.getUGI()); 598 } 599 600 @Test 601 public void testCompactionAffectedByScanners() throws Exception { 602 byte[] family = Bytes.toBytes("family"); 603 this.region = initHRegion(tableName, method, CONF, family); 604 605 Put put = new Put(Bytes.toBytes("r1")); 606 put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); 607 region.put(put); 608 region.flush(true); 609 610 Scan scan = new Scan(); 611 scan.readVersions(3); 612 // open the first scanner 613 RegionScanner scanner1 = region.getScanner(scan); 614 615 Delete delete = new Delete(Bytes.toBytes("r1")); 616 region.delete(delete); 617 region.flush(true); 618 619 // open the second scanner 620 RegionScanner scanner2 = region.getScanner(scan); 621 622 List<Cell> results = new ArrayList<>(); 623 624 System.out.println("Smallest read point:" + region.getSmallestReadPoint()); 625 626 // make a major compaction 627 region.compact(true); 628 629 // open the third scanner 630 RegionScanner scanner3 = region.getScanner(scan); 631 632 // get data from scanner 1, 2, 3 after major compaction 633 scanner1.next(results); 634 System.out.println(results); 635 assertEquals(1, results.size()); 636 637 results.clear(); 638 scanner2.next(results); 639 System.out.println(results); 640 assertEquals(0, results.size()); 641 642 results.clear(); 643 scanner3.next(results); 644 System.out.println(results); 645 assertEquals(0, results.size()); 646 } 647 648 @Test 649 public void testToShowNPEOnRegionScannerReseek() throws Exception { 650 byte[] family = Bytes.toBytes("family"); 651 this.region = initHRegion(tableName, method, CONF, family); 652 653 Put put = new Put(Bytes.toBytes("r1")); 654 put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); 655 region.put(put); 656 put = new Put(Bytes.toBytes("r2")); 657 put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); 658 region.put(put); 659 region.flush(true); 660 661 Scan scan = new Scan(); 662 scan.readVersions(3); 663 // open the first scanner 664 RegionScanner scanner1 = region.getScanner(scan); 665 666 System.out.println("Smallest read point:" + region.getSmallestReadPoint()); 667 668 region.compact(true); 669 670 scanner1.reseek(Bytes.toBytes("r2")); 671 List<Cell> results = new ArrayList<>(); 672 scanner1.next(results); 673 Cell keyValue = results.get(0); 674 Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0); 675 scanner1.close(); 676 } 677 678 @Test 679 public void testArchiveRecoveredEditsReplay() throws Exception { 680 byte[] family = Bytes.toBytes("family"); 681 this.region = initHRegion(tableName, method, CONF, family); 682 final WALFactory wals = new WALFactory(CONF, method); 683 try { 684 Path regiondir = region.getRegionFileSystem().getRegionDir(); 685 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 686 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 687 688 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 689 690 long maxSeqId = 1050; 691 long minSeqId = 1000; 692 693 for (long i = minSeqId; i <= maxSeqId; i += 10) { 694 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); 695 fs.create(recoveredEdits); 696 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 697 698 long time = System.nanoTime(); 699 WALEdit edit = new WALEdit(); 700 edit.add( 701 new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i))); 702 writer.append(new WAL.Entry( 703 new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); 704 705 writer.close(); 706 } 707 MonitoredTask status = TaskMonitor.get().createStatus(method); 708 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 709 for (HStore store : region.getStores()) { 710 maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId - 1); 711 } 712 CONF.set("hbase.region.archive.recovered.edits", "true"); 713 CONF.set(CommonFSUtils.HBASE_WAL_DIR, "/custom_wal_dir"); 714 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status); 715 assertEquals(maxSeqId, seqId); 716 region.getMVCC().advanceTo(seqId); 717 String fakeFamilyName = recoveredEditsDir.getName(); 718 Path rootDir = new Path(CONF.get(HConstants.HBASE_DIR)); 719 Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePathForRootDir(rootDir, 720 region.getRegionInfo(), Bytes.toBytes(fakeFamilyName)); 721 FileStatus[] list = TEST_UTIL.getTestFileSystem().listStatus(storeArchiveDir); 722 assertEquals(6, list.length); 723 } finally { 724 CONF.set("hbase.region.archive.recovered.edits", "false"); 725 CONF.set(CommonFSUtils.HBASE_WAL_DIR, ""); 726 HBaseTestingUtil.closeRegionAndWAL(this.region); 727 this.region = null; 728 wals.close(); 729 } 730 } 731 732 @Test 733 public void testSkipRecoveredEditsReplay() throws Exception { 734 byte[] family = Bytes.toBytes("family"); 735 this.region = initHRegion(tableName, method, CONF, family); 736 final WALFactory wals = new WALFactory(CONF, method); 737 try { 738 Path regiondir = region.getRegionFileSystem().getRegionDir(); 739 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 740 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 741 742 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 743 744 long maxSeqId = 1050; 745 long minSeqId = 1000; 746 747 for (long i = minSeqId; i <= maxSeqId; i += 10) { 748 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); 749 fs.create(recoveredEdits); 750 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 751 752 long time = System.nanoTime(); 753 WALEdit edit = new WALEdit(); 754 edit.add( 755 new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i))); 756 writer.append(new WAL.Entry( 757 new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); 758 759 writer.close(); 760 } 761 MonitoredTask status = TaskMonitor.get().createStatus(method); 762 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 763 for (HStore store : region.getStores()) { 764 maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId - 1); 765 } 766 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status); 767 assertEquals(maxSeqId, seqId); 768 region.getMVCC().advanceTo(seqId); 769 Get get = new Get(row); 770 Result result = region.get(get); 771 for (long i = minSeqId; i <= maxSeqId; i += 10) { 772 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i)); 773 assertEquals(1, kvs.size()); 774 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0))); 775 } 776 } finally { 777 HBaseTestingUtil.closeRegionAndWAL(this.region); 778 this.region = null; 779 wals.close(); 780 } 781 } 782 783 @Test 784 public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception { 785 byte[] family = Bytes.toBytes("family"); 786 this.region = initHRegion(tableName, method, CONF, family); 787 final WALFactory wals = new WALFactory(CONF, method); 788 try { 789 Path regiondir = region.getRegionFileSystem().getRegionDir(); 790 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 791 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 792 793 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 794 795 long maxSeqId = 1050; 796 long minSeqId = 1000; 797 798 for (long i = minSeqId; i <= maxSeqId; i += 10) { 799 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); 800 fs.create(recoveredEdits); 801 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 802 803 long time = System.nanoTime(); 804 WALEdit edit = new WALEdit(); 805 edit.add( 806 new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i))); 807 writer.append(new WAL.Entry( 808 new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); 809 810 writer.close(); 811 } 812 long recoverSeqId = 1030; 813 MonitoredTask status = TaskMonitor.get().createStatus(method); 814 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 815 for (HStore store : region.getStores()) { 816 maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), recoverSeqId - 1); 817 } 818 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status); 819 assertEquals(maxSeqId, seqId); 820 region.getMVCC().advanceTo(seqId); 821 Get get = new Get(row); 822 Result result = region.get(get); 823 for (long i = minSeqId; i <= maxSeqId; i += 10) { 824 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i)); 825 if (i < recoverSeqId) { 826 assertEquals(0, kvs.size()); 827 } else { 828 assertEquals(1, kvs.size()); 829 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0))); 830 } 831 } 832 } finally { 833 HBaseTestingUtil.closeRegionAndWAL(this.region); 834 this.region = null; 835 wals.close(); 836 } 837 } 838 839 @Test 840 public void testSkipRecoveredEditsReplayAllIgnored() throws Exception { 841 byte[] family = Bytes.toBytes("family"); 842 this.region = initHRegion(tableName, method, CONF, family); 843 Path regiondir = region.getRegionFileSystem().getRegionDir(); 844 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 845 846 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 847 for (int i = 1000; i < 1050; i += 10) { 848 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); 849 FSDataOutputStream dos = fs.create(recoveredEdits); 850 dos.writeInt(i); 851 dos.close(); 852 } 853 long minSeqId = 2000; 854 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1)); 855 FSDataOutputStream dos = fs.create(recoveredEdits); 856 dos.close(); 857 858 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 859 for (HStore store : region.getStores()) { 860 maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId); 861 } 862 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, null); 863 assertEquals(minSeqId, seqId); 864 } 865 866 @Test 867 public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception { 868 byte[] family = Bytes.toBytes("family"); 869 this.region = initHRegion(tableName, method, CONF, family); 870 final WALFactory wals = new WALFactory(CONF, method); 871 try { 872 Path regiondir = region.getRegionFileSystem().getRegionDir(); 873 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 874 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 875 byte[][] columns = region.getTableDescriptor().getColumnFamilyNames().toArray(new byte[0][]); 876 877 assertEquals(0, region.getStoreFileList(columns).size()); 878 879 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 880 881 long maxSeqId = 1050; 882 long minSeqId = 1000; 883 884 for (long i = minSeqId; i <= maxSeqId; i += 10) { 885 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); 886 fs.create(recoveredEdits); 887 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 888 889 long time = System.nanoTime(); 890 WALEdit edit = null; 891 if (i == maxSeqId) { 892 edit = WALEdit.createCompaction(region.getRegionInfo(), 893 CompactionDescriptor.newBuilder().setTableName(ByteString.copyFrom(tableName.getName())) 894 .setFamilyName(ByteString.copyFrom(regionName)) 895 .setEncodedRegionName(ByteString.copyFrom(regionName)) 896 .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString()))) 897 .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName())).build()); 898 } else { 899 edit = new WALEdit(); 900 edit.add( 901 new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i))); 902 } 903 writer.append(new WAL.Entry( 904 new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); 905 writer.close(); 906 } 907 908 long recoverSeqId = 1030; 909 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 910 MonitoredTask status = TaskMonitor.get().createStatus(method); 911 for (HStore store : region.getStores()) { 912 maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), recoverSeqId - 1); 913 } 914 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status); 915 assertEquals(maxSeqId, seqId); 916 917 // assert that the files are flushed 918 assertEquals(1, region.getStoreFileList(columns).size()); 919 920 } finally { 921 HBaseTestingUtil.closeRegionAndWAL(this.region); 922 this.region = null; 923 wals.close(); 924 } 925 } 926 927 @Test 928 public void testRecoveredEditsReplayCompaction() throws Exception { 929 testRecoveredEditsReplayCompaction(false); 930 testRecoveredEditsReplayCompaction(true); 931 } 932 933 public void testRecoveredEditsReplayCompaction(boolean mismatchedRegionName) throws Exception { 934 CONF.setClass(HConstants.REGION_IMPL, HRegionForTesting.class, Region.class); 935 byte[] family = Bytes.toBytes("family"); 936 this.region = initHRegion(tableName, method, CONF, family); 937 final WALFactory wals = new WALFactory(CONF, method); 938 try { 939 Path regiondir = region.getRegionFileSystem().getRegionDir(); 940 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 941 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 942 943 long maxSeqId = 3; 944 long minSeqId = 0; 945 946 for (long i = minSeqId; i < maxSeqId; i++) { 947 Put put = new Put(Bytes.toBytes(i)); 948 put.addColumn(family, Bytes.toBytes(i), Bytes.toBytes(i)); 949 region.put(put); 950 region.flush(true); 951 } 952 953 // this will create a region with 3 files 954 assertEquals(3, region.getStore(family).getStorefilesCount()); 955 List<Path> storeFiles = new ArrayList<>(3); 956 for (HStoreFile sf : region.getStore(family).getStorefiles()) { 957 storeFiles.add(sf.getPath()); 958 } 959 960 // disable compaction completion 961 CONF.setBoolean("hbase.hstore.compaction.complete", false); 962 region.compactStores(); 963 964 // ensure that nothing changed 965 assertEquals(3, region.getStore(family).getStorefilesCount()); 966 967 // now find the compacted file, and manually add it to the recovered edits 968 Path tmpDir = new Path(region.getRegionFileSystem().getTempDir(), Bytes.toString(family)); 969 FileStatus[] files = CommonFSUtils.listStatus(fs, tmpDir); 970 String errorMsg = "Expected to find 1 file in the region temp directory " 971 + "from the compaction, could not find any"; 972 assertNotNull(errorMsg, files); 973 assertEquals(errorMsg, 1, files.length); 974 // move the file inside region dir 975 Path newFile = 976 region.getRegionFileSystem().commitStoreFile(Bytes.toString(family), files[0].getPath()); 977 978 byte[] encodedNameAsBytes = this.region.getRegionInfo().getEncodedNameAsBytes(); 979 byte[] fakeEncodedNameAsBytes = new byte[encodedNameAsBytes.length]; 980 for (int i = 0; i < encodedNameAsBytes.length; i++) { 981 // Mix the byte array to have a new encodedName 982 fakeEncodedNameAsBytes[i] = (byte) (encodedNameAsBytes[i] + 1); 983 } 984 985 CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor( 986 this.region.getRegionInfo(), mismatchedRegionName ? fakeEncodedNameAsBytes : null, family, 987 storeFiles, Lists.newArrayList(newFile), 988 region.getRegionFileSystem().getStoreDir(Bytes.toString(family))); 989 990 WALUtil.writeCompactionMarker(region.getWAL(), this.region.getReplicationScope(), 991 this.region.getRegionInfo(), compactionDescriptor, region.getMVCC(), null); 992 993 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 994 995 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000)); 996 fs.create(recoveredEdits); 997 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 998 999 long time = System.nanoTime(); 1000 1001 writer.append(new WAL.Entry( 1002 new WALKeyImpl(regionName, tableName, 10, time, HConstants.DEFAULT_CLUSTER_ID), 1003 WALEdit.createCompaction(region.getRegionInfo(), compactionDescriptor))); 1004 writer.close(); 1005 1006 // close the region now, and reopen again 1007 region.getTableDescriptor(); 1008 region.getRegionInfo(); 1009 HBaseTestingUtil.closeRegionAndWAL(this.region); 1010 try { 1011 region = HRegion.openHRegion(region, null); 1012 } catch (WrongRegionException wre) { 1013 fail("Matching encoded region name should not have produced WrongRegionException"); 1014 } 1015 1016 // now check whether we have only one store file, the compacted one 1017 Collection<HStoreFile> sfs = region.getStore(family).getStorefiles(); 1018 for (HStoreFile sf : sfs) { 1019 LOG.info(Objects.toString(sf.getPath())); 1020 } 1021 if (!mismatchedRegionName) { 1022 assertEquals(1, region.getStore(family).getStorefilesCount()); 1023 } 1024 files = CommonFSUtils.listStatus(fs, tmpDir); 1025 assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0); 1026 1027 for (long i = minSeqId; i < maxSeqId; i++) { 1028 Get get = new Get(Bytes.toBytes(i)); 1029 Result result = region.get(get); 1030 byte[] value = result.getValue(family, Bytes.toBytes(i)); 1031 assertArrayEquals(Bytes.toBytes(i), value); 1032 } 1033 } finally { 1034 HBaseTestingUtil.closeRegionAndWAL(this.region); 1035 this.region = null; 1036 wals.close(); 1037 CONF.setClass(HConstants.REGION_IMPL, HRegion.class, Region.class); 1038 } 1039 } 1040 1041 @Test 1042 public void testFlushMarkers() throws Exception { 1043 // tests that flush markers are written to WAL and handled at recovered edits 1044 byte[] family = Bytes.toBytes("family"); 1045 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + ".log"); 1046 final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration()); 1047 CommonFSUtils.setRootDir(walConf, logDir); 1048 final WALFactory wals = new WALFactory(walConf, method); 1049 final WAL wal = wals.getWAL(RegionInfoBuilder.newBuilder(tableName).build()); 1050 1051 this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF, 1052 false, Durability.USE_DEFAULT, wal, family); 1053 try { 1054 Path regiondir = region.getRegionFileSystem().getRegionDir(); 1055 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 1056 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 1057 1058 long maxSeqId = 3; 1059 long minSeqId = 0; 1060 1061 for (long i = minSeqId; i < maxSeqId; i++) { 1062 Put put = new Put(Bytes.toBytes(i)); 1063 put.addColumn(family, Bytes.toBytes(i), Bytes.toBytes(i)); 1064 region.put(put); 1065 region.flush(true); 1066 } 1067 1068 // this will create a region with 3 files from flush 1069 assertEquals(3, region.getStore(family).getStorefilesCount()); 1070 List<String> storeFiles = new ArrayList<>(3); 1071 for (HStoreFile sf : region.getStore(family).getStorefiles()) { 1072 storeFiles.add(sf.getPath().getName()); 1073 } 1074 1075 // now verify that the flush markers are written 1076 wal.shutdown(); 1077 WAL.Reader reader = WALFactory.createReader(fs, AbstractFSWALProvider.getCurrentFileName(wal), 1078 TEST_UTIL.getConfiguration()); 1079 try { 1080 List<WAL.Entry> flushDescriptors = new ArrayList<>(); 1081 long lastFlushSeqId = -1; 1082 while (true) { 1083 WAL.Entry entry = reader.next(); 1084 if (entry == null) { 1085 break; 1086 } 1087 Cell cell = entry.getEdit().getCells().get(0); 1088 if (WALEdit.isMetaEditFamily(cell)) { 1089 FlushDescriptor flushDesc = WALEdit.getFlushDescriptor(cell); 1090 assertNotNull(flushDesc); 1091 assertArrayEquals(tableName.getName(), flushDesc.getTableName().toByteArray()); 1092 if (flushDesc.getAction() == FlushAction.START_FLUSH) { 1093 assertTrue(flushDesc.getFlushSequenceNumber() > lastFlushSeqId); 1094 } else if (flushDesc.getAction() == FlushAction.COMMIT_FLUSH) { 1095 assertTrue(flushDesc.getFlushSequenceNumber() == lastFlushSeqId); 1096 } 1097 lastFlushSeqId = flushDesc.getFlushSequenceNumber(); 1098 assertArrayEquals(regionName, flushDesc.getEncodedRegionName().toByteArray()); 1099 assertEquals(1, flushDesc.getStoreFlushesCount()); // only one store 1100 StoreFlushDescriptor storeFlushDesc = flushDesc.getStoreFlushes(0); 1101 assertArrayEquals(family, storeFlushDesc.getFamilyName().toByteArray()); 1102 assertEquals("family", storeFlushDesc.getStoreHomeDir()); 1103 if (flushDesc.getAction() == FlushAction.START_FLUSH) { 1104 assertEquals(0, storeFlushDesc.getFlushOutputCount()); 1105 } else { 1106 assertEquals(1, storeFlushDesc.getFlushOutputCount()); // only one file from flush 1107 assertTrue(storeFiles.contains(storeFlushDesc.getFlushOutput(0))); 1108 } 1109 1110 flushDescriptors.add(entry); 1111 } 1112 } 1113 1114 assertEquals(3 * 2, flushDescriptors.size()); // START_FLUSH and COMMIT_FLUSH per flush 1115 1116 // now write those markers to the recovered edits again. 1117 1118 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 1119 1120 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000)); 1121 fs.create(recoveredEdits); 1122 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 1123 1124 for (WAL.Entry entry : flushDescriptors) { 1125 writer.append(entry); 1126 } 1127 writer.close(); 1128 } finally { 1129 if (null != reader) { 1130 try { 1131 reader.close(); 1132 } catch (IOException exception) { 1133 LOG.warn("Problem closing wal: " + exception.getMessage()); 1134 LOG.debug("exception details", exception); 1135 } 1136 } 1137 } 1138 1139 // close the region now, and reopen again 1140 HBaseTestingUtil.closeRegionAndWAL(this.region); 1141 region = HRegion.openHRegion(region, null); 1142 1143 // now check whether we have can read back the data from region 1144 for (long i = minSeqId; i < maxSeqId; i++) { 1145 Get get = new Get(Bytes.toBytes(i)); 1146 Result result = region.get(get); 1147 byte[] value = result.getValue(family, Bytes.toBytes(i)); 1148 assertArrayEquals(Bytes.toBytes(i), value); 1149 } 1150 } finally { 1151 HBaseTestingUtil.closeRegionAndWAL(this.region); 1152 this.region = null; 1153 wals.close(); 1154 } 1155 } 1156 1157 static class IsFlushWALMarker implements ArgumentMatcher<WALEdit> { 1158 volatile FlushAction[] actions; 1159 1160 public IsFlushWALMarker(FlushAction... actions) { 1161 this.actions = actions; 1162 } 1163 1164 @Override 1165 public boolean matches(WALEdit edit) { 1166 List<Cell> cells = edit.getCells(); 1167 if (cells.isEmpty()) { 1168 return false; 1169 } 1170 if (WALEdit.isMetaEditFamily(cells.get(0))) { 1171 FlushDescriptor desc; 1172 try { 1173 desc = WALEdit.getFlushDescriptor(cells.get(0)); 1174 } catch (IOException e) { 1175 LOG.warn(e.toString(), e); 1176 return false; 1177 } 1178 if (desc != null) { 1179 for (FlushAction action : actions) { 1180 if (desc.getAction() == action) { 1181 return true; 1182 } 1183 } 1184 } 1185 } 1186 return false; 1187 } 1188 1189 public IsFlushWALMarker set(FlushAction... actions) { 1190 this.actions = actions; 1191 return this; 1192 } 1193 } 1194 1195 @Test 1196 public void testFlushMarkersWALFail() throws Exception { 1197 // test the cases where the WAL append for flush markers fail. 1198 byte[] family = Bytes.toBytes("family"); 1199 1200 // spy an actual WAL implementation to throw exception (was not able to mock) 1201 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + "log"); 1202 1203 final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration()); 1204 CommonFSUtils.setRootDir(walConf, logDir); 1205 // Make up a WAL that we can manipulate at append time. 1206 class FailAppendFlushMarkerWAL extends FSHLog { 1207 volatile FlushAction[] flushActions = null; 1208 1209 public FailAppendFlushMarkerWAL(FileSystem fs, Path root, String logDir, Configuration conf) 1210 throws IOException { 1211 super(fs, root, logDir, conf); 1212 } 1213 1214 @Override 1215 protected Writer createWriterInstance(Path path) throws IOException { 1216 final Writer w = super.createWriterInstance(path); 1217 return new Writer() { 1218 @Override 1219 public void close() throws IOException { 1220 w.close(); 1221 } 1222 1223 @Override 1224 public void sync(boolean forceSync) throws IOException { 1225 w.sync(forceSync); 1226 } 1227 1228 @Override 1229 public void append(Entry entry) throws IOException { 1230 List<Cell> cells = entry.getEdit().getCells(); 1231 if (WALEdit.isMetaEditFamily(cells.get(0))) { 1232 FlushDescriptor desc = WALEdit.getFlushDescriptor(cells.get(0)); 1233 if (desc != null) { 1234 for (FlushAction flushAction : flushActions) { 1235 if (desc.getAction().equals(flushAction)) { 1236 throw new IOException("Failed to append flush marker! " + flushAction); 1237 } 1238 } 1239 } 1240 } 1241 w.append(entry); 1242 } 1243 1244 @Override 1245 public long getLength() { 1246 return w.getLength(); 1247 } 1248 1249 @Override 1250 public long getSyncedLength() { 1251 return w.getSyncedLength(); 1252 } 1253 }; 1254 } 1255 } 1256 FailAppendFlushMarkerWAL wal = new FailAppendFlushMarkerWAL(FileSystem.get(walConf), 1257 CommonFSUtils.getRootDir(walConf), method, walConf); 1258 wal.init(); 1259 this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF, 1260 false, Durability.USE_DEFAULT, wal, family); 1261 int i = 0; 1262 Put put = new Put(Bytes.toBytes(i)); 1263 put.setDurability(Durability.SKIP_WAL); // have to skip mocked wal 1264 put.addColumn(family, Bytes.toBytes(i), Bytes.toBytes(i)); 1265 region.put(put); 1266 1267 // 1. Test case where START_FLUSH throws exception 1268 wal.flushActions = new FlushAction[] { FlushAction.START_FLUSH }; 1269 1270 // start cache flush will throw exception 1271 try { 1272 region.flush(true); 1273 fail("This should have thrown exception"); 1274 } catch (DroppedSnapshotException unexpected) { 1275 // this should not be a dropped snapshot exception. Meaning that RS will not abort 1276 throw unexpected; 1277 } catch (IOException expected) { 1278 // expected 1279 } 1280 // The WAL is hosed now. It has two edits appended. We cannot roll the log without it 1281 // throwing a DroppedSnapshotException to force an abort. Just clean up the mess. 1282 region.close(true); 1283 wal.close(); 1284 1285 // 2. Test case where START_FLUSH succeeds but COMMIT_FLUSH will throw exception 1286 wal.flushActions = new FlushAction[] { FlushAction.COMMIT_FLUSH }; 1287 wal = new FailAppendFlushMarkerWAL(FileSystem.get(walConf), CommonFSUtils.getRootDir(walConf), 1288 method, walConf); 1289 wal.init(); 1290 this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF, 1291 false, Durability.USE_DEFAULT, wal, family); 1292 region.put(put); 1293 // 3. Test case where ABORT_FLUSH will throw exception. 1294 // Even if ABORT_FLUSH throws exception, we should not fail with IOE, but continue with 1295 // DroppedSnapshotException. Below COMMIT_FLUSH will cause flush to abort 1296 wal.flushActions = new FlushAction[] { FlushAction.COMMIT_FLUSH, FlushAction.ABORT_FLUSH }; 1297 1298 try { 1299 region.flush(true); 1300 fail("This should have thrown exception"); 1301 } catch (DroppedSnapshotException expected) { 1302 // we expect this exception, since we were able to write the snapshot, but failed to 1303 // write the flush marker to WAL 1304 } catch (IOException unexpected) { 1305 throw unexpected; 1306 } 1307 } 1308 1309 @Test 1310 public void testGetWhileRegionClose() throws IOException { 1311 Configuration hc = initSplit(); 1312 int numRows = 100; 1313 byte[][] families = { fam1, fam2, fam3 }; 1314 1315 // Setting up region 1316 this.region = initHRegion(tableName, method, hc, families); 1317 // Put data in region 1318 final int startRow = 100; 1319 putData(startRow, numRows, qual1, families); 1320 putData(startRow, numRows, qual2, families); 1321 putData(startRow, numRows, qual3, families); 1322 final AtomicBoolean done = new AtomicBoolean(false); 1323 final AtomicInteger gets = new AtomicInteger(0); 1324 GetTillDoneOrException[] threads = new GetTillDoneOrException[10]; 1325 try { 1326 // Set ten threads running concurrently getting from the region. 1327 for (int i = 0; i < threads.length / 2; i++) { 1328 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets); 1329 threads[i].setDaemon(true); 1330 threads[i].start(); 1331 } 1332 // Artificially make the condition by setting closing flag explicitly. 1333 // I can't make the issue happen with a call to region.close(). 1334 this.region.closing.set(true); 1335 for (int i = threads.length / 2; i < threads.length; i++) { 1336 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets); 1337 threads[i].setDaemon(true); 1338 threads[i].start(); 1339 } 1340 } finally { 1341 if (this.region != null) { 1342 HBaseTestingUtil.closeRegionAndWAL(this.region); 1343 this.region = null; 1344 } 1345 } 1346 done.set(true); 1347 for (GetTillDoneOrException t : threads) { 1348 try { 1349 t.join(); 1350 } catch (InterruptedException e) { 1351 e.printStackTrace(); 1352 } 1353 if (t.e != null) { 1354 LOG.info("Exception=" + t.e); 1355 assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException); 1356 } 1357 } 1358 } 1359 1360 /* 1361 * Thread that does get on single row until 'done' flag is flipped. If an exception causes us to 1362 * fail, it records it. 1363 */ 1364 class GetTillDoneOrException extends Thread { 1365 private final Get g; 1366 private final AtomicBoolean done; 1367 private final AtomicInteger count; 1368 private Exception e; 1369 1370 GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, 1371 final AtomicInteger c) { 1372 super("getter." + i); 1373 this.g = new Get(r); 1374 this.done = d; 1375 this.count = c; 1376 } 1377 1378 @Override 1379 public void run() { 1380 while (!this.done.get()) { 1381 try { 1382 assertTrue(region.get(g).size() > 0); 1383 this.count.incrementAndGet(); 1384 } catch (Exception e) { 1385 this.e = e; 1386 break; 1387 } 1388 } 1389 } 1390 } 1391 1392 /* 1393 * An involved filter test. Has multiple column families and deletes in mix. 1394 */ 1395 @Test 1396 public void testWeirdCacheBehaviour() throws Exception { 1397 final TableName tableName = TableName.valueOf(name.getMethodName()); 1398 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"), 1399 Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") }; 1400 this.region = initHRegion(tableName, method, CONF, FAMILIES); 1401 String value = "this is the value"; 1402 String value2 = "this is some other value"; 1403 String keyPrefix1 = "prefix1"; 1404 String keyPrefix2 = "prefix2"; 1405 String keyPrefix3 = "prefix3"; 1406 putRows(this.region, 3, value, keyPrefix1); 1407 putRows(this.region, 3, value, keyPrefix2); 1408 putRows(this.region, 3, value, keyPrefix3); 1409 putRows(this.region, 3, value2, keyPrefix1); 1410 putRows(this.region, 3, value2, keyPrefix2); 1411 putRows(this.region, 3, value2, keyPrefix3); 1412 System.out.println("Checking values for key: " + keyPrefix1); 1413 assertEquals("Got back incorrect number of rows from scan", 3, 1414 getNumberOfRows(keyPrefix1, value2, this.region)); 1415 System.out.println("Checking values for key: " + keyPrefix2); 1416 assertEquals("Got back incorrect number of rows from scan", 3, 1417 getNumberOfRows(keyPrefix2, value2, this.region)); 1418 System.out.println("Checking values for key: " + keyPrefix3); 1419 assertEquals("Got back incorrect number of rows from scan", 3, 1420 getNumberOfRows(keyPrefix3, value2, this.region)); 1421 deleteColumns(this.region, value2, keyPrefix1); 1422 deleteColumns(this.region, value2, keyPrefix2); 1423 deleteColumns(this.region, value2, keyPrefix3); 1424 System.out.println("Starting important checks....."); 1425 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0, 1426 getNumberOfRows(keyPrefix1, value2, this.region)); 1427 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0, 1428 getNumberOfRows(keyPrefix2, value2, this.region)); 1429 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0, 1430 getNumberOfRows(keyPrefix3, value2, this.region)); 1431 } 1432 1433 @Test 1434 public void testAppendWithReadOnlyTable() throws Exception { 1435 final TableName tableName = TableName.valueOf(name.getMethodName()); 1436 this.region = initHRegion(tableName, method, CONF, true, Bytes.toBytes("somefamily")); 1437 boolean exceptionCaught = false; 1438 Append append = new Append(Bytes.toBytes("somerow")); 1439 append.setDurability(Durability.SKIP_WAL); 1440 append.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1441 Bytes.toBytes("somevalue")); 1442 try { 1443 region.append(append); 1444 } catch (IOException e) { 1445 exceptionCaught = true; 1446 } 1447 assertTrue(exceptionCaught == true); 1448 } 1449 1450 @Test 1451 public void testIncrWithReadOnlyTable() throws Exception { 1452 final TableName tableName = TableName.valueOf(name.getMethodName()); 1453 this.region = initHRegion(tableName, method, CONF, true, Bytes.toBytes("somefamily")); 1454 boolean exceptionCaught = false; 1455 Increment inc = new Increment(Bytes.toBytes("somerow")); 1456 inc.setDurability(Durability.SKIP_WAL); 1457 inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L); 1458 try { 1459 region.increment(inc); 1460 } catch (IOException e) { 1461 exceptionCaught = true; 1462 } 1463 assertTrue(exceptionCaught == true); 1464 } 1465 1466 private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException { 1467 InternalScanner scanner = buildScanner(keyPrefix, value, r); 1468 int count = 0; 1469 boolean more = false; 1470 List<Cell> results = new ArrayList<>(); 1471 do { 1472 more = scanner.next(results); 1473 if (results != null && !results.isEmpty()) count++; 1474 else break; 1475 Delete delete = new Delete(CellUtil.cloneRow(results.get(0))); 1476 delete.addColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2")); 1477 r.delete(delete); 1478 results.clear(); 1479 } while (more); 1480 assertEquals("Did not perform correct number of deletes", 3, count); 1481 } 1482 1483 private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception { 1484 InternalScanner resultScanner = buildScanner(keyPrefix, value, r); 1485 int numberOfResults = 0; 1486 List<Cell> results = new ArrayList<>(); 1487 boolean more = false; 1488 do { 1489 more = resultScanner.next(results); 1490 if (results != null && !results.isEmpty()) numberOfResults++; 1491 else break; 1492 for (Cell kv : results) { 1493 System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv))); 1494 } 1495 results.clear(); 1496 } while (more); 1497 return numberOfResults; 1498 } 1499 1500 private InternalScanner buildScanner(String keyPrefix, String value, HRegion r) 1501 throws IOException { 1502 // Defaults FilterList.Operator.MUST_PASS_ALL. 1503 FilterList allFilters = new FilterList(); 1504 allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix))); 1505 // Only return rows where this column value exists in the row. 1506 SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"), 1507 Bytes.toBytes("qual2"), CompareOperator.EQUAL, Bytes.toBytes(value)); 1508 filter.setFilterIfMissing(true); 1509 allFilters.addFilter(filter); 1510 Scan scan = new Scan(); 1511 scan.addFamily(Bytes.toBytes("trans-blob")); 1512 scan.addFamily(Bytes.toBytes("trans-type")); 1513 scan.addFamily(Bytes.toBytes("trans-date")); 1514 scan.addFamily(Bytes.toBytes("trans-tags")); 1515 scan.addFamily(Bytes.toBytes("trans-group")); 1516 scan.setFilter(allFilters); 1517 return r.getScanner(scan); 1518 } 1519 1520 private void putRows(HRegion r, int numRows, String value, String key) throws IOException { 1521 for (int i = 0; i < numRows; i++) { 1522 String row = key + "_" + i/* UUID.randomUUID().toString() */; 1523 System.out.println(String.format("Saving row: %s, with value %s", row, value)); 1524 Put put = new Put(Bytes.toBytes(row)); 1525 put.setDurability(Durability.SKIP_WAL); 1526 put.addColumn(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob")); 1527 put.addColumn(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement")); 1528 put.addColumn(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999")); 1529 put.addColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value)); 1530 put.addColumn(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId")); 1531 r.put(put); 1532 } 1533 } 1534 1535 @Test 1536 public void testFamilyWithAndWithoutColon() throws Exception { 1537 byte[] cf = Bytes.toBytes(COLUMN_FAMILY); 1538 this.region = initHRegion(tableName, method, CONF, cf); 1539 Put p = new Put(tableName.toBytes()); 1540 byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":"); 1541 p.addColumn(cfwithcolon, cfwithcolon, cfwithcolon); 1542 boolean exception = false; 1543 try { 1544 this.region.put(p); 1545 } catch (NoSuchColumnFamilyException e) { 1546 exception = true; 1547 } 1548 assertTrue(exception); 1549 } 1550 1551 @Test 1552 public void testBatchPut_whileNoRowLocksHeld() throws IOException { 1553 final Put[] puts = new Put[10]; 1554 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); 1555 long syncs = prepareRegionForBachPut(puts, source, false); 1556 1557 OperationStatus[] codes = this.region.batchMutate(puts); 1558 assertEquals(10, codes.length); 1559 for (int i = 0; i < 10; i++) { 1560 assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode()); 1561 } 1562 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source); 1563 1564 LOG.info("Next a batch put with one invalid family"); 1565 puts[5].addColumn(Bytes.toBytes("BAD_CF"), qual, value); 1566 codes = this.region.batchMutate(puts); 1567 assertEquals(10, codes.length); 1568 for (int i = 0; i < 10; i++) { 1569 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS, 1570 codes[i].getOperationStatusCode()); 1571 } 1572 1573 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source); 1574 } 1575 1576 @Test 1577 public void testBatchPut_whileMultipleRowLocksHeld() throws Exception { 1578 final Put[] puts = new Put[10]; 1579 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); 1580 long syncs = prepareRegionForBachPut(puts, source, false); 1581 1582 puts[5].addColumn(Bytes.toBytes("BAD_CF"), qual, value); 1583 1584 LOG.info("batchPut will have to break into four batches to avoid row locks"); 1585 RowLock rowLock1 = region.getRowLock(Bytes.toBytes("row_2")); 1586 RowLock rowLock2 = region.getRowLock(Bytes.toBytes("row_1")); 1587 RowLock rowLock3 = region.getRowLock(Bytes.toBytes("row_3")); 1588 RowLock rowLock4 = region.getRowLock(Bytes.toBytes("row_3"), true); 1589 1590 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF); 1591 final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<>(); 1592 final CountDownLatch startingPuts = new CountDownLatch(1); 1593 final CountDownLatch startingClose = new CountDownLatch(1); 1594 TestThread putter = new TestThread(ctx) { 1595 @Override 1596 public void doWork() throws IOException { 1597 startingPuts.countDown(); 1598 retFromThread.set(region.batchMutate(puts)); 1599 } 1600 }; 1601 LOG.info("...starting put thread while holding locks"); 1602 ctx.addThread(putter); 1603 ctx.startThreads(); 1604 1605 // Now attempt to close the region from another thread. Prior to HBASE-12565 1606 // this would cause the in-progress batchMutate operation to to fail with 1607 // exception because it use to release and re-acquire the close-guard lock 1608 // between batches. Caller then didn't get status indicating which writes succeeded. 1609 // We now expect this thread to block until the batchMutate call finishes. 1610 Thread regionCloseThread = new TestThread(ctx) { 1611 @Override 1612 public void doWork() { 1613 try { 1614 startingPuts.await(); 1615 // Give some time for the batch mutate to get in. 1616 // We don't want to race with the mutate 1617 Thread.sleep(10); 1618 startingClose.countDown(); 1619 HBaseTestingUtil.closeRegionAndWAL(region); 1620 region = null; 1621 } catch (IOException e) { 1622 throw new RuntimeException(e); 1623 } catch (InterruptedException e) { 1624 throw new RuntimeException(e); 1625 } 1626 } 1627 }; 1628 regionCloseThread.start(); 1629 1630 startingClose.await(); 1631 startingPuts.await(); 1632 Thread.sleep(100); 1633 LOG.info("...releasing row lock 1, which should let put thread continue"); 1634 rowLock1.release(); 1635 rowLock2.release(); 1636 rowLock3.release(); 1637 waitForCounter(source, "syncTimeNumOps", syncs + 1); 1638 1639 LOG.info("...joining on put thread"); 1640 ctx.stop(); 1641 regionCloseThread.join(); 1642 1643 OperationStatus[] codes = retFromThread.get(); 1644 for (int i = 0; i < codes.length; i++) { 1645 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS, 1646 codes[i].getOperationStatusCode()); 1647 } 1648 rowLock4.release(); 1649 } 1650 1651 private void waitForCounter(MetricsWALSource source, String metricName, long expectedCount) 1652 throws InterruptedException { 1653 long startWait = EnvironmentEdgeManager.currentTime(); 1654 long currentCount; 1655 while ((currentCount = metricsAssertHelper.getCounter(metricName, source)) < expectedCount) { 1656 Thread.sleep(100); 1657 if (EnvironmentEdgeManager.currentTime() - startWait > 10000) { 1658 fail(String.format("Timed out waiting for '%s' >= '%s', currentCount=%s", metricName, 1659 expectedCount, currentCount)); 1660 } 1661 } 1662 } 1663 1664 @Test 1665 public void testAtomicBatchPut() throws IOException { 1666 final Put[] puts = new Put[10]; 1667 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); 1668 long syncs = prepareRegionForBachPut(puts, source, false); 1669 1670 // 1. Straight forward case, should succeed 1671 OperationStatus[] codes = this.region.batchMutate(puts, true); 1672 assertEquals(10, codes.length); 1673 for (int i = 0; i < 10; i++) { 1674 assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode()); 1675 } 1676 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source); 1677 1678 // 2. Failed to get lock 1679 RowLock lock = region.getRowLock(Bytes.toBytes("row_" + 3)); 1680 // Method {@link HRegion#getRowLock(byte[])} is reentrant. As 'row_3' is locked in this 1681 // thread, need to run {@link HRegion#batchMutate(HRegion.BatchOperation)} in different thread 1682 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF); 1683 final AtomicReference<IOException> retFromThread = new AtomicReference<>(); 1684 final CountDownLatch finishedPuts = new CountDownLatch(1); 1685 TestThread putter = new TestThread(ctx) { 1686 @Override 1687 public void doWork() throws IOException { 1688 try { 1689 region.batchMutate(puts, true); 1690 } catch (IOException ioe) { 1691 LOG.error("test failed!", ioe); 1692 retFromThread.set(ioe); 1693 } 1694 finishedPuts.countDown(); 1695 } 1696 }; 1697 LOG.info("...starting put thread while holding locks"); 1698 ctx.addThread(putter); 1699 ctx.startThreads(); 1700 LOG.info("...waiting for batch puts while holding locks"); 1701 try { 1702 finishedPuts.await(); 1703 } catch (InterruptedException e) { 1704 LOG.error("Interrupted!", e); 1705 } finally { 1706 if (lock != null) { 1707 lock.release(); 1708 } 1709 } 1710 assertNotNull(retFromThread.get()); 1711 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source); 1712 1713 // 3. Exception thrown in validation 1714 LOG.info("Next a batch put with one invalid family"); 1715 puts[5].addColumn(Bytes.toBytes("BAD_CF"), qual, value); 1716 thrown.expect(NoSuchColumnFamilyException.class); 1717 this.region.batchMutate(puts, true); 1718 } 1719 1720 @Test 1721 public void testBatchPutWithTsSlop() throws Exception { 1722 // add data with a timestamp that is too recent for range. Ensure assert 1723 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000); 1724 final Put[] puts = new Put[10]; 1725 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); 1726 1727 long syncs = prepareRegionForBachPut(puts, source, true); 1728 1729 OperationStatus[] codes = this.region.batchMutate(puts); 1730 assertEquals(10, codes.length); 1731 for (int i = 0; i < 10; i++) { 1732 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode()); 1733 } 1734 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source); 1735 } 1736 1737 /** 1738 * @return syncs initial syncTimeNumOps 1739 */ 1740 private long prepareRegionForBachPut(final Put[] puts, final MetricsWALSource source, 1741 boolean slop) throws IOException { 1742 this.region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES); 1743 1744 LOG.info("First a batch put with all valid puts"); 1745 for (int i = 0; i < puts.length; i++) { 1746 puts[i] = slop 1747 ? new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100) 1748 : new Put(Bytes.toBytes("row_" + i)); 1749 puts[i].addColumn(COLUMN_FAMILY_BYTES, qual, value); 1750 } 1751 1752 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source); 1753 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source); 1754 return syncs; 1755 } 1756 1757 // //////////////////////////////////////////////////////////////////////////// 1758 // checkAndMutate tests 1759 // //////////////////////////////////////////////////////////////////////////// 1760 @Test 1761 @Deprecated 1762 public void testCheckAndMutate_WithEmptyRowValue() throws IOException { 1763 byte[] row1 = Bytes.toBytes("row1"); 1764 byte[] fam1 = Bytes.toBytes("fam1"); 1765 byte[] qf1 = Bytes.toBytes("qualifier"); 1766 byte[] emptyVal = new byte[] {}; 1767 byte[] val1 = Bytes.toBytes("value1"); 1768 byte[] val2 = Bytes.toBytes("value2"); 1769 1770 // Setting up region 1771 this.region = initHRegion(tableName, method, CONF, fam1); 1772 // Putting empty data in key 1773 Put put = new Put(row1); 1774 put.addColumn(fam1, qf1, emptyVal); 1775 1776 // checkAndPut with empty value 1777 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1778 new BinaryComparator(emptyVal), put); 1779 assertTrue(res); 1780 1781 // Putting data in key 1782 put = new Put(row1); 1783 put.addColumn(fam1, qf1, val1); 1784 1785 // checkAndPut with correct value 1786 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1787 new BinaryComparator(emptyVal), put); 1788 assertTrue(res); 1789 1790 // not empty anymore 1791 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1792 new BinaryComparator(emptyVal), put); 1793 assertFalse(res); 1794 1795 Delete delete = new Delete(row1); 1796 delete.addColumn(fam1, qf1); 1797 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1798 new BinaryComparator(emptyVal), delete); 1799 assertFalse(res); 1800 1801 put = new Put(row1); 1802 put.addColumn(fam1, qf1, val2); 1803 // checkAndPut with correct value 1804 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val1), 1805 put); 1806 assertTrue(res); 1807 1808 // checkAndDelete with correct value 1809 delete = new Delete(row1); 1810 delete.addColumn(fam1, qf1); 1811 delete.addColumn(fam1, qf1); 1812 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val2), 1813 delete); 1814 assertTrue(res); 1815 1816 delete = new Delete(row1); 1817 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1818 new BinaryComparator(emptyVal), delete); 1819 assertTrue(res); 1820 1821 // checkAndPut looking for a null value 1822 put = new Put(row1); 1823 put.addColumn(fam1, qf1, val1); 1824 1825 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new NullComparator(), put); 1826 assertTrue(res); 1827 } 1828 1829 @Test 1830 @Deprecated 1831 public void testCheckAndMutate_WithWrongValue() throws IOException { 1832 byte[] row1 = Bytes.toBytes("row1"); 1833 byte[] fam1 = Bytes.toBytes("fam1"); 1834 byte[] qf1 = Bytes.toBytes("qualifier"); 1835 byte[] val1 = Bytes.toBytes("value1"); 1836 byte[] val2 = Bytes.toBytes("value2"); 1837 BigDecimal bd1 = new BigDecimal(Double.MAX_VALUE); 1838 BigDecimal bd2 = new BigDecimal(Double.MIN_VALUE); 1839 1840 // Setting up region 1841 this.region = initHRegion(tableName, method, CONF, fam1); 1842 // Putting data in key 1843 Put put = new Put(row1); 1844 put.addColumn(fam1, qf1, val1); 1845 region.put(put); 1846 1847 // checkAndPut with wrong value 1848 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1849 new BinaryComparator(val2), put); 1850 assertEquals(false, res); 1851 1852 // checkAndDelete with wrong value 1853 Delete delete = new Delete(row1); 1854 delete.addFamily(fam1); 1855 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val2), 1856 put); 1857 assertEquals(false, res); 1858 1859 // Putting data in key 1860 put = new Put(row1); 1861 put.addColumn(fam1, qf1, Bytes.toBytes(bd1)); 1862 region.put(put); 1863 1864 // checkAndPut with wrong value 1865 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1866 new BigDecimalComparator(bd2), put); 1867 assertEquals(false, res); 1868 1869 // checkAndDelete with wrong value 1870 delete = new Delete(row1); 1871 delete.addFamily(fam1); 1872 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1873 new BigDecimalComparator(bd2), put); 1874 assertEquals(false, res); 1875 } 1876 1877 @Test 1878 @Deprecated 1879 public void testCheckAndMutate_WithCorrectValue() throws IOException { 1880 byte[] row1 = Bytes.toBytes("row1"); 1881 byte[] fam1 = Bytes.toBytes("fam1"); 1882 byte[] qf1 = Bytes.toBytes("qualifier"); 1883 byte[] val1 = Bytes.toBytes("value1"); 1884 BigDecimal bd1 = new BigDecimal(Double.MIN_VALUE); 1885 1886 // Setting up region 1887 this.region = initHRegion(tableName, method, CONF, fam1); 1888 // Putting data in key 1889 long now = EnvironmentEdgeManager.currentTime(); 1890 Put put = new Put(row1); 1891 put.addColumn(fam1, qf1, now, val1); 1892 region.put(put); 1893 1894 // checkAndPut with correct value 1895 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1896 new BinaryComparator(val1), put); 1897 assertEquals("First", true, res); 1898 1899 // checkAndDelete with correct value 1900 Delete delete = new Delete(row1, now + 1); 1901 delete.addColumn(fam1, qf1); 1902 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val1), 1903 delete); 1904 assertEquals("Delete", true, res); 1905 1906 // Putting data in key 1907 put = new Put(row1); 1908 put.addColumn(fam1, qf1, now + 2, Bytes.toBytes(bd1)); 1909 region.put(put); 1910 1911 // checkAndPut with correct value 1912 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1913 new BigDecimalComparator(bd1), put); 1914 assertEquals("Second put", true, res); 1915 1916 // checkAndDelete with correct value 1917 delete = new Delete(row1, now + 3); 1918 delete.addColumn(fam1, qf1); 1919 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1920 new BigDecimalComparator(bd1), delete); 1921 assertEquals("Second delete", true, res); 1922 } 1923 1924 @Test 1925 @Deprecated 1926 public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException { 1927 byte[] row1 = Bytes.toBytes("row1"); 1928 byte[] fam1 = Bytes.toBytes("fam1"); 1929 byte[] qf1 = Bytes.toBytes("qualifier"); 1930 byte[] val1 = Bytes.toBytes("value1"); 1931 byte[] val2 = Bytes.toBytes("value2"); 1932 byte[] val3 = Bytes.toBytes("value3"); 1933 byte[] val4 = Bytes.toBytes("value4"); 1934 1935 // Setting up region 1936 this.region = initHRegion(tableName, method, CONF, fam1); 1937 // Putting val3 in key 1938 Put put = new Put(row1); 1939 put.addColumn(fam1, qf1, val3); 1940 region.put(put); 1941 1942 // Test CompareOp.LESS: original = val3, compare with val3, fail 1943 boolean res = 1944 region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS, new BinaryComparator(val3), put); 1945 assertEquals(false, res); 1946 1947 // Test CompareOp.LESS: original = val3, compare with val4, fail 1948 res = 1949 region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS, new BinaryComparator(val4), put); 1950 assertEquals(false, res); 1951 1952 // Test CompareOp.LESS: original = val3, compare with val2, 1953 // succeed (now value = val2) 1954 put = new Put(row1); 1955 put.addColumn(fam1, qf1, val2); 1956 res = 1957 region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS, new BinaryComparator(val2), put); 1958 assertEquals(true, res); 1959 1960 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val3, fail 1961 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS_OR_EQUAL, 1962 new BinaryComparator(val3), put); 1963 assertEquals(false, res); 1964 1965 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val2, 1966 // succeed (value still = val2) 1967 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS_OR_EQUAL, 1968 new BinaryComparator(val2), put); 1969 assertEquals(true, res); 1970 1971 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val1, 1972 // succeed (now value = val3) 1973 put = new Put(row1); 1974 put.addColumn(fam1, qf1, val3); 1975 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS_OR_EQUAL, 1976 new BinaryComparator(val1), put); 1977 assertEquals(true, res); 1978 1979 // Test CompareOp.GREATER: original = val3, compare with val3, fail 1980 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER, 1981 new BinaryComparator(val3), put); 1982 assertEquals(false, res); 1983 1984 // Test CompareOp.GREATER: original = val3, compare with val2, fail 1985 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER, 1986 new BinaryComparator(val2), put); 1987 assertEquals(false, res); 1988 1989 // Test CompareOp.GREATER: original = val3, compare with val4, 1990 // succeed (now value = val2) 1991 put = new Put(row1); 1992 put.addColumn(fam1, qf1, val2); 1993 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER, 1994 new BinaryComparator(val4), put); 1995 assertEquals(true, res); 1996 1997 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val1, fail 1998 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER_OR_EQUAL, 1999 new BinaryComparator(val1), put); 2000 assertEquals(false, res); 2001 2002 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val2, 2003 // succeed (value still = val2) 2004 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER_OR_EQUAL, 2005 new BinaryComparator(val2), put); 2006 assertEquals(true, res); 2007 2008 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val3, succeed 2009 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER_OR_EQUAL, 2010 new BinaryComparator(val3), put); 2011 assertEquals(true, res); 2012 } 2013 2014 @Test 2015 @Deprecated 2016 public void testCheckAndPut_ThatPutWasWritten() throws IOException { 2017 byte[] row1 = Bytes.toBytes("row1"); 2018 byte[] fam1 = Bytes.toBytes("fam1"); 2019 byte[] fam2 = Bytes.toBytes("fam2"); 2020 byte[] qf1 = Bytes.toBytes("qualifier"); 2021 byte[] val1 = Bytes.toBytes("value1"); 2022 byte[] val2 = Bytes.toBytes("value2"); 2023 2024 byte[][] families = { fam1, fam2 }; 2025 2026 // Setting up region 2027 this.region = initHRegion(tableName, method, CONF, families); 2028 // Putting data in the key to check 2029 Put put = new Put(row1); 2030 put.addColumn(fam1, qf1, val1); 2031 region.put(put); 2032 2033 // Creating put to add 2034 long ts = EnvironmentEdgeManager.currentTime(); 2035 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2); 2036 put = new Put(row1); 2037 put.add(kv); 2038 2039 // checkAndPut with wrong value 2040 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 2041 new BinaryComparator(val1), put); 2042 assertEquals(true, res); 2043 2044 Get get = new Get(row1); 2045 get.addColumn(fam2, qf1); 2046 Cell[] actual = region.get(get).rawCells(); 2047 2048 Cell[] expected = { kv }; 2049 2050 assertEquals(expected.length, actual.length); 2051 for (int i = 0; i < actual.length; i++) { 2052 assertEquals(expected[i], actual[i]); 2053 } 2054 } 2055 2056 @Test 2057 @Deprecated 2058 public void testCheckAndPut_wrongRowInPut() throws IOException { 2059 this.region = initHRegion(tableName, method, CONF, COLUMNS); 2060 Put put = new Put(row2); 2061 put.addColumn(fam1, qual1, value1); 2062 try { 2063 region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(value2), 2064 put); 2065 fail(); 2066 } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) { 2067 // expected exception. 2068 } 2069 } 2070 2071 @Test 2072 @Deprecated 2073 public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException { 2074 byte[] row1 = Bytes.toBytes("row1"); 2075 byte[] fam1 = Bytes.toBytes("fam1"); 2076 byte[] fam2 = Bytes.toBytes("fam2"); 2077 byte[] qf1 = Bytes.toBytes("qualifier1"); 2078 byte[] qf2 = Bytes.toBytes("qualifier2"); 2079 byte[] qf3 = Bytes.toBytes("qualifier3"); 2080 byte[] val1 = Bytes.toBytes("value1"); 2081 byte[] val2 = Bytes.toBytes("value2"); 2082 byte[] val3 = Bytes.toBytes("value3"); 2083 byte[] emptyVal = new byte[] {}; 2084 2085 byte[][] families = { fam1, fam2 }; 2086 2087 // Setting up region 2088 this.region = initHRegion(tableName, method, CONF, families); 2089 // Put content 2090 Put put = new Put(row1); 2091 put.addColumn(fam1, qf1, val1); 2092 region.put(put); 2093 Threads.sleep(2); 2094 2095 put = new Put(row1); 2096 put.addColumn(fam1, qf1, val2); 2097 put.addColumn(fam2, qf1, val3); 2098 put.addColumn(fam2, qf2, val2); 2099 put.addColumn(fam2, qf3, val1); 2100 put.addColumn(fam1, qf3, val1); 2101 region.put(put); 2102 2103 LOG.info("get={}", region.get(new Get(row1).addColumn(fam1, qf1)).toString()); 2104 2105 // Multi-column delete 2106 Delete delete = new Delete(row1); 2107 delete.addColumn(fam1, qf1); 2108 delete.addColumn(fam2, qf1); 2109 delete.addColumn(fam1, qf3); 2110 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 2111 new BinaryComparator(val2), delete); 2112 assertEquals(true, res); 2113 2114 Get get = new Get(row1); 2115 get.addColumn(fam1, qf1); 2116 get.addColumn(fam1, qf3); 2117 get.addColumn(fam2, qf2); 2118 Result r = region.get(get); 2119 assertEquals(2, r.size()); 2120 assertArrayEquals(val1, r.getValue(fam1, qf1)); 2121 assertArrayEquals(val2, r.getValue(fam2, qf2)); 2122 2123 // Family delete 2124 delete = new Delete(row1); 2125 delete.addFamily(fam2); 2126 res = region.checkAndMutate(row1, fam2, qf1, CompareOperator.EQUAL, 2127 new BinaryComparator(emptyVal), delete); 2128 assertEquals(true, res); 2129 2130 get = new Get(row1); 2131 r = region.get(get); 2132 assertEquals(1, r.size()); 2133 assertArrayEquals(val1, r.getValue(fam1, qf1)); 2134 2135 // Row delete 2136 delete = new Delete(row1); 2137 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val1), 2138 delete); 2139 assertEquals(true, res); 2140 get = new Get(row1); 2141 r = region.get(get); 2142 assertEquals(0, r.size()); 2143 } 2144 2145 @Test 2146 @Deprecated 2147 public void testCheckAndMutate_WithFilters() throws Throwable { 2148 final byte[] FAMILY = Bytes.toBytes("fam"); 2149 2150 // Setting up region 2151 this.region = initHRegion(tableName, method, CONF, FAMILY); 2152 2153 // Put one row 2154 Put put = new Put(row); 2155 put.addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")); 2156 put.addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")); 2157 put.addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")); 2158 region.put(put); 2159 2160 // Put with success 2161 boolean ok = region.checkAndMutate(row, 2162 new FilterList( 2163 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2164 Bytes.toBytes("a")), 2165 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2166 Bytes.toBytes("b"))), 2167 new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d"))); 2168 assertTrue(ok); 2169 2170 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))); 2171 assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D")))); 2172 2173 // Put with failure 2174 ok = region.checkAndMutate(row, 2175 new FilterList( 2176 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2177 Bytes.toBytes("a")), 2178 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2179 Bytes.toBytes("c"))), 2180 new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e"))); 2181 assertFalse(ok); 2182 2183 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E"))).isEmpty()); 2184 2185 // Delete with success 2186 ok = region.checkAndMutate(row, 2187 new FilterList( 2188 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2189 Bytes.toBytes("a")), 2190 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2191 Bytes.toBytes("b"))), 2192 new Delete(row).addColumns(FAMILY, Bytes.toBytes("D"))); 2193 assertTrue(ok); 2194 2195 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))).isEmpty()); 2196 2197 // Mutate with success 2198 ok = region.checkAndRowMutate(row, 2199 new FilterList( 2200 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2201 Bytes.toBytes("a")), 2202 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2203 Bytes.toBytes("b"))), 2204 new RowMutations(row) 2205 .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e"))) 2206 .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A")))); 2207 assertTrue(ok); 2208 2209 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E"))); 2210 assertEquals("e", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("E")))); 2211 2212 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty()); 2213 } 2214 2215 @Test 2216 @Deprecated 2217 public void testCheckAndMutate_WithFiltersAndTimeRange() throws Throwable { 2218 final byte[] FAMILY = Bytes.toBytes("fam"); 2219 2220 // Setting up region 2221 this.region = initHRegion(tableName, method, CONF, FAMILY); 2222 2223 // Put with specifying the timestamp 2224 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), 100, Bytes.toBytes("a"))); 2225 2226 // Put with success 2227 boolean ok = region.checkAndMutate(row, 2228 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2229 Bytes.toBytes("a")), 2230 TimeRange.between(0, 101), 2231 new Put(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b"))); 2232 assertTrue(ok); 2233 2234 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2235 assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2236 2237 // Put with failure 2238 ok = region.checkAndMutate(row, 2239 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2240 Bytes.toBytes("a")), 2241 TimeRange.between(0, 100), 2242 new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c"))); 2243 assertFalse(ok); 2244 2245 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("C"))).isEmpty()); 2246 2247 // Mutate with success 2248 ok = region.checkAndRowMutate(row, 2249 new SingleColumnValueFilter( 2250 FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, Bytes.toBytes("a")), 2251 TimeRange.between(0, 101), 2252 new RowMutations(row) 2253 .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d"))) 2254 .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A")))); 2255 assertTrue(ok); 2256 2257 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))); 2258 assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D")))); 2259 2260 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty()); 2261 } 2262 2263 @Test 2264 @Deprecated 2265 public void testCheckAndMutate_wrongMutationType() throws Throwable { 2266 // Setting up region 2267 this.region = initHRegion(tableName, method, CONF, fam1); 2268 2269 try { 2270 region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(value1), 2271 new Increment(row).addColumn(fam1, qual1, 1)); 2272 fail("should throw DoNotRetryIOException"); 2273 } catch (DoNotRetryIOException e) { 2274 assertEquals("Unsupported mutate type: INCREMENT", e.getMessage()); 2275 } 2276 2277 try { 2278 region.checkAndMutate(row, 2279 new SingleColumnValueFilter(fam1, qual1, CompareOperator.EQUAL, value1), 2280 new Increment(row).addColumn(fam1, qual1, 1)); 2281 fail("should throw DoNotRetryIOException"); 2282 } catch (DoNotRetryIOException e) { 2283 assertEquals("Unsupported mutate type: INCREMENT", e.getMessage()); 2284 } 2285 } 2286 2287 @Test 2288 @Deprecated 2289 public void testCheckAndMutate_wrongRow() throws Throwable { 2290 final byte[] wrongRow = Bytes.toBytes("wrongRow"); 2291 2292 // Setting up region 2293 this.region = initHRegion(tableName, method, CONF, fam1); 2294 2295 try { 2296 region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(value1), 2297 new Put(wrongRow).addColumn(fam1, qual1, value1)); 2298 fail("should throw DoNotRetryIOException"); 2299 } catch (DoNotRetryIOException e) { 2300 assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>", 2301 e.getMessage()); 2302 } 2303 2304 try { 2305 region.checkAndMutate(row, 2306 new SingleColumnValueFilter(fam1, qual1, CompareOperator.EQUAL, value1), 2307 new Put(wrongRow).addColumn(fam1, qual1, value1)); 2308 fail("should throw DoNotRetryIOException"); 2309 } catch (DoNotRetryIOException e) { 2310 assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>", 2311 e.getMessage()); 2312 } 2313 2314 try { 2315 region.checkAndRowMutate(row, fam1, qual1, CompareOperator.EQUAL, 2316 new BinaryComparator(value1), 2317 new RowMutations(wrongRow).add((Mutation) new Put(wrongRow).addColumn(fam1, qual1, value1)) 2318 .add((Mutation) new Delete(wrongRow).addColumns(fam1, qual2))); 2319 fail("should throw DoNotRetryIOException"); 2320 } catch (DoNotRetryIOException e) { 2321 assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>", 2322 e.getMessage()); 2323 } 2324 2325 try { 2326 region.checkAndRowMutate(row, 2327 new SingleColumnValueFilter(fam1, qual1, CompareOperator.EQUAL, value1), 2328 new RowMutations(wrongRow).add((Mutation) new Put(wrongRow).addColumn(fam1, qual1, value1)) 2329 .add((Mutation) new Delete(wrongRow).addColumns(fam1, qual2))); 2330 fail("should throw DoNotRetryIOException"); 2331 } catch (DoNotRetryIOException e) { 2332 assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>", 2333 e.getMessage()); 2334 } 2335 } 2336 2337 @Test 2338 public void testCheckAndMutateWithEmptyRowValue() throws IOException { 2339 byte[] row1 = Bytes.toBytes("row1"); 2340 byte[] fam1 = Bytes.toBytes("fam1"); 2341 byte[] qf1 = Bytes.toBytes("qualifier"); 2342 byte[] emptyVal = new byte[] {}; 2343 byte[] val1 = Bytes.toBytes("value1"); 2344 byte[] val2 = Bytes.toBytes("value2"); 2345 2346 // Setting up region 2347 this.region = initHRegion(tableName, method, CONF, fam1); 2348 // Putting empty data in key 2349 Put put = new Put(row1); 2350 put.addColumn(fam1, qf1, emptyVal); 2351 2352 // checkAndPut with empty value 2353 CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2354 .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(put)); 2355 assertTrue(res.isSuccess()); 2356 assertNull(res.getResult()); 2357 2358 // Putting data in key 2359 put = new Put(row1); 2360 put.addColumn(fam1, qf1, val1); 2361 2362 // checkAndPut with correct value 2363 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2364 .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(put)); 2365 assertTrue(res.isSuccess()); 2366 assertNull(res.getResult()); 2367 2368 // not empty anymore 2369 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2370 .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(put)); 2371 assertFalse(res.isSuccess()); 2372 assertNull(res.getResult()); 2373 2374 Delete delete = new Delete(row1); 2375 delete.addColumn(fam1, qf1); 2376 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2377 .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(delete)); 2378 assertFalse(res.isSuccess()); 2379 assertNull(res.getResult()); 2380 2381 put = new Put(row1); 2382 put.addColumn(fam1, qf1, val2); 2383 // checkAndPut with correct value 2384 res = region.checkAndMutate( 2385 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(put)); 2386 assertTrue(res.isSuccess()); 2387 assertNull(res.getResult()); 2388 2389 // checkAndDelete with correct value 2390 delete = new Delete(row1); 2391 delete.addColumn(fam1, qf1); 2392 delete.addColumn(fam1, qf1); 2393 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2394 .ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(delete)); 2395 assertTrue(res.isSuccess()); 2396 assertNull(res.getResult()); 2397 2398 delete = new Delete(row1); 2399 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2400 .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(delete)); 2401 assertTrue(res.isSuccess()); 2402 assertNull(res.getResult()); 2403 2404 // checkAndPut looking for a null value 2405 put = new Put(row1); 2406 put.addColumn(fam1, qf1, val1); 2407 2408 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1).ifNotExists(fam1, qf1).build(put)); 2409 assertTrue(res.isSuccess()); 2410 assertNull(res.getResult()); 2411 } 2412 2413 @Test 2414 public void testCheckAndMutateWithWrongValue() throws IOException { 2415 byte[] row1 = Bytes.toBytes("row1"); 2416 byte[] fam1 = Bytes.toBytes("fam1"); 2417 byte[] qf1 = Bytes.toBytes("qualifier"); 2418 byte[] val1 = Bytes.toBytes("value1"); 2419 byte[] val2 = Bytes.toBytes("value2"); 2420 BigDecimal bd1 = new BigDecimal(Double.MAX_VALUE); 2421 BigDecimal bd2 = new BigDecimal(Double.MIN_VALUE); 2422 2423 // Setting up region 2424 this.region = initHRegion(tableName, method, CONF, fam1); 2425 // Putting data in key 2426 Put put = new Put(row1); 2427 put.addColumn(fam1, qf1, val1); 2428 region.put(put); 2429 2430 // checkAndPut with wrong value 2431 CheckAndMutateResult res = region.checkAndMutate( 2432 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(put)); 2433 assertFalse(res.isSuccess()); 2434 assertNull(res.getResult()); 2435 2436 // checkAndDelete with wrong value 2437 Delete delete = new Delete(row1); 2438 delete.addFamily(fam1); 2439 res = region.checkAndMutate( 2440 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(put)); 2441 assertFalse(res.isSuccess()); 2442 assertNull(res.getResult()); 2443 2444 // Putting data in key 2445 put = new Put(row1); 2446 put.addColumn(fam1, qf1, Bytes.toBytes(bd1)); 2447 region.put(put); 2448 2449 // checkAndPut with wrong value 2450 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2451 .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd2)).build(put)); 2452 assertFalse(res.isSuccess()); 2453 assertNull(res.getResult()); 2454 2455 // checkAndDelete with wrong value 2456 delete = new Delete(row1); 2457 delete.addFamily(fam1); 2458 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2459 .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd2)).build(delete)); 2460 assertFalse(res.isSuccess()); 2461 assertNull(res.getResult()); 2462 } 2463 2464 @Test 2465 public void testCheckAndMutateWithCorrectValue() throws IOException { 2466 byte[] row1 = Bytes.toBytes("row1"); 2467 byte[] fam1 = Bytes.toBytes("fam1"); 2468 byte[] qf1 = Bytes.toBytes("qualifier"); 2469 byte[] val1 = Bytes.toBytes("value1"); 2470 BigDecimal bd1 = new BigDecimal(Double.MIN_VALUE); 2471 2472 // Setting up region 2473 this.region = initHRegion(tableName, method, CONF, fam1); 2474 // Putting data in key 2475 long now = EnvironmentEdgeManager.currentTime(); 2476 Put put = new Put(row1); 2477 put.addColumn(fam1, qf1, now, val1); 2478 region.put(put); 2479 2480 // checkAndPut with correct value 2481 CheckAndMutateResult res = region.checkAndMutate( 2482 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(put)); 2483 assertTrue("First", res.isSuccess()); 2484 2485 // checkAndDelete with correct value 2486 Delete delete = new Delete(row1, now + 1); 2487 delete.addColumn(fam1, qf1); 2488 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2489 .ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(delete)); 2490 assertTrue("Delete", res.isSuccess()); 2491 assertNull(res.getResult()); 2492 2493 // Putting data in key 2494 put = new Put(row1); 2495 put.addColumn(fam1, qf1, now + 2, Bytes.toBytes(bd1)); 2496 region.put(put); 2497 2498 // checkAndPut with correct value 2499 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2500 .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd1)).build(put)); 2501 assertTrue("Second put", res.isSuccess()); 2502 assertNull(res.getResult()); 2503 2504 // checkAndDelete with correct value 2505 delete = new Delete(row1, now + 3); 2506 delete.addColumn(fam1, qf1); 2507 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2508 .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd1)).build(delete)); 2509 assertTrue("Second delete", res.isSuccess()); 2510 assertNull(res.getResult()); 2511 } 2512 2513 @Test 2514 public void testCheckAndMutateWithNonEqualCompareOp() throws IOException { 2515 byte[] row1 = Bytes.toBytes("row1"); 2516 byte[] fam1 = Bytes.toBytes("fam1"); 2517 byte[] qf1 = Bytes.toBytes("qualifier"); 2518 byte[] val1 = Bytes.toBytes("value1"); 2519 byte[] val2 = Bytes.toBytes("value2"); 2520 byte[] val3 = Bytes.toBytes("value3"); 2521 byte[] val4 = Bytes.toBytes("value4"); 2522 2523 // Setting up region 2524 this.region = initHRegion(tableName, method, CONF, fam1); 2525 // Putting val3 in key 2526 Put put = new Put(row1); 2527 put.addColumn(fam1, qf1, val3); 2528 region.put(put); 2529 2530 // Test CompareOp.LESS: original = val3, compare with val3, fail 2531 CheckAndMutateResult res = region.checkAndMutate( 2532 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.LESS, val3).build(put)); 2533 assertFalse(res.isSuccess()); 2534 assertNull(res.getResult()); 2535 2536 // Test CompareOp.LESS: original = val3, compare with val4, fail 2537 res = region.checkAndMutate( 2538 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.LESS, val4).build(put)); 2539 assertFalse(res.isSuccess()); 2540 assertNull(res.getResult()); 2541 2542 // Test CompareOp.LESS: original = val3, compare with val2, 2543 // succeed (now value = val2) 2544 put = new Put(row1); 2545 put.addColumn(fam1, qf1, val2); 2546 res = region.checkAndMutate( 2547 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.LESS, val2).build(put)); 2548 assertTrue(res.isSuccess()); 2549 assertNull(res.getResult()); 2550 2551 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val3, fail 2552 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2553 .ifMatches(fam1, qf1, CompareOperator.LESS_OR_EQUAL, val3).build(put)); 2554 assertFalse(res.isSuccess()); 2555 assertNull(res.getResult()); 2556 2557 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val2, 2558 // succeed (value still = val2) 2559 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2560 .ifMatches(fam1, qf1, CompareOperator.LESS_OR_EQUAL, val2).build(put)); 2561 assertTrue(res.isSuccess()); 2562 assertNull(res.getResult()); 2563 2564 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val1, 2565 // succeed (now value = val3) 2566 put = new Put(row1); 2567 put.addColumn(fam1, qf1, val3); 2568 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2569 .ifMatches(fam1, qf1, CompareOperator.LESS_OR_EQUAL, val1).build(put)); 2570 assertTrue(res.isSuccess()); 2571 assertNull(res.getResult()); 2572 2573 // Test CompareOp.GREATER: original = val3, compare with val3, fail 2574 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2575 .ifMatches(fam1, qf1, CompareOperator.GREATER, val3).build(put)); 2576 assertFalse(res.isSuccess()); 2577 assertNull(res.getResult()); 2578 2579 // Test CompareOp.GREATER: original = val3, compare with val2, fail 2580 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2581 .ifMatches(fam1, qf1, CompareOperator.GREATER, val2).build(put)); 2582 assertFalse(res.isSuccess()); 2583 assertNull(res.getResult()); 2584 2585 // Test CompareOp.GREATER: original = val3, compare with val4, 2586 // succeed (now value = val2) 2587 put = new Put(row1); 2588 put.addColumn(fam1, qf1, val2); 2589 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2590 .ifMatches(fam1, qf1, CompareOperator.GREATER, val4).build(put)); 2591 assertTrue(res.isSuccess()); 2592 assertNull(res.getResult()); 2593 2594 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val1, fail 2595 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2596 .ifMatches(fam1, qf1, CompareOperator.GREATER_OR_EQUAL, val1).build(put)); 2597 assertFalse(res.isSuccess()); 2598 assertNull(res.getResult()); 2599 2600 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val2, 2601 // succeed (value still = val2) 2602 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2603 .ifMatches(fam1, qf1, CompareOperator.GREATER_OR_EQUAL, val2).build(put)); 2604 assertTrue(res.isSuccess()); 2605 assertNull(res.getResult()); 2606 2607 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val3, succeed 2608 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2609 .ifMatches(fam1, qf1, CompareOperator.GREATER_OR_EQUAL, val3).build(put)); 2610 assertTrue(res.isSuccess()); 2611 assertNull(res.getResult()); 2612 } 2613 2614 @Test 2615 public void testCheckAndPutThatPutWasWritten() throws IOException { 2616 byte[] row1 = Bytes.toBytes("row1"); 2617 byte[] fam1 = Bytes.toBytes("fam1"); 2618 byte[] fam2 = Bytes.toBytes("fam2"); 2619 byte[] qf1 = Bytes.toBytes("qualifier"); 2620 byte[] val1 = Bytes.toBytes("value1"); 2621 byte[] val2 = Bytes.toBytes("value2"); 2622 2623 byte[][] families = { fam1, fam2 }; 2624 2625 // Setting up region 2626 this.region = initHRegion(tableName, method, CONF, families); 2627 // Putting data in the key to check 2628 Put put = new Put(row1); 2629 put.addColumn(fam1, qf1, val1); 2630 region.put(put); 2631 2632 // Creating put to add 2633 long ts = EnvironmentEdgeManager.currentTime(); 2634 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2); 2635 put = new Put(row1); 2636 put.add(kv); 2637 2638 // checkAndPut with wrong value 2639 CheckAndMutateResult res = region.checkAndMutate( 2640 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(put)); 2641 assertTrue(res.isSuccess()); 2642 assertNull(res.getResult()); 2643 2644 Get get = new Get(row1); 2645 get.addColumn(fam2, qf1); 2646 Cell[] actual = region.get(get).rawCells(); 2647 2648 Cell[] expected = { kv }; 2649 2650 assertEquals(expected.length, actual.length); 2651 for (int i = 0; i < actual.length; i++) { 2652 assertEquals(expected[i], actual[i]); 2653 } 2654 } 2655 2656 @Test 2657 public void testCheckAndDeleteThatDeleteWasWritten() throws IOException { 2658 byte[] row1 = Bytes.toBytes("row1"); 2659 byte[] fam1 = Bytes.toBytes("fam1"); 2660 byte[] fam2 = Bytes.toBytes("fam2"); 2661 byte[] qf1 = Bytes.toBytes("qualifier1"); 2662 byte[] qf2 = Bytes.toBytes("qualifier2"); 2663 byte[] qf3 = Bytes.toBytes("qualifier3"); 2664 byte[] val1 = Bytes.toBytes("value1"); 2665 byte[] val2 = Bytes.toBytes("value2"); 2666 byte[] val3 = Bytes.toBytes("value3"); 2667 byte[] emptyVal = new byte[] {}; 2668 2669 byte[][] families = { fam1, fam2 }; 2670 2671 // Setting up region 2672 this.region = initHRegion(tableName, method, CONF, families); 2673 // Put content 2674 Put put = new Put(row1); 2675 put.addColumn(fam1, qf1, val1); 2676 region.put(put); 2677 Threads.sleep(2); 2678 2679 put = new Put(row1); 2680 put.addColumn(fam1, qf1, val2); 2681 put.addColumn(fam2, qf1, val3); 2682 put.addColumn(fam2, qf2, val2); 2683 put.addColumn(fam2, qf3, val1); 2684 put.addColumn(fam1, qf3, val1); 2685 region.put(put); 2686 2687 LOG.info("get={}", region.get(new Get(row1).addColumn(fam1, qf1)).toString()); 2688 2689 // Multi-column delete 2690 Delete delete = new Delete(row1); 2691 delete.addColumn(fam1, qf1); 2692 delete.addColumn(fam2, qf1); 2693 delete.addColumn(fam1, qf3); 2694 CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2695 .ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(delete)); 2696 assertTrue(res.isSuccess()); 2697 assertNull(res.getResult()); 2698 2699 Get get = new Get(row1); 2700 get.addColumn(fam1, qf1); 2701 get.addColumn(fam1, qf3); 2702 get.addColumn(fam2, qf2); 2703 Result r = region.get(get); 2704 assertEquals(2, r.size()); 2705 assertArrayEquals(val1, r.getValue(fam1, qf1)); 2706 assertArrayEquals(val2, r.getValue(fam2, qf2)); 2707 2708 // Family delete 2709 delete = new Delete(row1); 2710 delete.addFamily(fam2); 2711 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2712 .ifMatches(fam2, qf1, CompareOperator.EQUAL, emptyVal).build(delete)); 2713 assertTrue(res.isSuccess()); 2714 assertNull(res.getResult()); 2715 2716 get = new Get(row1); 2717 r = region.get(get); 2718 assertEquals(1, r.size()); 2719 assertArrayEquals(val1, r.getValue(fam1, qf1)); 2720 2721 // Row delete 2722 delete = new Delete(row1); 2723 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2724 .ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(delete)); 2725 assertTrue(res.isSuccess()); 2726 assertNull(res.getResult()); 2727 2728 get = new Get(row1); 2729 r = region.get(get); 2730 assertEquals(0, r.size()); 2731 } 2732 2733 @Test 2734 public void testCheckAndMutateWithFilters() throws Throwable { 2735 final byte[] FAMILY = Bytes.toBytes("fam"); 2736 2737 // Setting up region 2738 this.region = initHRegion(tableName, method, CONF, FAMILY); 2739 2740 // Put one row 2741 Put put = new Put(row); 2742 put.addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")); 2743 put.addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")); 2744 put.addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")); 2745 region.put(put); 2746 2747 // Put with success 2748 CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2749 .ifMatches(new FilterList( 2750 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2751 Bytes.toBytes("a")), 2752 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2753 Bytes.toBytes("b")))) 2754 .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d")))); 2755 assertTrue(res.isSuccess()); 2756 assertNull(res.getResult()); 2757 2758 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))); 2759 assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D")))); 2760 2761 // Put with failure 2762 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2763 .ifMatches(new FilterList( 2764 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2765 Bytes.toBytes("a")), 2766 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2767 Bytes.toBytes("c")))) 2768 .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e")))); 2769 assertFalse(res.isSuccess()); 2770 assertNull(res.getResult()); 2771 2772 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E"))).isEmpty()); 2773 2774 // Delete with success 2775 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2776 .ifMatches(new FilterList( 2777 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2778 Bytes.toBytes("a")), 2779 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2780 Bytes.toBytes("b")))) 2781 .build(new Delete(row).addColumns(FAMILY, Bytes.toBytes("D")))); 2782 assertTrue(res.isSuccess()); 2783 assertNull(res.getResult()); 2784 2785 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))).isEmpty()); 2786 2787 // Mutate with success 2788 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2789 .ifMatches(new FilterList( 2790 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2791 Bytes.toBytes("a")), 2792 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2793 Bytes.toBytes("b")))) 2794 .build(new RowMutations(row) 2795 .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e"))) 2796 .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A"))))); 2797 assertTrue(res.isSuccess()); 2798 assertNull(res.getResult()); 2799 2800 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E"))); 2801 assertEquals("e", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("E")))); 2802 2803 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty()); 2804 } 2805 2806 @Test 2807 public void testCheckAndMutateWithFiltersAndTimeRange() throws Throwable { 2808 final byte[] FAMILY = Bytes.toBytes("fam"); 2809 2810 // Setting up region 2811 this.region = initHRegion(tableName, method, CONF, FAMILY); 2812 2813 // Put with specifying the timestamp 2814 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), 100, Bytes.toBytes("a"))); 2815 2816 // Put with success 2817 CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2818 .ifMatches(new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2819 Bytes.toBytes("a"))) 2820 .timeRange(TimeRange.between(0, 101)) 2821 .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")))); 2822 assertTrue(res.isSuccess()); 2823 assertNull(res.getResult()); 2824 2825 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2826 assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2827 2828 // Put with failure 2829 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2830 .ifMatches(new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2831 Bytes.toBytes("a"))) 2832 .timeRange(TimeRange.between(0, 100)) 2833 .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")))); 2834 assertFalse(res.isSuccess()); 2835 assertNull(res.getResult()); 2836 2837 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("C"))).isEmpty()); 2838 2839 // RowMutations with success 2840 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2841 .ifMatches(new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2842 Bytes.toBytes("a"))) 2843 .timeRange(TimeRange.between(0, 101)) 2844 .build(new RowMutations(row) 2845 .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d"))) 2846 .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A"))))); 2847 assertTrue(res.isSuccess()); 2848 assertNull(res.getResult()); 2849 2850 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))); 2851 assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D")))); 2852 2853 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty()); 2854 } 2855 2856 @Test 2857 public void testCheckAndIncrement() throws Throwable { 2858 final byte[] FAMILY = Bytes.toBytes("fam"); 2859 2860 // Setting up region 2861 this.region = initHRegion(tableName, method, CONF, FAMILY); 2862 2863 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a"))); 2864 2865 // CheckAndIncrement with correct value 2866 CheckAndMutateResult res = region.checkAndMutate( 2867 CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")) 2868 .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 1))); 2869 assertTrue(res.isSuccess()); 2870 assertEquals(1, Bytes.toLong(res.getResult().getValue(FAMILY, Bytes.toBytes("B")))); 2871 2872 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2873 assertEquals(1, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B")))); 2874 2875 // CheckAndIncrement with wrong value 2876 res = region.checkAndMutate( 2877 CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("b")) 2878 .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 1))); 2879 assertFalse(res.isSuccess()); 2880 assertNull(res.getResult()); 2881 2882 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2883 assertEquals(1, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B")))); 2884 2885 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c"))); 2886 2887 // CheckAndIncrement with a filter and correct value 2888 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2889 .ifMatches(new FilterList( 2890 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2891 Bytes.toBytes("a")), 2892 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL, 2893 Bytes.toBytes("c")))) 2894 .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 2))); 2895 assertTrue(res.isSuccess()); 2896 assertEquals(3, Bytes.toLong(res.getResult().getValue(FAMILY, Bytes.toBytes("B")))); 2897 2898 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2899 assertEquals(3, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B")))); 2900 2901 // CheckAndIncrement with a filter and correct value 2902 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2903 .ifMatches(new FilterList( 2904 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2905 Bytes.toBytes("b")), 2906 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL, 2907 Bytes.toBytes("d")))) 2908 .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 2))); 2909 assertFalse(res.isSuccess()); 2910 assertNull(res.getResult()); 2911 2912 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2913 assertEquals(3, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B")))); 2914 } 2915 2916 @Test 2917 public void testCheckAndAppend() throws Throwable { 2918 final byte[] FAMILY = Bytes.toBytes("fam"); 2919 2920 // Setting up region 2921 this.region = initHRegion(tableName, method, CONF, FAMILY); 2922 2923 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a"))); 2924 2925 // CheckAndAppend with correct value 2926 CheckAndMutateResult res = region.checkAndMutate( 2927 CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")) 2928 .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")))); 2929 assertTrue(res.isSuccess()); 2930 assertEquals("b", Bytes.toString(res.getResult().getValue(FAMILY, Bytes.toBytes("B")))); 2931 2932 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2933 assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2934 2935 // CheckAndAppend with wrong value 2936 res = region.checkAndMutate( 2937 CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("b")) 2938 .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")))); 2939 assertFalse(res.isSuccess()); 2940 assertNull(res.getResult()); 2941 2942 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2943 assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2944 2945 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c"))); 2946 2947 // CheckAndAppend with a filter and correct value 2948 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2949 .ifMatches(new FilterList( 2950 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2951 Bytes.toBytes("a")), 2952 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL, 2953 Bytes.toBytes("c")))) 2954 .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("bb")))); 2955 assertTrue(res.isSuccess()); 2956 assertEquals("bbb", Bytes.toString(res.getResult().getValue(FAMILY, Bytes.toBytes("B")))); 2957 2958 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2959 assertEquals("bbb", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2960 2961 // CheckAndAppend with a filter and wrong value 2962 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2963 .ifMatches(new FilterList( 2964 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2965 Bytes.toBytes("b")), 2966 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL, 2967 Bytes.toBytes("d")))) 2968 .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("bb")))); 2969 assertFalse(res.isSuccess()); 2970 assertNull(res.getResult()); 2971 2972 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2973 assertEquals("bbb", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2974 } 2975 2976 @Test 2977 public void testCheckAndIncrementAndAppend() throws Throwable { 2978 // Setting up region 2979 this.region = initHRegion(tableName, method, CONF, fam1); 2980 2981 // CheckAndMutate with Increment and Append 2982 CheckAndMutate checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, qual) 2983 .build(new RowMutations(row).add((Mutation) new Increment(row).addColumn(fam1, qual1, 1L)) 2984 .add((Mutation) new Append(row).addColumn(fam1, qual2, Bytes.toBytes("a")))); 2985 2986 CheckAndMutateResult result = region.checkAndMutate(checkAndMutate); 2987 assertTrue(result.isSuccess()); 2988 assertEquals(1L, Bytes.toLong(result.getResult().getValue(fam1, qual1))); 2989 assertEquals("a", Bytes.toString(result.getResult().getValue(fam1, qual2))); 2990 2991 Result r = region.get(new Get(row)); 2992 assertEquals(1L, Bytes.toLong(r.getValue(fam1, qual1))); 2993 assertEquals("a", Bytes.toString(r.getValue(fam1, qual2))); 2994 2995 // Set return results to false 2996 checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, qual) 2997 .build(new RowMutations(row) 2998 .add((Mutation) new Increment(row).addColumn(fam1, qual1, 1L).setReturnResults(false)) 2999 .add((Mutation) new Append(row).addColumn(fam1, qual2, Bytes.toBytes("a")) 3000 .setReturnResults(false))); 3001 3002 result = region.checkAndMutate(checkAndMutate); 3003 assertTrue(result.isSuccess()); 3004 assertNull(result.getResult().getValue(fam1, qual1)); 3005 assertNull(result.getResult().getValue(fam1, qual2)); 3006 3007 r = region.get(new Get(row)); 3008 assertEquals(2L, Bytes.toLong(r.getValue(fam1, qual1))); 3009 assertEquals("aa", Bytes.toString(r.getValue(fam1, qual2))); 3010 3011 checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, qual) 3012 .build(new RowMutations(row).add((Mutation) new Increment(row).addColumn(fam1, qual1, 1L)) 3013 .add((Mutation) new Append(row).addColumn(fam1, qual2, Bytes.toBytes("a")) 3014 .setReturnResults(false))); 3015 3016 result = region.checkAndMutate(checkAndMutate); 3017 assertTrue(result.isSuccess()); 3018 assertEquals(3L, Bytes.toLong(result.getResult().getValue(fam1, qual1))); 3019 assertNull(result.getResult().getValue(fam1, qual2)); 3020 3021 r = region.get(new Get(row)); 3022 assertEquals(3L, Bytes.toLong(r.getValue(fam1, qual1))); 3023 assertEquals("aaa", Bytes.toString(r.getValue(fam1, qual2))); 3024 } 3025 3026 @Test 3027 public void testCheckAndRowMutations() throws Throwable { 3028 final byte[] row = Bytes.toBytes("row"); 3029 final byte[] q1 = Bytes.toBytes("q1"); 3030 final byte[] q2 = Bytes.toBytes("q2"); 3031 final byte[] q3 = Bytes.toBytes("q3"); 3032 final byte[] q4 = Bytes.toBytes("q4"); 3033 final String v1 = "v1"; 3034 3035 region = initHRegion(tableName, method, CONF, fam1); 3036 3037 // Initial values 3038 region 3039 .batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q2, Bytes.toBytes("toBeDeleted")), 3040 new Put(row).addColumn(fam1, q3, Bytes.toBytes(5L)), 3041 new Put(row).addColumn(fam1, q4, Bytes.toBytes("a")), }); 3042 3043 // Do CheckAndRowMutations 3044 CheckAndMutate checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, q1).build( 3045 new RowMutations(row).add(Arrays.asList(new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1)), 3046 new Delete(row).addColumns(fam1, q2), new Increment(row).addColumn(fam1, q3, 1), 3047 new Append(row).addColumn(fam1, q4, Bytes.toBytes("b"))))); 3048 3049 CheckAndMutateResult result = region.checkAndMutate(checkAndMutate); 3050 assertTrue(result.isSuccess()); 3051 assertEquals(6L, Bytes.toLong(result.getResult().getValue(fam1, q3))); 3052 assertEquals("ab", Bytes.toString(result.getResult().getValue(fam1, q4))); 3053 3054 // Verify the value 3055 Result r = region.get(new Get(row)); 3056 assertEquals(v1, Bytes.toString(r.getValue(fam1, q1))); 3057 assertNull(r.getValue(fam1, q2)); 3058 assertEquals(6L, Bytes.toLong(r.getValue(fam1, q3))); 3059 assertEquals("ab", Bytes.toString(r.getValue(fam1, q4))); 3060 3061 // Do CheckAndRowMutations again 3062 checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, q1) 3063 .build(new RowMutations(row).add(Arrays.asList(new Delete(row).addColumns(fam1, q1), 3064 new Put(row).addColumn(fam1, q2, Bytes.toBytes(v1)), 3065 new Increment(row).addColumn(fam1, q3, 1), 3066 new Append(row).addColumn(fam1, q4, Bytes.toBytes("b"))))); 3067 3068 result = region.checkAndMutate(checkAndMutate); 3069 assertFalse(result.isSuccess()); 3070 assertNull(result.getResult()); 3071 3072 // Verify the value 3073 r = region.get(new Get(row)); 3074 assertEquals(v1, Bytes.toString(r.getValue(fam1, q1))); 3075 assertNull(r.getValue(fam1, q2)); 3076 assertEquals(6L, Bytes.toLong(r.getValue(fam1, q3))); 3077 assertEquals("ab", Bytes.toString(r.getValue(fam1, q4))); 3078 } 3079 3080 // //////////////////////////////////////////////////////////////////////////// 3081 // Delete tests 3082 // //////////////////////////////////////////////////////////////////////////// 3083 @Test 3084 public void testDelete_multiDeleteColumn() throws IOException { 3085 byte[] row1 = Bytes.toBytes("row1"); 3086 byte[] fam1 = Bytes.toBytes("fam1"); 3087 byte[] qual = Bytes.toBytes("qualifier"); 3088 byte[] value = Bytes.toBytes("value"); 3089 3090 Put put = new Put(row1); 3091 put.addColumn(fam1, qual, 1, value); 3092 put.addColumn(fam1, qual, 2, value); 3093 3094 this.region = initHRegion(tableName, method, CONF, fam1); 3095 region.put(put); 3096 3097 // We do support deleting more than 1 'latest' version 3098 Delete delete = new Delete(row1); 3099 delete.addColumn(fam1, qual); 3100 delete.addColumn(fam1, qual); 3101 region.delete(delete); 3102 3103 Get get = new Get(row1); 3104 get.addFamily(fam1); 3105 Result r = region.get(get); 3106 assertEquals(0, r.size()); 3107 } 3108 3109 @Test 3110 public void testDelete_CheckFamily() throws IOException { 3111 byte[] row1 = Bytes.toBytes("row1"); 3112 byte[] fam1 = Bytes.toBytes("fam1"); 3113 byte[] fam2 = Bytes.toBytes("fam2"); 3114 byte[] fam3 = Bytes.toBytes("fam3"); 3115 byte[] fam4 = Bytes.toBytes("fam4"); 3116 3117 // Setting up region 3118 this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3); 3119 List<Cell> kvs = new ArrayList<>(); 3120 kvs.add(new KeyValue(row1, fam4, null, null)); 3121 3122 byte[] forUnitTestsOnly = Bytes.toBytes("ForUnitTestsOnly"); 3123 3124 // testing existing family 3125 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<>(Bytes.BYTES_COMPARATOR); 3126 deleteMap.put(fam2, kvs); 3127 region.delete(new Delete(forUnitTestsOnly, HConstants.LATEST_TIMESTAMP, deleteMap)); 3128 3129 // testing non existing family 3130 NavigableMap<byte[], List<Cell>> deleteMap2 = new TreeMap<>(Bytes.BYTES_COMPARATOR); 3131 deleteMap2.put(fam4, kvs); 3132 assertThrows("Family " + Bytes.toString(fam4) + " does exist", 3133 NoSuchColumnFamilyException.class, 3134 () -> region.delete(new Delete(forUnitTestsOnly, HConstants.LATEST_TIMESTAMP, deleteMap2))); 3135 } 3136 3137 @Test 3138 public void testDelete_mixed() throws IOException, InterruptedException { 3139 byte[] fam = Bytes.toBytes("info"); 3140 byte[][] families = { fam }; 3141 this.region = initHRegion(tableName, method, CONF, families); 3142 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge()); 3143 3144 byte[] row = Bytes.toBytes("table_name"); 3145 // column names 3146 byte[] serverinfo = Bytes.toBytes("serverinfo"); 3147 byte[] splitA = Bytes.toBytes("splitA"); 3148 byte[] splitB = Bytes.toBytes("splitB"); 3149 3150 // add some data: 3151 Put put = new Put(row); 3152 put.addColumn(fam, splitA, Bytes.toBytes("reference_A")); 3153 region.put(put); 3154 3155 put = new Put(row); 3156 put.addColumn(fam, splitB, Bytes.toBytes("reference_B")); 3157 region.put(put); 3158 3159 put = new Put(row); 3160 put.addColumn(fam, serverinfo, Bytes.toBytes("ip_address")); 3161 region.put(put); 3162 3163 // ok now delete a split: 3164 Delete delete = new Delete(row); 3165 delete.addColumns(fam, splitA); 3166 region.delete(delete); 3167 3168 // assert some things: 3169 Get get = new Get(row).addColumn(fam, serverinfo); 3170 Result result = region.get(get); 3171 assertEquals(1, result.size()); 3172 3173 get = new Get(row).addColumn(fam, splitA); 3174 result = region.get(get); 3175 assertEquals(0, result.size()); 3176 3177 get = new Get(row).addColumn(fam, splitB); 3178 result = region.get(get); 3179 assertEquals(1, result.size()); 3180 3181 // Assert that after a delete, I can put. 3182 put = new Put(row); 3183 put.addColumn(fam, splitA, Bytes.toBytes("reference_A")); 3184 region.put(put); 3185 get = new Get(row); 3186 result = region.get(get); 3187 assertEquals(3, result.size()); 3188 3189 // Now delete all... then test I can add stuff back 3190 delete = new Delete(row); 3191 region.delete(delete); 3192 assertEquals(0, region.get(get).size()); 3193 3194 region.put(new Put(row).addColumn(fam, splitA, Bytes.toBytes("reference_A"))); 3195 result = region.get(get); 3196 assertEquals(1, result.size()); 3197 } 3198 3199 @Test 3200 public void testDeleteRowWithFutureTs() throws IOException { 3201 byte[] fam = Bytes.toBytes("info"); 3202 byte[][] families = { fam }; 3203 this.region = initHRegion(tableName, method, CONF, families); 3204 byte[] row = Bytes.toBytes("table_name"); 3205 // column names 3206 byte[] serverinfo = Bytes.toBytes("serverinfo"); 3207 3208 // add data in the far future 3209 Put put = new Put(row); 3210 put.addColumn(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value")); 3211 region.put(put); 3212 3213 // now delete something in the present 3214 Delete delete = new Delete(row); 3215 region.delete(delete); 3216 3217 // make sure we still see our data 3218 Get get = new Get(row).addColumn(fam, serverinfo); 3219 Result result = region.get(get); 3220 assertEquals(1, result.size()); 3221 3222 // delete the future row 3223 delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3); 3224 region.delete(delete); 3225 3226 // make sure it is gone 3227 get = new Get(row).addColumn(fam, serverinfo); 3228 result = region.get(get); 3229 assertEquals(0, result.size()); 3230 } 3231 3232 /** 3233 * Tests that the special LATEST_TIMESTAMP option for puts gets replaced by the actual timestamp 3234 */ 3235 @Test 3236 public void testPutWithLatestTS() throws IOException { 3237 byte[] fam = Bytes.toBytes("info"); 3238 byte[][] families = { fam }; 3239 this.region = initHRegion(tableName, method, CONF, families); 3240 byte[] row = Bytes.toBytes("row1"); 3241 // column names 3242 byte[] qual = Bytes.toBytes("qual"); 3243 3244 // add data with LATEST_TIMESTAMP, put without WAL 3245 Put put = new Put(row); 3246 put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value")); 3247 region.put(put); 3248 3249 // Make sure it shows up with an actual timestamp 3250 Get get = new Get(row).addColumn(fam, qual); 3251 Result result = region.get(get); 3252 assertEquals(1, result.size()); 3253 Cell kv = result.rawCells()[0]; 3254 LOG.info("Got: " + kv); 3255 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp", 3256 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP); 3257 3258 // Check same with WAL enabled (historically these took different 3259 // code paths, so check both) 3260 row = Bytes.toBytes("row2"); 3261 put = new Put(row); 3262 put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value")); 3263 region.put(put); 3264 3265 // Make sure it shows up with an actual timestamp 3266 get = new Get(row).addColumn(fam, qual); 3267 result = region.get(get); 3268 assertEquals(1, result.size()); 3269 kv = result.rawCells()[0]; 3270 LOG.info("Got: " + kv); 3271 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp", 3272 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP); 3273 } 3274 3275 /** 3276 * Tests that there is server-side filtering for invalid timestamp upper bound. Note that the 3277 * timestamp lower bound is automatically handled for us by the TTL field. 3278 */ 3279 @Test 3280 public void testPutWithTsSlop() throws IOException { 3281 byte[] fam = Bytes.toBytes("info"); 3282 byte[][] families = { fam }; 3283 3284 // add data with a timestamp that is too recent for range. Ensure assert 3285 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000); 3286 this.region = initHRegion(tableName, method, CONF, families); 3287 boolean caughtExcep = false; 3288 try { 3289 // no TS specified == use latest. should not error 3290 region.put(new Put(row).addColumn(fam, Bytes.toBytes("qual"), Bytes.toBytes("value"))); 3291 // TS out of range. should error 3292 region.put(new Put(row).addColumn(fam, Bytes.toBytes("qual"), 3293 EnvironmentEdgeManager.currentTime() + 2000, Bytes.toBytes("value"))); 3294 fail("Expected IOE for TS out of configured timerange"); 3295 } catch (FailedSanityCheckException ioe) { 3296 LOG.debug("Received expected exception", ioe); 3297 caughtExcep = true; 3298 } 3299 assertTrue("Should catch FailedSanityCheckException", caughtExcep); 3300 } 3301 3302 @Test 3303 public void testScanner_DeleteOneFamilyNotAnother() throws IOException { 3304 byte[] fam1 = Bytes.toBytes("columnA"); 3305 byte[] fam2 = Bytes.toBytes("columnB"); 3306 this.region = initHRegion(tableName, method, CONF, fam1, fam2); 3307 byte[] rowA = Bytes.toBytes("rowA"); 3308 byte[] rowB = Bytes.toBytes("rowB"); 3309 3310 byte[] value = Bytes.toBytes("value"); 3311 3312 Delete delete = new Delete(rowA); 3313 delete.addFamily(fam1); 3314 3315 region.delete(delete); 3316 3317 // now create data. 3318 Put put = new Put(rowA); 3319 put.addColumn(fam2, null, value); 3320 region.put(put); 3321 3322 put = new Put(rowB); 3323 put.addColumn(fam1, null, value); 3324 put.addColumn(fam2, null, value); 3325 region.put(put); 3326 3327 Scan scan = new Scan(); 3328 scan.addFamily(fam1).addFamily(fam2); 3329 InternalScanner s = region.getScanner(scan); 3330 List<Cell> results = new ArrayList<>(); 3331 s.next(results); 3332 assertTrue(CellUtil.matchingRows(results.get(0), rowA)); 3333 3334 results.clear(); 3335 s.next(results); 3336 assertTrue(CellUtil.matchingRows(results.get(0), rowB)); 3337 } 3338 3339 @Test 3340 public void testDataInMemoryWithoutWAL() throws IOException { 3341 FileSystem fs = FileSystem.get(CONF); 3342 Path rootDir = new Path(dir + "testDataInMemoryWithoutWAL"); 3343 FSHLog hLog = new FSHLog(fs, rootDir, "testDataInMemoryWithoutWAL", CONF); 3344 hLog.init(); 3345 // This chunk creation is done throughout the code base. Do we want to move it into core? 3346 // It is missing from this test. W/o it we NPE. 3347 region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, hLog, 3348 COLUMN_FAMILY_BYTES); 3349 3350 Cell originalCell = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row) 3351 .setFamily(COLUMN_FAMILY_BYTES).setQualifier(qual1) 3352 .setTimestamp(EnvironmentEdgeManager.currentTime()).setType(KeyValue.Type.Put.getCode()) 3353 .setValue(value1).build(); 3354 final long originalSize = originalCell.getSerializedSize(); 3355 3356 Cell addCell = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row) 3357 .setFamily(COLUMN_FAMILY_BYTES).setQualifier(qual1) 3358 .setTimestamp(EnvironmentEdgeManager.currentTime()).setType(KeyValue.Type.Put.getCode()) 3359 .setValue(Bytes.toBytes("xxxxxxxxxx")).build(); 3360 final long addSize = addCell.getSerializedSize(); 3361 3362 LOG.info("originalSize:" + originalSize + ", addSize:" + addSize); 3363 // start test. We expect that the addPut's durability will be replaced 3364 // by originalPut's durability. 3365 3366 // case 1: 3367 testDataInMemoryWithoutWAL(region, 3368 new Put(row).add(originalCell).setDurability(Durability.SKIP_WAL), 3369 new Put(row).add(addCell).setDurability(Durability.SKIP_WAL), originalSize + addSize); 3370 3371 // case 2: 3372 testDataInMemoryWithoutWAL(region, 3373 new Put(row).add(originalCell).setDurability(Durability.SKIP_WAL), 3374 new Put(row).add(addCell).setDurability(Durability.SYNC_WAL), originalSize + addSize); 3375 3376 // case 3: 3377 testDataInMemoryWithoutWAL(region, 3378 new Put(row).add(originalCell).setDurability(Durability.SYNC_WAL), 3379 new Put(row).add(addCell).setDurability(Durability.SKIP_WAL), 0); 3380 3381 // case 4: 3382 testDataInMemoryWithoutWAL(region, 3383 new Put(row).add(originalCell).setDurability(Durability.SYNC_WAL), 3384 new Put(row).add(addCell).setDurability(Durability.SYNC_WAL), 0); 3385 } 3386 3387 private static void testDataInMemoryWithoutWAL(HRegion region, Put originalPut, final Put addPut, 3388 long delta) throws IOException { 3389 final long initSize = region.getDataInMemoryWithoutWAL(); 3390 // save normalCPHost and replaced by mockedCPHost 3391 RegionCoprocessorHost normalCPHost = region.getCoprocessorHost(); 3392 RegionCoprocessorHost mockedCPHost = Mockito.mock(RegionCoprocessorHost.class); 3393 // Because the preBatchMutate returns void, we can't do usual Mockito when...then form. Must 3394 // do below format (from Mockito doc). 3395 Mockito.doAnswer(new Answer<Void>() { 3396 @Override 3397 public Void answer(InvocationOnMock invocation) throws Throwable { 3398 MiniBatchOperationInProgress<Mutation> mb = invocation.getArgument(0); 3399 mb.addOperationsFromCP(0, new Mutation[] { addPut }); 3400 return null; 3401 } 3402 }).when(mockedCPHost).preBatchMutate(Mockito.isA(MiniBatchOperationInProgress.class)); 3403 ColumnFamilyDescriptorBuilder builder = 3404 ColumnFamilyDescriptorBuilder.newBuilder(COLUMN_FAMILY_BYTES); 3405 ScanInfo info = new ScanInfo(CONF, builder.build(), Long.MAX_VALUE, Long.MAX_VALUE, 3406 region.getCellComparator()); 3407 Mockito.when(mockedCPHost.preFlushScannerOpen(Mockito.any(HStore.class), Mockito.any())) 3408 .thenReturn(info); 3409 Mockito 3410 .when(mockedCPHost.preFlush(Mockito.any(), Mockito.any(StoreScanner.class), Mockito.any())) 3411 .thenAnswer(i -> i.getArgument(1)); 3412 region.setCoprocessorHost(mockedCPHost); 3413 3414 region.put(originalPut); 3415 region.setCoprocessorHost(normalCPHost); 3416 final long finalSize = region.getDataInMemoryWithoutWAL(); 3417 assertEquals("finalSize:" + finalSize + ", initSize:" + initSize + ", delta:" + delta, 3418 finalSize, initSize + delta); 3419 } 3420 3421 @Test 3422 public void testDeleteColumns_PostInsert() throws IOException, InterruptedException { 3423 Delete delete = new Delete(row); 3424 delete.addColumns(fam1, qual1); 3425 doTestDelete_AndPostInsert(delete); 3426 } 3427 3428 @Test 3429 public void testaddFamily_PostInsert() throws IOException, InterruptedException { 3430 Delete delete = new Delete(row); 3431 delete.addFamily(fam1); 3432 doTestDelete_AndPostInsert(delete); 3433 } 3434 3435 public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException { 3436 this.region = initHRegion(tableName, method, CONF, fam1); 3437 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge()); 3438 Put put = new Put(row); 3439 put.addColumn(fam1, qual1, value1); 3440 region.put(put); 3441 3442 // now delete the value: 3443 region.delete(delete); 3444 3445 // ok put data: 3446 put = new Put(row); 3447 put.addColumn(fam1, qual1, value2); 3448 region.put(put); 3449 3450 // ok get: 3451 Get get = new Get(row); 3452 get.addColumn(fam1, qual1); 3453 3454 Result r = region.get(get); 3455 assertEquals(1, r.size()); 3456 assertArrayEquals(value2, r.getValue(fam1, qual1)); 3457 3458 // next: 3459 Scan scan = new Scan().withStartRow(row); 3460 scan.addColumn(fam1, qual1); 3461 InternalScanner s = region.getScanner(scan); 3462 3463 List<Cell> results = new ArrayList<>(); 3464 assertEquals(false, s.next(results)); 3465 assertEquals(1, results.size()); 3466 Cell kv = results.get(0); 3467 3468 assertArrayEquals(value2, CellUtil.cloneValue(kv)); 3469 assertArrayEquals(fam1, CellUtil.cloneFamily(kv)); 3470 assertArrayEquals(qual1, CellUtil.cloneQualifier(kv)); 3471 assertArrayEquals(row, CellUtil.cloneRow(kv)); 3472 } 3473 3474 @Test 3475 public void testDelete_CheckTimestampUpdated() throws IOException { 3476 byte[] row1 = Bytes.toBytes("row1"); 3477 byte[] col1 = Bytes.toBytes("col1"); 3478 byte[] col2 = Bytes.toBytes("col2"); 3479 byte[] col3 = Bytes.toBytes("col3"); 3480 3481 byte[] forUnitTestsOnly = Bytes.toBytes("ForUnitTestsOnly"); 3482 3483 // Setting up region 3484 this.region = initHRegion(tableName, method, CONF, fam1); 3485 // Building checkerList 3486 List<Cell> kvs = new ArrayList<>(); 3487 kvs.add(new KeyValue(row1, fam1, col1, null)); 3488 kvs.add(new KeyValue(row1, fam1, col2, null)); 3489 kvs.add(new KeyValue(row1, fam1, col3, null)); 3490 3491 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<>(Bytes.BYTES_COMPARATOR); 3492 deleteMap.put(fam1, kvs); 3493 region.delete(new Delete(forUnitTestsOnly, HConstants.LATEST_TIMESTAMP, deleteMap)); 3494 3495 // extract the key values out the memstore: 3496 // This is kinda hacky, but better than nothing... 3497 long now = EnvironmentEdgeManager.currentTime(); 3498 AbstractMemStore memstore = (AbstractMemStore) region.getStore(fam1).memstore; 3499 Cell firstCell = memstore.getActive().first(); 3500 assertTrue(firstCell.getTimestamp() <= now); 3501 now = firstCell.getTimestamp(); 3502 for (Cell cell : memstore.getActive().getCellSet()) { 3503 assertTrue(cell.getTimestamp() <= now); 3504 now = cell.getTimestamp(); 3505 } 3506 } 3507 3508 // //////////////////////////////////////////////////////////////////////////// 3509 // Get tests 3510 // //////////////////////////////////////////////////////////////////////////// 3511 @Test 3512 public void testGet_FamilyChecker() throws IOException { 3513 byte[] row1 = Bytes.toBytes("row1"); 3514 byte[] fam1 = Bytes.toBytes("fam1"); 3515 byte[] fam2 = Bytes.toBytes("False"); 3516 byte[] col1 = Bytes.toBytes("col1"); 3517 3518 // Setting up region 3519 this.region = initHRegion(tableName, method, CONF, fam1); 3520 Get get = new Get(row1); 3521 get.addColumn(fam2, col1); 3522 3523 // Test 3524 try { 3525 region.get(get); 3526 fail("Expecting DoNotRetryIOException in get but did not get any"); 3527 } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) { 3528 LOG.info("Got expected DoNotRetryIOException successfully"); 3529 } 3530 } 3531 3532 @Test 3533 public void testGet_Basic() throws IOException { 3534 byte[] row1 = Bytes.toBytes("row1"); 3535 byte[] fam1 = Bytes.toBytes("fam1"); 3536 byte[] col1 = Bytes.toBytes("col1"); 3537 byte[] col2 = Bytes.toBytes("col2"); 3538 byte[] col3 = Bytes.toBytes("col3"); 3539 byte[] col4 = Bytes.toBytes("col4"); 3540 byte[] col5 = Bytes.toBytes("col5"); 3541 3542 // Setting up region 3543 this.region = initHRegion(tableName, method, CONF, fam1); 3544 // Add to memstore 3545 Put put = new Put(row1); 3546 put.addColumn(fam1, col1, null); 3547 put.addColumn(fam1, col2, null); 3548 put.addColumn(fam1, col3, null); 3549 put.addColumn(fam1, col4, null); 3550 put.addColumn(fam1, col5, null); 3551 region.put(put); 3552 3553 Get get = new Get(row1); 3554 get.addColumn(fam1, col2); 3555 get.addColumn(fam1, col4); 3556 // Expected result 3557 KeyValue kv1 = new KeyValue(row1, fam1, col2); 3558 KeyValue kv2 = new KeyValue(row1, fam1, col4); 3559 KeyValue[] expected = { kv1, kv2 }; 3560 3561 // Test 3562 Result res = region.get(get); 3563 assertEquals(expected.length, res.size()); 3564 for (int i = 0; i < res.size(); i++) { 3565 assertTrue(CellUtil.matchingRows(expected[i], res.rawCells()[i])); 3566 assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i])); 3567 assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i])); 3568 } 3569 3570 // Test using a filter on a Get 3571 Get g = new Get(row1); 3572 final int count = 2; 3573 g.setFilter(new ColumnCountGetFilter(count)); 3574 res = region.get(g); 3575 assertEquals(count, res.size()); 3576 } 3577 3578 @Test 3579 public void testGet_Empty() throws IOException { 3580 byte[] row = Bytes.toBytes("row"); 3581 byte[] fam = Bytes.toBytes("fam"); 3582 3583 this.region = initHRegion(tableName, method, CONF, fam); 3584 Get get = new Get(row); 3585 get.addFamily(fam); 3586 Result r = region.get(get); 3587 3588 assertTrue(r.isEmpty()); 3589 } 3590 3591 @Test 3592 public void testGetWithFilter() throws IOException, InterruptedException { 3593 byte[] row1 = Bytes.toBytes("row1"); 3594 byte[] fam1 = Bytes.toBytes("fam1"); 3595 byte[] col1 = Bytes.toBytes("col1"); 3596 byte[] value1 = Bytes.toBytes("value1"); 3597 byte[] value2 = Bytes.toBytes("value2"); 3598 3599 final int maxVersions = 3; 3600 TableDescriptor tableDescriptor = 3601 TableDescriptorBuilder.newBuilder(TableName.valueOf("testFilterAndColumnTracker")) 3602 .setColumnFamily( 3603 ColumnFamilyDescriptorBuilder.newBuilder(fam1).setMaxVersions(maxVersions).build()) 3604 .build(); 3605 ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null, 3606 MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT); 3607 RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 3608 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + ".log"); 3609 final WAL wal = HBaseTestingUtil.createWal(TEST_UTIL.getConfiguration(), logDir, info); 3610 this.region = TEST_UTIL.createLocalHRegion(info, CONF, tableDescriptor, wal); 3611 3612 // Put 4 version to memstore 3613 long ts = 0; 3614 Put put = new Put(row1, ts); 3615 put.addColumn(fam1, col1, value1); 3616 region.put(put); 3617 put = new Put(row1, ts + 1); 3618 put.addColumn(fam1, col1, Bytes.toBytes("filter1")); 3619 region.put(put); 3620 put = new Put(row1, ts + 2); 3621 put.addColumn(fam1, col1, Bytes.toBytes("filter2")); 3622 region.put(put); 3623 put = new Put(row1, ts + 3); 3624 put.addColumn(fam1, col1, value2); 3625 region.put(put); 3626 3627 Get get = new Get(row1); 3628 get.readAllVersions(); 3629 Result res = region.get(get); 3630 // Get 3 versions, the oldest version has gone from user view 3631 assertEquals(maxVersions, res.size()); 3632 3633 get.setFilter(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("value"))); 3634 res = region.get(get); 3635 // When use value filter, the oldest version should still gone from user view and it 3636 // should only return one key vaule 3637 assertEquals(1, res.size()); 3638 assertTrue(CellUtil.matchingValue(new KeyValue(row1, fam1, col1, value2), res.rawCells()[0])); 3639 assertEquals(ts + 3, res.rawCells()[0].getTimestamp()); 3640 3641 region.flush(true); 3642 region.compact(true); 3643 Thread.sleep(1000); 3644 res = region.get(get); 3645 // After flush and compact, the result should be consistent with previous result 3646 assertEquals(1, res.size()); 3647 assertTrue(CellUtil.matchingValue(new KeyValue(row1, fam1, col1, value2), res.rawCells()[0])); 3648 } 3649 3650 // //////////////////////////////////////////////////////////////////////////// 3651 // Scanner tests 3652 // //////////////////////////////////////////////////////////////////////////// 3653 @Test 3654 public void testGetScanner_WithOkFamilies() throws IOException { 3655 byte[] fam1 = Bytes.toBytes("fam1"); 3656 byte[] fam2 = Bytes.toBytes("fam2"); 3657 3658 byte[][] families = { fam1, fam2 }; 3659 3660 // Setting up region 3661 this.region = initHRegion(tableName, method, CONF, families); 3662 Scan scan = new Scan(); 3663 scan.addFamily(fam1); 3664 scan.addFamily(fam2); 3665 try { 3666 region.getScanner(scan); 3667 } catch (Exception e) { 3668 assertTrue("Families could not be found in Region", false); 3669 } 3670 } 3671 3672 @Test 3673 public void testGetScanner_WithNotOkFamilies() throws IOException { 3674 byte[] fam1 = Bytes.toBytes("fam1"); 3675 byte[] fam2 = Bytes.toBytes("fam2"); 3676 3677 byte[][] families = { fam1 }; 3678 3679 // Setting up region 3680 this.region = initHRegion(tableName, method, CONF, families); 3681 Scan scan = new Scan(); 3682 scan.addFamily(fam2); 3683 boolean ok = false; 3684 try { 3685 region.getScanner(scan); 3686 } catch (Exception e) { 3687 ok = true; 3688 } 3689 assertTrue("Families could not be found in Region", ok); 3690 } 3691 3692 @Test 3693 public void testGetScanner_WithNoFamilies() throws IOException { 3694 byte[] row1 = Bytes.toBytes("row1"); 3695 byte[] fam1 = Bytes.toBytes("fam1"); 3696 byte[] fam2 = Bytes.toBytes("fam2"); 3697 byte[] fam3 = Bytes.toBytes("fam3"); 3698 byte[] fam4 = Bytes.toBytes("fam4"); 3699 3700 byte[][] families = { fam1, fam2, fam3, fam4 }; 3701 3702 // Setting up region 3703 this.region = initHRegion(tableName, method, CONF, families); 3704 // Putting data in Region 3705 Put put = new Put(row1); 3706 put.addColumn(fam1, null, null); 3707 put.addColumn(fam2, null, null); 3708 put.addColumn(fam3, null, null); 3709 put.addColumn(fam4, null, null); 3710 region.put(put); 3711 3712 Scan scan = null; 3713 RegionScannerImpl is = null; 3714 3715 // Testing to see how many scanners that is produced by getScanner, 3716 // starting 3717 // with known number, 2 - current = 1 3718 scan = new Scan(); 3719 scan.addFamily(fam2); 3720 scan.addFamily(fam4); 3721 is = region.getScanner(scan); 3722 assertEquals(1, is.storeHeap.getHeap().size()); 3723 3724 scan = new Scan(); 3725 is = region.getScanner(scan); 3726 assertEquals(families.length - 1, is.storeHeap.getHeap().size()); 3727 } 3728 3729 /** 3730 * This method tests https://issues.apache.org/jira/browse/HBASE-2516. n 3731 */ 3732 @Test 3733 public void testGetScanner_WithRegionClosed() throws IOException { 3734 byte[] fam1 = Bytes.toBytes("fam1"); 3735 byte[] fam2 = Bytes.toBytes("fam2"); 3736 3737 byte[][] families = { fam1, fam2 }; 3738 3739 // Setting up region 3740 try { 3741 this.region = initHRegion(tableName, method, CONF, families); 3742 } catch (IOException e) { 3743 e.printStackTrace(); 3744 fail("Got IOException during initHRegion, " + e.getMessage()); 3745 } 3746 region.closed.set(true); 3747 try { 3748 region.getScanner(null); 3749 fail("Expected to get an exception during getScanner on a region that is closed"); 3750 } catch (NotServingRegionException e) { 3751 // this is the correct exception that is expected 3752 } catch (IOException e) { 3753 fail("Got wrong type of exception - should be a NotServingRegionException, " 3754 + "but was an IOException: " + e.getMessage()); 3755 } 3756 } 3757 3758 @Test 3759 public void testRegionScanner_Next() throws IOException { 3760 byte[] row1 = Bytes.toBytes("row1"); 3761 byte[] row2 = Bytes.toBytes("row2"); 3762 byte[] fam1 = Bytes.toBytes("fam1"); 3763 byte[] fam2 = Bytes.toBytes("fam2"); 3764 byte[] fam3 = Bytes.toBytes("fam3"); 3765 byte[] fam4 = Bytes.toBytes("fam4"); 3766 3767 byte[][] families = { fam1, fam2, fam3, fam4 }; 3768 long ts = EnvironmentEdgeManager.currentTime(); 3769 3770 // Setting up region 3771 this.region = initHRegion(tableName, method, CONF, families); 3772 // Putting data in Region 3773 Put put = null; 3774 put = new Put(row1); 3775 put.addColumn(fam1, (byte[]) null, ts, null); 3776 put.addColumn(fam2, (byte[]) null, ts, null); 3777 put.addColumn(fam3, (byte[]) null, ts, null); 3778 put.addColumn(fam4, (byte[]) null, ts, null); 3779 region.put(put); 3780 3781 put = new Put(row2); 3782 put.addColumn(fam1, (byte[]) null, ts, null); 3783 put.addColumn(fam2, (byte[]) null, ts, null); 3784 put.addColumn(fam3, (byte[]) null, ts, null); 3785 put.addColumn(fam4, (byte[]) null, ts, null); 3786 region.put(put); 3787 3788 Scan scan = new Scan(); 3789 scan.addFamily(fam2); 3790 scan.addFamily(fam4); 3791 InternalScanner is = region.getScanner(scan); 3792 3793 List<Cell> res = null; 3794 3795 // Result 1 3796 List<Cell> expected1 = new ArrayList<>(); 3797 expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null)); 3798 expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null)); 3799 3800 res = new ArrayList<>(); 3801 is.next(res); 3802 for (int i = 0; i < res.size(); i++) { 3803 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected1.get(i), res.get(i))); 3804 } 3805 3806 // Result 2 3807 List<Cell> expected2 = new ArrayList<>(); 3808 expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null)); 3809 expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null)); 3810 3811 res = new ArrayList<>(); 3812 is.next(res); 3813 for (int i = 0; i < res.size(); i++) { 3814 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected2.get(i), res.get(i))); 3815 } 3816 } 3817 3818 @Test 3819 public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException { 3820 byte[] row1 = Bytes.toBytes("row1"); 3821 byte[] qf1 = Bytes.toBytes("qualifier1"); 3822 byte[] qf2 = Bytes.toBytes("qualifier2"); 3823 byte[] fam1 = Bytes.toBytes("fam1"); 3824 byte[][] families = { fam1 }; 3825 3826 long ts1 = EnvironmentEdgeManager.currentTime(); 3827 long ts2 = ts1 + 1; 3828 long ts3 = ts1 + 2; 3829 3830 // Setting up region 3831 this.region = initHRegion(tableName, method, CONF, families); 3832 // Putting data in Region 3833 Put put = null; 3834 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 3835 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 3836 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 3837 3838 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 3839 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 3840 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 3841 3842 put = new Put(row1); 3843 put.add(kv13); 3844 put.add(kv12); 3845 put.add(kv11); 3846 put.add(kv23); 3847 put.add(kv22); 3848 put.add(kv21); 3849 region.put(put); 3850 3851 // Expected 3852 List<Cell> expected = new ArrayList<>(); 3853 expected.add(kv13); 3854 expected.add(kv12); 3855 3856 Scan scan = new Scan().withStartRow(row1); 3857 scan.addColumn(fam1, qf1); 3858 scan.readVersions(MAX_VERSIONS); 3859 List<Cell> actual = new ArrayList<>(); 3860 InternalScanner scanner = region.getScanner(scan); 3861 3862 boolean hasNext = scanner.next(actual); 3863 assertEquals(false, hasNext); 3864 3865 // Verify result 3866 for (int i = 0; i < expected.size(); i++) { 3867 assertEquals(expected.get(i), actual.get(i)); 3868 } 3869 } 3870 3871 @Test 3872 public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException { 3873 byte[] row1 = Bytes.toBytes("row1"); 3874 byte[] qf1 = Bytes.toBytes("qualifier1"); 3875 byte[] qf2 = Bytes.toBytes("qualifier2"); 3876 byte[] fam1 = Bytes.toBytes("fam1"); 3877 byte[][] families = { fam1 }; 3878 3879 long ts1 = 1; 3880 long ts2 = ts1 + 1; 3881 long ts3 = ts1 + 2; 3882 3883 // Setting up region 3884 this.region = initHRegion(tableName, method, CONF, families); 3885 // Putting data in Region 3886 Put put = null; 3887 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 3888 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 3889 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 3890 3891 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 3892 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 3893 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 3894 3895 put = new Put(row1); 3896 put.add(kv13); 3897 put.add(kv12); 3898 put.add(kv11); 3899 put.add(kv23); 3900 put.add(kv22); 3901 put.add(kv21); 3902 region.put(put); 3903 region.flush(true); 3904 3905 // Expected 3906 List<Cell> expected = new ArrayList<>(); 3907 expected.add(kv13); 3908 expected.add(kv12); 3909 expected.add(kv23); 3910 expected.add(kv22); 3911 3912 Scan scan = new Scan().withStartRow(row1); 3913 scan.addColumn(fam1, qf1); 3914 scan.addColumn(fam1, qf2); 3915 scan.readVersions(MAX_VERSIONS); 3916 List<Cell> actual = new ArrayList<>(); 3917 InternalScanner scanner = region.getScanner(scan); 3918 3919 boolean hasNext = scanner.next(actual); 3920 assertEquals(false, hasNext); 3921 3922 // Verify result 3923 for (int i = 0; i < expected.size(); i++) { 3924 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); 3925 } 3926 } 3927 3928 @Test 3929 public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() 3930 throws IOException { 3931 byte[] row1 = Bytes.toBytes("row1"); 3932 byte[] fam1 = Bytes.toBytes("fam1"); 3933 byte[][] families = { fam1 }; 3934 byte[] qf1 = Bytes.toBytes("qualifier1"); 3935 byte[] qf2 = Bytes.toBytes("qualifier2"); 3936 3937 long ts1 = 1; 3938 long ts2 = ts1 + 1; 3939 long ts3 = ts1 + 2; 3940 long ts4 = ts1 + 3; 3941 3942 // Setting up region 3943 this.region = initHRegion(tableName, method, CONF, families); 3944 // Putting data in Region 3945 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null); 3946 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 3947 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 3948 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 3949 3950 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null); 3951 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 3952 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 3953 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 3954 3955 Put put = null; 3956 put = new Put(row1); 3957 put.add(kv14); 3958 put.add(kv24); 3959 region.put(put); 3960 region.flush(true); 3961 3962 put = new Put(row1); 3963 put.add(kv23); 3964 put.add(kv13); 3965 region.put(put); 3966 region.flush(true); 3967 3968 put = new Put(row1); 3969 put.add(kv22); 3970 put.add(kv12); 3971 region.put(put); 3972 region.flush(true); 3973 3974 put = new Put(row1); 3975 put.add(kv21); 3976 put.add(kv11); 3977 region.put(put); 3978 3979 // Expected 3980 List<Cell> expected = new ArrayList<>(); 3981 expected.add(kv14); 3982 expected.add(kv13); 3983 expected.add(kv12); 3984 expected.add(kv24); 3985 expected.add(kv23); 3986 expected.add(kv22); 3987 3988 Scan scan = new Scan().withStartRow(row1); 3989 scan.addColumn(fam1, qf1); 3990 scan.addColumn(fam1, qf2); 3991 int versions = 3; 3992 scan.readVersions(versions); 3993 List<Cell> actual = new ArrayList<>(); 3994 InternalScanner scanner = region.getScanner(scan); 3995 3996 boolean hasNext = scanner.next(actual); 3997 assertEquals(false, hasNext); 3998 3999 // Verify result 4000 for (int i = 0; i < expected.size(); i++) { 4001 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); 4002 } 4003 } 4004 4005 @Test 4006 public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException { 4007 byte[] row1 = Bytes.toBytes("row1"); 4008 byte[] qf1 = Bytes.toBytes("qualifier1"); 4009 byte[] qf2 = Bytes.toBytes("qualifier2"); 4010 byte[] fam1 = Bytes.toBytes("fam1"); 4011 byte[][] families = { fam1 }; 4012 4013 long ts1 = EnvironmentEdgeManager.currentTime(); 4014 long ts2 = ts1 + 1; 4015 long ts3 = ts1 + 2; 4016 4017 // Setting up region 4018 this.region = initHRegion(tableName, method, CONF, families); 4019 // Putting data in Region 4020 Put put = null; 4021 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 4022 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 4023 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 4024 4025 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 4026 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 4027 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 4028 4029 put = new Put(row1); 4030 put.add(kv13); 4031 put.add(kv12); 4032 put.add(kv11); 4033 put.add(kv23); 4034 put.add(kv22); 4035 put.add(kv21); 4036 region.put(put); 4037 4038 // Expected 4039 List<Cell> expected = new ArrayList<>(); 4040 expected.add(kv13); 4041 expected.add(kv12); 4042 expected.add(kv23); 4043 expected.add(kv22); 4044 4045 Scan scan = new Scan().withStartRow(row1); 4046 scan.addFamily(fam1); 4047 scan.readVersions(MAX_VERSIONS); 4048 List<Cell> actual = new ArrayList<>(); 4049 InternalScanner scanner = region.getScanner(scan); 4050 4051 boolean hasNext = scanner.next(actual); 4052 assertEquals(false, hasNext); 4053 4054 // Verify result 4055 for (int i = 0; i < expected.size(); i++) { 4056 assertEquals(expected.get(i), actual.get(i)); 4057 } 4058 } 4059 4060 @Test 4061 public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException { 4062 byte[] row1 = Bytes.toBytes("row1"); 4063 byte[] qf1 = Bytes.toBytes("qualifier1"); 4064 byte[] qf2 = Bytes.toBytes("qualifier2"); 4065 byte[] fam1 = Bytes.toBytes("fam1"); 4066 4067 long ts1 = 1; 4068 long ts2 = ts1 + 1; 4069 long ts3 = ts1 + 2; 4070 4071 // Setting up region 4072 this.region = initHRegion(tableName, method, CONF, fam1); 4073 // Putting data in Region 4074 Put put = null; 4075 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 4076 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 4077 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 4078 4079 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 4080 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 4081 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 4082 4083 put = new Put(row1); 4084 put.add(kv13); 4085 put.add(kv12); 4086 put.add(kv11); 4087 put.add(kv23); 4088 put.add(kv22); 4089 put.add(kv21); 4090 region.put(put); 4091 region.flush(true); 4092 4093 // Expected 4094 List<Cell> expected = new ArrayList<>(); 4095 expected.add(kv13); 4096 expected.add(kv12); 4097 expected.add(kv23); 4098 expected.add(kv22); 4099 4100 Scan scan = new Scan().withStartRow(row1); 4101 scan.addFamily(fam1); 4102 scan.readVersions(MAX_VERSIONS); 4103 List<Cell> actual = new ArrayList<>(); 4104 InternalScanner scanner = region.getScanner(scan); 4105 4106 boolean hasNext = scanner.next(actual); 4107 assertEquals(false, hasNext); 4108 4109 // Verify result 4110 for (int i = 0; i < expected.size(); i++) { 4111 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); 4112 } 4113 } 4114 4115 @Test 4116 public void testScanner_StopRow1542() throws IOException { 4117 byte[] family = Bytes.toBytes("testFamily"); 4118 this.region = initHRegion(tableName, method, CONF, family); 4119 byte[] row1 = Bytes.toBytes("row111"); 4120 byte[] row2 = Bytes.toBytes("row222"); 4121 byte[] row3 = Bytes.toBytes("row333"); 4122 byte[] row4 = Bytes.toBytes("row444"); 4123 byte[] row5 = Bytes.toBytes("row555"); 4124 4125 byte[] col1 = Bytes.toBytes("Pub111"); 4126 byte[] col2 = Bytes.toBytes("Pub222"); 4127 4128 Put put = new Put(row1); 4129 put.addColumn(family, col1, Bytes.toBytes(10L)); 4130 region.put(put); 4131 4132 put = new Put(row2); 4133 put.addColumn(family, col1, Bytes.toBytes(15L)); 4134 region.put(put); 4135 4136 put = new Put(row3); 4137 put.addColumn(family, col2, Bytes.toBytes(20L)); 4138 region.put(put); 4139 4140 put = new Put(row4); 4141 put.addColumn(family, col2, Bytes.toBytes(30L)); 4142 region.put(put); 4143 4144 put = new Put(row5); 4145 put.addColumn(family, col1, Bytes.toBytes(40L)); 4146 region.put(put); 4147 4148 Scan scan = new Scan().withStartRow(row3).withStopRow(row4); 4149 scan.readAllVersions(); 4150 scan.addColumn(family, col1); 4151 InternalScanner s = region.getScanner(scan); 4152 4153 List<Cell> results = new ArrayList<>(); 4154 assertEquals(false, s.next(results)); 4155 assertEquals(0, results.size()); 4156 } 4157 4158 @Test 4159 public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException { 4160 byte[] row1 = Bytes.toBytes("row1"); 4161 byte[] fam1 = Bytes.toBytes("fam1"); 4162 byte[] qf1 = Bytes.toBytes("qualifier1"); 4163 byte[] qf2 = Bytes.toBytes("quateslifier2"); 4164 4165 long ts1 = 1; 4166 long ts2 = ts1 + 1; 4167 long ts3 = ts1 + 2; 4168 long ts4 = ts1 + 3; 4169 4170 // Setting up region 4171 this.region = initHRegion(tableName, method, CONF, fam1); 4172 // Putting data in Region 4173 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null); 4174 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 4175 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 4176 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 4177 4178 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null); 4179 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 4180 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 4181 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 4182 4183 Put put = null; 4184 put = new Put(row1); 4185 put.add(kv14); 4186 put.add(kv24); 4187 region.put(put); 4188 region.flush(true); 4189 4190 put = new Put(row1); 4191 put.add(kv23); 4192 put.add(kv13); 4193 region.put(put); 4194 region.flush(true); 4195 4196 put = new Put(row1); 4197 put.add(kv22); 4198 put.add(kv12); 4199 region.put(put); 4200 region.flush(true); 4201 4202 put = new Put(row1); 4203 put.add(kv21); 4204 put.add(kv11); 4205 region.put(put); 4206 4207 // Expected 4208 List<KeyValue> expected = new ArrayList<>(); 4209 expected.add(kv14); 4210 expected.add(kv13); 4211 expected.add(kv12); 4212 expected.add(kv24); 4213 expected.add(kv23); 4214 expected.add(kv22); 4215 4216 Scan scan = new Scan().withStartRow(row1); 4217 int versions = 3; 4218 scan.readVersions(versions); 4219 List<Cell> actual = new ArrayList<>(); 4220 InternalScanner scanner = region.getScanner(scan); 4221 4222 boolean hasNext = scanner.next(actual); 4223 assertEquals(false, hasNext); 4224 4225 // Verify result 4226 for (int i = 0; i < expected.size(); i++) { 4227 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); 4228 } 4229 } 4230 4231 /** 4232 * Added for HBASE-5416 Here we test scan optimization when only subset of CFs are used in filter 4233 * conditions. 4234 */ 4235 @Test 4236 public void testScanner_JoinedScanners() throws IOException { 4237 byte[] cf_essential = Bytes.toBytes("essential"); 4238 byte[] cf_joined = Bytes.toBytes("joined"); 4239 byte[] cf_alpha = Bytes.toBytes("alpha"); 4240 this.region = initHRegion(tableName, method, CONF, cf_essential, cf_joined, cf_alpha); 4241 byte[] row1 = Bytes.toBytes("row1"); 4242 byte[] row2 = Bytes.toBytes("row2"); 4243 byte[] row3 = Bytes.toBytes("row3"); 4244 4245 byte[] col_normal = Bytes.toBytes("d"); 4246 byte[] col_alpha = Bytes.toBytes("a"); 4247 4248 byte[] filtered_val = Bytes.toBytes(3); 4249 4250 Put put = new Put(row1); 4251 put.addColumn(cf_essential, col_normal, Bytes.toBytes(1)); 4252 put.addColumn(cf_joined, col_alpha, Bytes.toBytes(1)); 4253 region.put(put); 4254 4255 put = new Put(row2); 4256 put.addColumn(cf_essential, col_alpha, Bytes.toBytes(2)); 4257 put.addColumn(cf_joined, col_normal, Bytes.toBytes(2)); 4258 put.addColumn(cf_alpha, col_alpha, Bytes.toBytes(2)); 4259 region.put(put); 4260 4261 put = new Put(row3); 4262 put.addColumn(cf_essential, col_normal, filtered_val); 4263 put.addColumn(cf_joined, col_normal, filtered_val); 4264 region.put(put); 4265 4266 // Check two things: 4267 // 1. result list contains expected values 4268 // 2. result list is sorted properly 4269 4270 Scan scan = new Scan(); 4271 Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal, 4272 CompareOperator.NOT_EQUAL, filtered_val); 4273 scan.setFilter(filter); 4274 scan.setLoadColumnFamiliesOnDemand(true); 4275 InternalScanner s = region.getScanner(scan); 4276 4277 List<Cell> results = new ArrayList<>(); 4278 assertTrue(s.next(results)); 4279 assertEquals(1, results.size()); 4280 results.clear(); 4281 4282 assertTrue(s.next(results)); 4283 assertEquals(3, results.size()); 4284 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha)); 4285 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential)); 4286 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined)); 4287 results.clear(); 4288 4289 assertFalse(s.next(results)); 4290 assertEquals(0, results.size()); 4291 } 4292 4293 /** 4294 * HBASE-5416 Test case when scan limits amount of KVs returned on each next() call. 4295 */ 4296 @Test 4297 public void testScanner_JoinedScannersWithLimits() throws IOException { 4298 final byte[] cf_first = Bytes.toBytes("first"); 4299 final byte[] cf_second = Bytes.toBytes("second"); 4300 4301 this.region = initHRegion(tableName, method, CONF, cf_first, cf_second); 4302 final byte[] col_a = Bytes.toBytes("a"); 4303 final byte[] col_b = Bytes.toBytes("b"); 4304 4305 Put put; 4306 4307 for (int i = 0; i < 10; i++) { 4308 put = new Put(Bytes.toBytes("r" + Integer.toString(i))); 4309 put.addColumn(cf_first, col_a, Bytes.toBytes(i)); 4310 if (i < 5) { 4311 put.addColumn(cf_first, col_b, Bytes.toBytes(i)); 4312 put.addColumn(cf_second, col_a, Bytes.toBytes(i)); 4313 put.addColumn(cf_second, col_b, Bytes.toBytes(i)); 4314 } 4315 region.put(put); 4316 } 4317 4318 Scan scan = new Scan(); 4319 scan.setLoadColumnFamiliesOnDemand(true); 4320 Filter bogusFilter = new FilterBase() { 4321 @Override 4322 public ReturnCode filterCell(final Cell ignored) throws IOException { 4323 return ReturnCode.INCLUDE; 4324 } 4325 4326 @Override 4327 public boolean isFamilyEssential(byte[] name) { 4328 return Bytes.equals(name, cf_first); 4329 } 4330 }; 4331 4332 scan.setFilter(bogusFilter); 4333 InternalScanner s = region.getScanner(scan); 4334 4335 // Our data looks like this: 4336 // r0: first:a, first:b, second:a, second:b 4337 // r1: first:a, first:b, second:a, second:b 4338 // r2: first:a, first:b, second:a, second:b 4339 // r3: first:a, first:b, second:a, second:b 4340 // r4: first:a, first:b, second:a, second:b 4341 // r5: first:a 4342 // r6: first:a 4343 // r7: first:a 4344 // r8: first:a 4345 // r9: first:a 4346 4347 // But due to next's limit set to 3, we should get this: 4348 // r0: first:a, first:b, second:a 4349 // r0: second:b 4350 // r1: first:a, first:b, second:a 4351 // r1: second:b 4352 // r2: first:a, first:b, second:a 4353 // r2: second:b 4354 // r3: first:a, first:b, second:a 4355 // r3: second:b 4356 // r4: first:a, first:b, second:a 4357 // r4: second:b 4358 // r5: first:a 4359 // r6: first:a 4360 // r7: first:a 4361 // r8: first:a 4362 // r9: first:a 4363 4364 List<Cell> results = new ArrayList<>(); 4365 int index = 0; 4366 ScannerContext scannerContext = ScannerContext.newBuilder().setBatchLimit(3).build(); 4367 while (true) { 4368 boolean more = s.next(results, scannerContext); 4369 if ((index >> 1) < 5) { 4370 if (index % 2 == 0) { 4371 assertEquals(3, results.size()); 4372 } else { 4373 assertEquals(1, results.size()); 4374 } 4375 } else { 4376 assertEquals(1, results.size()); 4377 } 4378 results.clear(); 4379 index++; 4380 if (!more) { 4381 break; 4382 } 4383 } 4384 } 4385 4386 @Test 4387 public void testScannerOperationId() throws IOException { 4388 region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES); 4389 Scan scan = new Scan(); 4390 RegionScanner scanner = region.getScanner(scan); 4391 assertNull(scanner.getOperationId()); 4392 scanner.close(); 4393 4394 String operationId = "test_operation_id_0101"; 4395 scan = new Scan().setId(operationId); 4396 scanner = region.getScanner(scan); 4397 assertEquals(operationId, scanner.getOperationId()); 4398 scanner.close(); 4399 4400 HBaseTestingUtil.closeRegionAndWAL(this.region); 4401 } 4402 4403 /** 4404 * Write an HFile block full with Cells whose qualifier that are identical between 0 and 4405 * Short.MAX_VALUE. See HBASE-13329. n 4406 */ 4407 @Test 4408 public void testLongQualifier() throws Exception { 4409 byte[] family = Bytes.toBytes("family"); 4410 this.region = initHRegion(tableName, method, CONF, family); 4411 byte[] q = new byte[Short.MAX_VALUE + 2]; 4412 Arrays.fill(q, 0, q.length - 1, (byte) 42); 4413 for (byte i = 0; i < 10; i++) { 4414 Put p = new Put(Bytes.toBytes("row")); 4415 // qualifiers that differ past Short.MAX_VALUE 4416 q[q.length - 1] = i; 4417 p.addColumn(family, q, q); 4418 region.put(p); 4419 } 4420 region.flush(false); 4421 } 4422 4423 /** 4424 * Flushes the cache in a thread while scanning. The tests verify that the scan is coherent - e.g. 4425 * the returned results are always of the same or later update as the previous results. n * scan / 4426 * compact n * thread join 4427 */ 4428 @Test 4429 public void testFlushCacheWhileScanning() throws IOException, InterruptedException { 4430 byte[] family = Bytes.toBytes("family"); 4431 int numRows = 1000; 4432 int flushAndScanInterval = 10; 4433 int compactInterval = 10 * flushAndScanInterval; 4434 4435 this.region = initHRegion(tableName, method, CONF, family); 4436 FlushThread flushThread = new FlushThread(); 4437 try { 4438 flushThread.start(); 4439 4440 Scan scan = new Scan(); 4441 scan.addFamily(family); 4442 scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOperator.EQUAL, 4443 new BinaryComparator(Bytes.toBytes(5L)))); 4444 4445 int expectedCount = 0; 4446 List<Cell> res = new ArrayList<>(); 4447 4448 boolean toggle = true; 4449 for (long i = 0; i < numRows; i++) { 4450 Put put = new Put(Bytes.toBytes(i)); 4451 put.setDurability(Durability.SKIP_WAL); 4452 put.addColumn(family, qual1, Bytes.toBytes(i % 10)); 4453 region.put(put); 4454 4455 if (i != 0 && i % compactInterval == 0) { 4456 LOG.debug("iteration = " + i + " ts=" + EnvironmentEdgeManager.currentTime()); 4457 region.compact(true); 4458 } 4459 4460 if (i % 10 == 5L) { 4461 expectedCount++; 4462 } 4463 4464 if (i != 0 && i % flushAndScanInterval == 0) { 4465 res.clear(); 4466 InternalScanner scanner = region.getScanner(scan); 4467 if (toggle) { 4468 flushThread.flush(); 4469 } 4470 while (scanner.next(res)) 4471 ; 4472 if (!toggle) { 4473 flushThread.flush(); 4474 } 4475 assertEquals( 4476 "toggle=" + toggle + "i=" + i + " ts=" + EnvironmentEdgeManager.currentTime(), 4477 expectedCount, res.size()); 4478 toggle = !toggle; 4479 } 4480 } 4481 4482 } finally { 4483 try { 4484 flushThread.done(); 4485 flushThread.join(); 4486 flushThread.checkNoError(); 4487 } catch (InterruptedException ie) { 4488 LOG.warn("Caught exception when joining with flushThread", ie); 4489 } 4490 HBaseTestingUtil.closeRegionAndWAL(this.region); 4491 this.region = null; 4492 } 4493 } 4494 4495 protected class FlushThread extends Thread { 4496 private volatile boolean done; 4497 private Throwable error = null; 4498 4499 FlushThread() { 4500 super("FlushThread"); 4501 } 4502 4503 public void done() { 4504 done = true; 4505 synchronized (this) { 4506 interrupt(); 4507 } 4508 } 4509 4510 public void checkNoError() { 4511 if (error != null) { 4512 assertNull(error); 4513 } 4514 } 4515 4516 @Override 4517 public void run() { 4518 done = false; 4519 while (!done) { 4520 synchronized (this) { 4521 try { 4522 wait(); 4523 } catch (InterruptedException ignored) { 4524 if (done) { 4525 break; 4526 } 4527 } 4528 } 4529 try { 4530 region.flush(true); 4531 } catch (IOException e) { 4532 if (!done) { 4533 LOG.error("Error while flushing cache", e); 4534 error = e; 4535 } 4536 break; 4537 } catch (Throwable t) { 4538 LOG.error("Uncaught exception", t); 4539 throw t; 4540 } 4541 } 4542 } 4543 4544 public void flush() { 4545 synchronized (this) { 4546 notify(); 4547 } 4548 } 4549 } 4550 4551 /** 4552 * So can be overridden in subclasses. 4553 */ 4554 int getNumQualifiersForTestWritesWhileScanning() { 4555 return 100; 4556 } 4557 4558 /** 4559 * So can be overridden in subclasses. 4560 */ 4561 int getTestCountForTestWritesWhileScanning() { 4562 return 100; 4563 } 4564 4565 /** 4566 * Writes very wide records and scans for the latest every time.. Flushes and compacts the region 4567 * every now and then to keep things realistic. n * by flush / scan / compaction n * when joining 4568 * threads 4569 */ 4570 @Test 4571 public void testWritesWhileScanning() throws IOException, InterruptedException { 4572 int testCount = getTestCountForTestWritesWhileScanning(); 4573 int numRows = 1; 4574 int numFamilies = 10; 4575 int numQualifiers = getNumQualifiersForTestWritesWhileScanning(); 4576 int flushInterval = 7; 4577 int compactInterval = 5 * flushInterval; 4578 byte[][] families = new byte[numFamilies][]; 4579 for (int i = 0; i < numFamilies; i++) { 4580 families[i] = Bytes.toBytes("family" + i); 4581 } 4582 byte[][] qualifiers = new byte[numQualifiers][]; 4583 for (int i = 0; i < numQualifiers; i++) { 4584 qualifiers[i] = Bytes.toBytes("qual" + i); 4585 } 4586 4587 this.region = initHRegion(tableName, method, CONF, families); 4588 FlushThread flushThread = new FlushThread(); 4589 PutThread putThread = new PutThread(numRows, families, qualifiers); 4590 try { 4591 putThread.start(); 4592 putThread.waitForFirstPut(); 4593 4594 flushThread.start(); 4595 4596 Scan scan = new Scan().withStartRow(Bytes.toBytes("row0")).withStopRow(Bytes.toBytes("row1")); 4597 4598 int expectedCount = numFamilies * numQualifiers; 4599 List<Cell> res = new ArrayList<>(); 4600 4601 long prevTimestamp = 0L; 4602 for (int i = 0; i < testCount; i++) { 4603 4604 if (i != 0 && i % compactInterval == 0) { 4605 region.compact(true); 4606 for (HStore store : region.getStores()) { 4607 store.closeAndArchiveCompactedFiles(); 4608 } 4609 } 4610 4611 if (i != 0 && i % flushInterval == 0) { 4612 flushThread.flush(); 4613 } 4614 4615 boolean previousEmpty = res.isEmpty(); 4616 res.clear(); 4617 try (InternalScanner scanner = region.getScanner(scan)) { 4618 boolean moreRows; 4619 do { 4620 moreRows = scanner.next(res); 4621 } while (moreRows); 4622 } 4623 if (!res.isEmpty() || !previousEmpty || i > compactInterval) { 4624 assertEquals("i=" + i, expectedCount, res.size()); 4625 long timestamp = res.get(0).getTimestamp(); 4626 assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp, 4627 timestamp >= prevTimestamp); 4628 prevTimestamp = timestamp; 4629 } 4630 } 4631 4632 putThread.done(); 4633 4634 region.flush(true); 4635 4636 } finally { 4637 try { 4638 flushThread.done(); 4639 flushThread.join(); 4640 flushThread.checkNoError(); 4641 4642 putThread.join(); 4643 putThread.checkNoError(); 4644 } catch (InterruptedException ie) { 4645 LOG.warn("Caught exception when joining with flushThread", ie); 4646 } 4647 4648 try { 4649 HBaseTestingUtil.closeRegionAndWAL(this.region); 4650 } catch (DroppedSnapshotException dse) { 4651 // We could get this on way out because we interrupt the background flusher and it could 4652 // fail anywhere causing a DSE over in the background flusher... only it is not properly 4653 // dealt with so could still be memory hanging out when we get to here -- memory we can't 4654 // flush because the accounting is 'off' since original DSE. 4655 } 4656 this.region = null; 4657 } 4658 } 4659 4660 protected class PutThread extends Thread { 4661 private volatile boolean done; 4662 private volatile int numPutsFinished = 0; 4663 4664 private Throwable error = null; 4665 private int numRows; 4666 private byte[][] families; 4667 private byte[][] qualifiers; 4668 4669 private PutThread(int numRows, byte[][] families, byte[][] qualifiers) { 4670 super("PutThread"); 4671 this.numRows = numRows; 4672 this.families = families; 4673 this.qualifiers = qualifiers; 4674 } 4675 4676 /** 4677 * Block calling thread until this instance of PutThread has put at least one row. 4678 */ 4679 public void waitForFirstPut() throws InterruptedException { 4680 // wait until put thread actually puts some data 4681 while (isAlive() && numPutsFinished == 0) { 4682 checkNoError(); 4683 Thread.sleep(50); 4684 } 4685 } 4686 4687 public void done() { 4688 done = true; 4689 synchronized (this) { 4690 interrupt(); 4691 } 4692 } 4693 4694 public void checkNoError() { 4695 if (error != null) { 4696 assertNull(error); 4697 } 4698 } 4699 4700 @Override 4701 public void run() { 4702 done = false; 4703 while (!done) { 4704 try { 4705 for (int r = 0; r < numRows; r++) { 4706 byte[] row = Bytes.toBytes("row" + r); 4707 Put put = new Put(row); 4708 put.setDurability(Durability.SKIP_WAL); 4709 byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished)); 4710 for (byte[] family : families) { 4711 for (byte[] qualifier : qualifiers) { 4712 put.addColumn(family, qualifier, numPutsFinished, value); 4713 } 4714 } 4715 region.put(put); 4716 numPutsFinished++; 4717 if (numPutsFinished > 0 && numPutsFinished % 47 == 0) { 4718 LOG.debug("put iteration = {}", numPutsFinished); 4719 Delete delete = new Delete(row, (long) numPutsFinished - 30); 4720 region.delete(delete); 4721 } 4722 numPutsFinished++; 4723 } 4724 } catch (InterruptedIOException e) { 4725 // This is fine. It means we are done, or didn't get the lock on time 4726 LOG.info("Interrupted", e); 4727 } catch (IOException e) { 4728 LOG.error("Error while putting records", e); 4729 error = e; 4730 break; 4731 } 4732 } 4733 4734 } 4735 4736 } 4737 4738 /** 4739 * Writes very wide records and gets the latest row every time.. Flushes and compacts the region 4740 * aggressivly to catch issues. n * by flush / scan / compaction n * when joining threads 4741 */ 4742 @Test 4743 public void testWritesWhileGetting() throws Exception { 4744 int testCount = 50; 4745 int numRows = 1; 4746 int numFamilies = 10; 4747 int numQualifiers = 100; 4748 int compactInterval = 100; 4749 byte[][] families = new byte[numFamilies][]; 4750 for (int i = 0; i < numFamilies; i++) { 4751 families[i] = Bytes.toBytes("family" + i); 4752 } 4753 byte[][] qualifiers = new byte[numQualifiers][]; 4754 for (int i = 0; i < numQualifiers; i++) { 4755 qualifiers[i] = Bytes.toBytes("qual" + i); 4756 } 4757 4758 // This test flushes constantly and can cause many files to be created, 4759 // possibly 4760 // extending over the ulimit. Make sure compactions are aggressive in 4761 // reducing 4762 // the number of HFiles created. 4763 Configuration conf = HBaseConfiguration.create(CONF); 4764 conf.setInt("hbase.hstore.compaction.min", 1); 4765 conf.setInt("hbase.hstore.compaction.max", 1000); 4766 this.region = initHRegion(tableName, method, conf, families); 4767 PutThread putThread = null; 4768 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf); 4769 try { 4770 putThread = new PutThread(numRows, families, qualifiers); 4771 putThread.start(); 4772 putThread.waitForFirstPut(); 4773 4774 // Add a thread that flushes as fast as possible 4775 ctx.addThread(new RepeatingTestThread(ctx) { 4776 4777 @Override 4778 public void doAnAction() throws Exception { 4779 region.flush(true); 4780 // Compact regularly to avoid creating too many files and exceeding 4781 // the ulimit. 4782 region.compact(false); 4783 for (HStore store : region.getStores()) { 4784 store.closeAndArchiveCompactedFiles(); 4785 } 4786 } 4787 }); 4788 ctx.startThreads(); 4789 4790 Get get = new Get(Bytes.toBytes("row0")); 4791 Result result = null; 4792 4793 int expectedCount = numFamilies * numQualifiers; 4794 4795 long prevTimestamp = 0L; 4796 for (int i = 0; i < testCount; i++) { 4797 LOG.info("testWritesWhileGetting verify turn " + i); 4798 boolean previousEmpty = result == null || result.isEmpty(); 4799 result = region.get(get); 4800 if (!result.isEmpty() || !previousEmpty || i > compactInterval) { 4801 assertEquals("i=" + i, expectedCount, result.size()); 4802 // TODO this was removed, now what dangit?! 4803 // search looking for the qualifier in question? 4804 long timestamp = 0; 4805 for (Cell kv : result.rawCells()) { 4806 if ( 4807 CellUtil.matchingFamily(kv, families[0]) 4808 && CellUtil.matchingQualifier(kv, qualifiers[0]) 4809 ) { 4810 timestamp = kv.getTimestamp(); 4811 } 4812 } 4813 assertTrue(timestamp >= prevTimestamp); 4814 prevTimestamp = timestamp; 4815 Cell previousKV = null; 4816 4817 for (Cell kv : result.rawCells()) { 4818 byte[] thisValue = CellUtil.cloneValue(kv); 4819 if (previousKV != null) { 4820 if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) { 4821 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV 4822 + "(memStoreTS:" + previousKV.getSequenceId() + ")" + ", New KV: " + kv 4823 + "(memStoreTS:" + kv.getSequenceId() + ")"); 4824 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue)); 4825 } 4826 } 4827 previousKV = kv; 4828 } 4829 } 4830 } 4831 } finally { 4832 if (putThread != null) putThread.done(); 4833 4834 region.flush(true); 4835 4836 if (putThread != null) { 4837 putThread.join(); 4838 putThread.checkNoError(); 4839 } 4840 4841 ctx.stop(); 4842 HBaseTestingUtil.closeRegionAndWAL(this.region); 4843 this.region = null; 4844 } 4845 } 4846 4847 @Test 4848 public void testHolesInMeta() throws Exception { 4849 byte[] family = Bytes.toBytes("family"); 4850 this.region = 4851 initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF, false, family); 4852 byte[] rowNotServed = Bytes.toBytes("a"); 4853 Get g = new Get(rowNotServed); 4854 try { 4855 region.get(g); 4856 fail(); 4857 } catch (WrongRegionException x) { 4858 // OK 4859 } 4860 byte[] row = Bytes.toBytes("y"); 4861 g = new Get(row); 4862 region.get(g); 4863 } 4864 4865 @Test 4866 public void testIndexesScanWithOneDeletedRow() throws IOException { 4867 byte[] family = Bytes.toBytes("family"); 4868 4869 // Setting up region 4870 this.region = initHRegion(tableName, method, CONF, family); 4871 Put put = new Put(Bytes.toBytes(1L)); 4872 put.addColumn(family, qual1, 1L, Bytes.toBytes(1L)); 4873 region.put(put); 4874 4875 region.flush(true); 4876 4877 Delete delete = new Delete(Bytes.toBytes(1L), 1L); 4878 region.delete(delete); 4879 4880 put = new Put(Bytes.toBytes(2L)); 4881 put.addColumn(family, qual1, 2L, Bytes.toBytes(2L)); 4882 region.put(put); 4883 4884 Scan idxScan = new Scan(); 4885 idxScan.addFamily(family); 4886 idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, 4887 Arrays.<Filter> asList( 4888 new SingleColumnValueFilter(family, qual1, CompareOperator.GREATER_OR_EQUAL, 4889 new BinaryComparator(Bytes.toBytes(0L))), 4890 new SingleColumnValueFilter(family, qual1, CompareOperator.LESS_OR_EQUAL, 4891 new BinaryComparator(Bytes.toBytes(3L)))))); 4892 InternalScanner scanner = region.getScanner(idxScan); 4893 List<Cell> res = new ArrayList<>(); 4894 4895 while (scanner.next(res)) { 4896 // Ignore res value. 4897 } 4898 assertEquals(1L, res.size()); 4899 } 4900 4901 // //////////////////////////////////////////////////////////////////////////// 4902 // Bloom filter test 4903 // //////////////////////////////////////////////////////////////////////////// 4904 @Test 4905 public void testBloomFilterSize() throws IOException { 4906 byte[] fam1 = Bytes.toBytes("fam1"); 4907 byte[] qf1 = Bytes.toBytes("col"); 4908 byte[] val1 = Bytes.toBytes("value1"); 4909 // Create Table 4910 TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName) 4911 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam1) 4912 .setMaxVersions(Integer.MAX_VALUE).setBloomFilterType(BloomType.ROWCOL).build()) 4913 .build(); 4914 RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 4915 this.region = TEST_UTIL.createLocalHRegion(info, tableDescriptor); 4916 int num_unique_rows = 10; 4917 int duplicate_multiplier = 2; 4918 int num_storefiles = 4; 4919 4920 int version = 0; 4921 for (int f = 0; f < num_storefiles; f++) { 4922 for (int i = 0; i < duplicate_multiplier; i++) { 4923 for (int j = 0; j < num_unique_rows; j++) { 4924 Put put = new Put(Bytes.toBytes("row" + j)); 4925 put.setDurability(Durability.SKIP_WAL); 4926 long ts = version++; 4927 put.addColumn(fam1, qf1, ts, val1); 4928 region.put(put); 4929 } 4930 } 4931 region.flush(true); 4932 } 4933 // before compaction 4934 HStore store = region.getStore(fam1); 4935 Collection<HStoreFile> storeFiles = store.getStorefiles(); 4936 for (HStoreFile storefile : storeFiles) { 4937 StoreFileReader reader = storefile.getReader(); 4938 reader.loadFileInfo(); 4939 reader.loadBloomfilter(); 4940 assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries()); 4941 assertEquals(num_unique_rows, reader.getFilterEntries()); 4942 } 4943 4944 region.compact(true); 4945 4946 // after compaction 4947 storeFiles = store.getStorefiles(); 4948 for (HStoreFile storefile : storeFiles) { 4949 StoreFileReader reader = storefile.getReader(); 4950 reader.loadFileInfo(); 4951 reader.loadBloomfilter(); 4952 assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries()); 4953 assertEquals(num_unique_rows, reader.getFilterEntries()); 4954 } 4955 } 4956 4957 @Test 4958 public void testAllColumnsWithBloomFilter() throws IOException { 4959 byte[] TABLE = Bytes.toBytes(name.getMethodName()); 4960 byte[] FAMILY = Bytes.toBytes("family"); 4961 4962 // Create table 4963 TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(TableName.valueOf(TABLE)) 4964 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY) 4965 .setMaxVersions(Integer.MAX_VALUE).setBloomFilterType(BloomType.ROWCOL).build()) 4966 .build(); 4967 RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 4968 this.region = TEST_UTIL.createLocalHRegion(info, tableDescriptor); 4969 // For row:0, col:0: insert versions 1 through 5. 4970 byte[] row = Bytes.toBytes("row:" + 0); 4971 byte[] column = Bytes.toBytes("column:" + 0); 4972 Put put = new Put(row); 4973 put.setDurability(Durability.SKIP_WAL); 4974 for (long idx = 1; idx <= 4; idx++) { 4975 put.addColumn(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx)); 4976 } 4977 region.put(put); 4978 4979 // Flush 4980 region.flush(true); 4981 4982 // Get rows 4983 Get get = new Get(row); 4984 get.readAllVersions(); 4985 Cell[] kvs = region.get(get).rawCells(); 4986 4987 // Check if rows are correct 4988 assertEquals(4, kvs.length); 4989 checkOneCell(kvs[0], FAMILY, 0, 0, 4); 4990 checkOneCell(kvs[1], FAMILY, 0, 0, 3); 4991 checkOneCell(kvs[2], FAMILY, 0, 0, 2); 4992 checkOneCell(kvs[3], FAMILY, 0, 0, 1); 4993 } 4994 4995 /** 4996 * Testcase to cover bug-fix for HBASE-2823 Ensures correct delete when issuing delete row on 4997 * columns with bloom filter set to row+col (BloomType.ROWCOL) 4998 */ 4999 @Test 5000 public void testDeleteRowWithBloomFilter() throws IOException { 5001 byte[] familyName = Bytes.toBytes("familyName"); 5002 5003 // Create Table 5004 TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName) 5005 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(familyName) 5006 .setMaxVersions(Integer.MAX_VALUE).setBloomFilterType(BloomType.ROWCOL).build()) 5007 .build(); 5008 RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 5009 this.region = TEST_UTIL.createLocalHRegion(info, tableDescriptor); 5010 // Insert some data 5011 byte[] row = Bytes.toBytes("row1"); 5012 byte[] col = Bytes.toBytes("col1"); 5013 5014 Put put = new Put(row); 5015 put.addColumn(familyName, col, 1, Bytes.toBytes("SomeRandomValue")); 5016 region.put(put); 5017 region.flush(true); 5018 5019 Delete del = new Delete(row); 5020 region.delete(del); 5021 region.flush(true); 5022 5023 // Get remaining rows (should have none) 5024 Get get = new Get(row); 5025 get.addColumn(familyName, col); 5026 5027 Cell[] keyValues = region.get(get).rawCells(); 5028 assertEquals(0, keyValues.length); 5029 } 5030 5031 @Test 5032 public void testgetHDFSBlocksDistribution() throws Exception { 5033 HBaseTestingUtil htu = new HBaseTestingUtil(); 5034 // Why do we set the block size in this test? If we set it smaller than the kvs, then we'll 5035 // break up the file in to more pieces that can be distributed across the three nodes and we 5036 // won't be able to have the condition this test asserts; that at least one node has 5037 // a copy of all replicas -- if small block size, then blocks are spread evenly across the 5038 // the three nodes. hfilev3 with tags seems to put us over the block size. St.Ack. 5039 // final int DEFAULT_BLOCK_SIZE = 1024; 5040 // htu.getConfiguration().setLong("dfs.blocksize", DEFAULT_BLOCK_SIZE); 5041 htu.getConfiguration().setInt("dfs.replication", 2); 5042 5043 // set up a cluster with 3 nodes 5044 SingleProcessHBaseCluster cluster = null; 5045 String dataNodeHosts[] = new String[] { "host1", "host2", "host3" }; 5046 int regionServersCount = 3; 5047 5048 try { 5049 StartTestingClusterOption option = StartTestingClusterOption.builder() 5050 .numRegionServers(regionServersCount).dataNodeHosts(dataNodeHosts).build(); 5051 cluster = htu.startMiniCluster(option); 5052 byte[][] families = { fam1, fam2 }; 5053 Table ht = htu.createTable(tableName, families); 5054 5055 // Setting up region 5056 byte row[] = Bytes.toBytes("row1"); 5057 byte col[] = Bytes.toBytes("col1"); 5058 5059 Put put = new Put(row); 5060 put.addColumn(fam1, col, 1, Bytes.toBytes("test1")); 5061 put.addColumn(fam2, col, 1, Bytes.toBytes("test2")); 5062 ht.put(put); 5063 5064 HRegion firstRegion = htu.getHBaseCluster().getRegions(tableName).get(0); 5065 firstRegion.flush(true); 5066 HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution(); 5067 5068 // Given the default replication factor is 2 and we have 2 HFiles, 5069 // we will have total of 4 replica of blocks on 3 datanodes; thus there 5070 // must be at least one host that have replica for 2 HFiles. That host's 5071 // weight will be equal to the unique block weight. 5072 long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight(); 5073 StringBuilder sb = new StringBuilder(); 5074 for (String host : blocksDistribution1.getTopHosts()) { 5075 if (sb.length() > 0) sb.append(", "); 5076 sb.append(host); 5077 sb.append("="); 5078 sb.append(blocksDistribution1.getWeight(host)); 5079 } 5080 5081 String topHost = blocksDistribution1.getTopHosts().get(0); 5082 long topHostWeight = blocksDistribution1.getWeight(topHost); 5083 String msg = "uniqueBlocksWeight=" + uniqueBlocksWeight1 + ", topHostWeight=" + topHostWeight 5084 + ", topHost=" + topHost + "; " + sb.toString(); 5085 LOG.info(msg); 5086 assertTrue(msg, uniqueBlocksWeight1 == topHostWeight); 5087 5088 // use the static method to compute the value, it should be the same. 5089 // static method is used by load balancer or other components 5090 HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution( 5091 htu.getConfiguration(), firstRegion.getTableDescriptor(), firstRegion.getRegionInfo()); 5092 long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight(); 5093 5094 assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2); 5095 5096 ht.close(); 5097 } finally { 5098 if (cluster != null) { 5099 htu.shutdownMiniCluster(); 5100 } 5101 } 5102 } 5103 5104 /** 5105 * Testcase to check state of region initialization task set to ABORTED or not if any exceptions 5106 * during initialization n 5107 */ 5108 @Test 5109 public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception { 5110 RegionInfo info; 5111 try { 5112 FileSystem fs = Mockito.mock(FileSystem.class); 5113 Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException()); 5114 TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(tableName); 5115 ColumnFamilyDescriptor columnFamilyDescriptor = 5116 ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf")).build(); 5117 tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor); 5118 info = RegionInfoBuilder.newBuilder(tableName).build(); 5119 Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization"); 5120 region = HRegion.newHRegion(path, null, fs, CONF, info, tableDescriptorBuilder.build(), null); 5121 // region initialization throws IOException and set task state to ABORTED. 5122 region.initialize(); 5123 fail("Region initialization should fail due to IOException"); 5124 } catch (IOException io) { 5125 List<MonitoredTask> tasks = TaskMonitor.get().getTasks(); 5126 for (MonitoredTask monitoredTask : tasks) { 5127 if ( 5128 !(monitoredTask instanceof MonitoredRPCHandler) 5129 && monitoredTask.getDescription().contains(region.toString()) 5130 ) { 5131 assertTrue("Region state should be ABORTED.", 5132 monitoredTask.getState().equals(MonitoredTask.State.ABORTED)); 5133 break; 5134 } 5135 } 5136 } 5137 } 5138 5139 /** 5140 * Verifies that the .regioninfo file is written on region creation and that is recreated if 5141 * missing during region opening. 5142 */ 5143 @Test 5144 public void testRegionInfoFileCreation() throws IOException { 5145 Path rootDir = new Path(dir + "testRegionInfoFileCreation"); 5146 5147 TableDescriptorBuilder tableDescriptorBuilder = 5148 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())); 5149 ColumnFamilyDescriptor columnFamilyDescriptor = 5150 ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf")).build(); 5151 tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor); 5152 TableDescriptor tableDescriptor = tableDescriptorBuilder.build(); 5153 5154 RegionInfo hri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 5155 5156 // Create a region and skip the initialization (like CreateTableHandler) 5157 region = HBaseTestingUtil.createRegionAndWAL(hri, rootDir, CONF, tableDescriptor, false); 5158 Path regionDir = region.getRegionFileSystem().getRegionDir(); 5159 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 5160 HBaseTestingUtil.closeRegionAndWAL(region); 5161 5162 Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE); 5163 5164 // Verify that the .regioninfo file is present 5165 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir", 5166 fs.exists(regionInfoFile)); 5167 5168 // Try to open the region 5169 region = HRegion.openHRegion(rootDir, hri, tableDescriptor, null, CONF); 5170 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir()); 5171 HBaseTestingUtil.closeRegionAndWAL(region); 5172 5173 // Verify that the .regioninfo file is still there 5174 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir", 5175 fs.exists(regionInfoFile)); 5176 5177 // Remove the .regioninfo file and verify is recreated on region open 5178 fs.delete(regionInfoFile, true); 5179 assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir", 5180 fs.exists(regionInfoFile)); 5181 5182 region = HRegion.openHRegion(rootDir, hri, tableDescriptor, null, CONF); 5183 // region = TEST_UTIL.openHRegion(hri, htd); 5184 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir()); 5185 HBaseTestingUtil.closeRegionAndWAL(region); 5186 5187 // Verify that the .regioninfo file is still there 5188 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir", 5189 fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE))); 5190 5191 region = null; 5192 } 5193 5194 /** 5195 * TestCase for increment 5196 */ 5197 private static class Incrementer implements Runnable { 5198 private HRegion region; 5199 private final static byte[] incRow = Bytes.toBytes("incRow"); 5200 private final static byte[] family = Bytes.toBytes("family"); 5201 private final static byte[] qualifier = Bytes.toBytes("qualifier"); 5202 private final static long ONE = 1L; 5203 private int incCounter; 5204 5205 public Incrementer(HRegion region, int incCounter) { 5206 this.region = region; 5207 this.incCounter = incCounter; 5208 } 5209 5210 @Override 5211 public void run() { 5212 int count = 0; 5213 while (count < incCounter) { 5214 Increment inc = new Increment(incRow); 5215 inc.addColumn(family, qualifier, ONE); 5216 count++; 5217 try { 5218 region.increment(inc); 5219 } catch (IOException e) { 5220 LOG.info("Count=" + count + ", " + e); 5221 break; 5222 } 5223 } 5224 } 5225 } 5226 5227 /** 5228 * Test case to check increment function with memstore flushing n 5229 */ 5230 @Test 5231 public void testParallelIncrementWithMemStoreFlush() throws Exception { 5232 byte[] family = Incrementer.family; 5233 this.region = initHRegion(tableName, method, CONF, family); 5234 final HRegion region = this.region; 5235 final AtomicBoolean incrementDone = new AtomicBoolean(false); 5236 Runnable flusher = new Runnable() { 5237 @Override 5238 public void run() { 5239 while (!incrementDone.get()) { 5240 try { 5241 region.flush(true); 5242 } catch (Exception e) { 5243 e.printStackTrace(); 5244 } 5245 } 5246 } 5247 }; 5248 5249 // after all increment finished, the row will increment to 20*100 = 2000 5250 int threadNum = 20; 5251 int incCounter = 100; 5252 long expected = (long) threadNum * incCounter; 5253 Thread[] incrementers = new Thread[threadNum]; 5254 Thread flushThread = new Thread(flusher); 5255 for (int i = 0; i < threadNum; i++) { 5256 incrementers[i] = new Thread(new Incrementer(this.region, incCounter)); 5257 incrementers[i].start(); 5258 } 5259 flushThread.start(); 5260 for (int i = 0; i < threadNum; i++) { 5261 incrementers[i].join(); 5262 } 5263 5264 incrementDone.set(true); 5265 flushThread.join(); 5266 5267 Get get = new Get(Incrementer.incRow); 5268 get.addColumn(Incrementer.family, Incrementer.qualifier); 5269 get.readVersions(1); 5270 Result res = this.region.get(get); 5271 List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier); 5272 5273 // we just got the latest version 5274 assertEquals(1, kvs.size()); 5275 Cell kv = kvs.get(0); 5276 assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset())); 5277 } 5278 5279 /** 5280 * TestCase for append 5281 */ 5282 private static class Appender implements Runnable { 5283 private HRegion region; 5284 private final static byte[] appendRow = Bytes.toBytes("appendRow"); 5285 private final static byte[] family = Bytes.toBytes("family"); 5286 private final static byte[] qualifier = Bytes.toBytes("qualifier"); 5287 private final static byte[] CHAR = Bytes.toBytes("a"); 5288 private int appendCounter; 5289 5290 public Appender(HRegion region, int appendCounter) { 5291 this.region = region; 5292 this.appendCounter = appendCounter; 5293 } 5294 5295 @Override 5296 public void run() { 5297 int count = 0; 5298 while (count < appendCounter) { 5299 Append app = new Append(appendRow); 5300 app.addColumn(family, qualifier, CHAR); 5301 count++; 5302 try { 5303 region.append(app); 5304 } catch (IOException e) { 5305 LOG.info("Count=" + count + ", max=" + appendCounter + ", " + e); 5306 break; 5307 } 5308 } 5309 } 5310 } 5311 5312 /** 5313 * Test case to check append function with memstore flushing n 5314 */ 5315 @Test 5316 public void testParallelAppendWithMemStoreFlush() throws Exception { 5317 byte[] family = Appender.family; 5318 this.region = initHRegion(tableName, method, CONF, family); 5319 final HRegion region = this.region; 5320 final AtomicBoolean appendDone = new AtomicBoolean(false); 5321 Runnable flusher = new Runnable() { 5322 @Override 5323 public void run() { 5324 while (!appendDone.get()) { 5325 try { 5326 region.flush(true); 5327 } catch (Exception e) { 5328 e.printStackTrace(); 5329 } 5330 } 5331 } 5332 }; 5333 5334 // After all append finished, the value will append to threadNum * 5335 // appendCounter Appender.CHAR 5336 int threadNum = 20; 5337 int appendCounter = 100; 5338 byte[] expected = new byte[threadNum * appendCounter]; 5339 for (int i = 0; i < threadNum * appendCounter; i++) { 5340 System.arraycopy(Appender.CHAR, 0, expected, i, 1); 5341 } 5342 Thread[] appenders = new Thread[threadNum]; 5343 Thread flushThread = new Thread(flusher); 5344 for (int i = 0; i < threadNum; i++) { 5345 appenders[i] = new Thread(new Appender(this.region, appendCounter)); 5346 appenders[i].start(); 5347 } 5348 flushThread.start(); 5349 for (int i = 0; i < threadNum; i++) { 5350 appenders[i].join(); 5351 } 5352 5353 appendDone.set(true); 5354 flushThread.join(); 5355 5356 Get get = new Get(Appender.appendRow); 5357 get.addColumn(Appender.family, Appender.qualifier); 5358 get.readVersions(1); 5359 Result res = this.region.get(get); 5360 List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier); 5361 5362 // we just got the latest version 5363 assertEquals(1, kvs.size()); 5364 Cell kv = kvs.get(0); 5365 byte[] appendResult = new byte[kv.getValueLength()]; 5366 System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength()); 5367 assertArrayEquals(expected, appendResult); 5368 } 5369 5370 /** 5371 * Test case to check put function with memstore flushing for same row, same ts n 5372 */ 5373 @Test 5374 public void testPutWithMemStoreFlush() throws Exception { 5375 byte[] family = Bytes.toBytes("family"); 5376 byte[] qualifier = Bytes.toBytes("qualifier"); 5377 byte[] row = Bytes.toBytes("putRow"); 5378 byte[] value = null; 5379 this.region = initHRegion(tableName, method, CONF, family); 5380 Put put = null; 5381 Get get = null; 5382 List<Cell> kvs = null; 5383 Result res = null; 5384 5385 put = new Put(row); 5386 value = Bytes.toBytes("value0"); 5387 put.addColumn(family, qualifier, 1234567L, value); 5388 region.put(put); 5389 get = new Get(row); 5390 get.addColumn(family, qualifier); 5391 get.readAllVersions(); 5392 res = this.region.get(get); 5393 kvs = res.getColumnCells(family, qualifier); 5394 assertEquals(1, kvs.size()); 5395 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0))); 5396 5397 region.flush(true); 5398 get = new Get(row); 5399 get.addColumn(family, qualifier); 5400 get.readAllVersions(); 5401 res = this.region.get(get); 5402 kvs = res.getColumnCells(family, qualifier); 5403 assertEquals(1, kvs.size()); 5404 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0))); 5405 5406 put = new Put(row); 5407 value = Bytes.toBytes("value1"); 5408 put.addColumn(family, qualifier, 1234567L, value); 5409 region.put(put); 5410 get = new Get(row); 5411 get.addColumn(family, qualifier); 5412 get.readAllVersions(); 5413 res = this.region.get(get); 5414 kvs = res.getColumnCells(family, qualifier); 5415 assertEquals(1, kvs.size()); 5416 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0))); 5417 5418 region.flush(true); 5419 get = new Get(row); 5420 get.addColumn(family, qualifier); 5421 get.readAllVersions(); 5422 res = this.region.get(get); 5423 kvs = res.getColumnCells(family, qualifier); 5424 assertEquals(1, kvs.size()); 5425 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0))); 5426 } 5427 5428 @Test 5429 public void testDurability() throws Exception { 5430 // there are 5 x 5 cases: 5431 // table durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT) x mutation 5432 // durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT) 5433 5434 // expected cases for append and sync wal 5435 durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false); 5436 durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false); 5437 durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false); 5438 5439 durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false); 5440 durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false); 5441 durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false); 5442 5443 durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false); 5444 durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false); 5445 5446 durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false); 5447 durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false); 5448 5449 durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false); 5450 durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false); 5451 durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false); 5452 5453 // expected cases for async wal 5454 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false); 5455 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false); 5456 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false); 5457 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false); 5458 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false); 5459 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false); 5460 5461 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true); 5462 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true); 5463 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true); 5464 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true); 5465 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true); 5466 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true); 5467 5468 // expect skip wal cases 5469 durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false); 5470 durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false); 5471 durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false); 5472 durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false); 5473 durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false); 5474 durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false); 5475 5476 } 5477 5478 private void durabilityTest(String method, Durability tableDurability, 5479 Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync, 5480 final boolean expectSyncFromLogSyncer) throws Exception { 5481 Configuration conf = HBaseConfiguration.create(CONF); 5482 method = method + "_" + tableDurability.name() + "_" + mutationDurability.name(); 5483 byte[] family = Bytes.toBytes("family"); 5484 Path logDir = new Path(new Path(dir + method), "log"); 5485 final Configuration walConf = new Configuration(conf); 5486 CommonFSUtils.setRootDir(walConf, logDir); 5487 // XXX: The spied AsyncFSWAL can not work properly because of a Mockito defect that can not 5488 // deal with classes which have a field of an inner class. See discussions in HBASE-15536. 5489 walConf.set(WALFactory.WAL_PROVIDER, "filesystem"); 5490 final WALFactory wals = new WALFactory(walConf, HBaseTestingUtil.getRandomUUID().toString()); 5491 final WAL wal = spy(wals.getWAL(RegionInfoBuilder.newBuilder(tableName).build())); 5492 this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF, 5493 false, tableDurability, wal, new byte[][] { family }); 5494 5495 Put put = new Put(Bytes.toBytes("r1")); 5496 put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); 5497 put.setDurability(mutationDurability); 5498 region.put(put); 5499 5500 // verify append called or not 5501 verify(wal, expectAppend ? times(1) : never()).appendData((RegionInfo) any(), 5502 (WALKeyImpl) any(), (WALEdit) any()); 5503 5504 // verify sync called or not 5505 if (expectSync || expectSyncFromLogSyncer) { 5506 TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() { 5507 @Override 5508 public boolean evaluate() throws Exception { 5509 try { 5510 if (expectSync) { 5511 verify(wal, times(1)).sync(anyLong()); // Hregion calls this one 5512 } else if (expectSyncFromLogSyncer) { 5513 verify(wal, times(1)).sync(); // wal syncer calls this one 5514 } 5515 } catch (Throwable ignore) { 5516 } 5517 return true; 5518 } 5519 }); 5520 } else { 5521 // verify(wal, never()).sync(anyLong()); 5522 verify(wal, never()).sync(); 5523 } 5524 5525 HBaseTestingUtil.closeRegionAndWAL(this.region); 5526 wals.close(); 5527 this.region = null; 5528 } 5529 5530 @Test 5531 public void testRegionReplicaSecondary() throws IOException { 5532 // create a primary region, load some data and flush 5533 // create a secondary region, and do a get against that 5534 Path rootDir = new Path(dir + name.getMethodName()); 5535 CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir); 5536 5537 byte[][] families = 5538 new byte[][] { Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3") }; 5539 byte[] cq = Bytes.toBytes("cq"); 5540 TableDescriptorBuilder builder = 5541 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())); 5542 for (byte[] family : families) { 5543 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)); 5544 } 5545 TableDescriptor tableDescriptor = builder.build(); 5546 long time = EnvironmentEdgeManager.currentTime(); 5547 RegionInfo primaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5548 .setRegionId(time).setReplicaId(0).build(); 5549 RegionInfo secondaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5550 .setRegionId(time).setReplicaId(1).build(); 5551 5552 HRegion primaryRegion = null, secondaryRegion = null; 5553 5554 try { 5555 primaryRegion = HBaseTestingUtil.createRegionAndWAL(primaryHri, rootDir, 5556 TEST_UTIL.getConfiguration(), tableDescriptor); 5557 5558 // load some data 5559 putData(primaryRegion, 0, 1000, cq, families); 5560 5561 // flush region 5562 primaryRegion.flush(true); 5563 5564 // open secondary region 5565 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, tableDescriptor, null, CONF); 5566 5567 verifyData(secondaryRegion, 0, 1000, cq, families); 5568 } finally { 5569 if (primaryRegion != null) { 5570 HBaseTestingUtil.closeRegionAndWAL(primaryRegion); 5571 } 5572 if (secondaryRegion != null) { 5573 HBaseTestingUtil.closeRegionAndWAL(secondaryRegion); 5574 } 5575 } 5576 } 5577 5578 @Test 5579 public void testRegionReplicaSecondaryIsReadOnly() throws IOException { 5580 // create a primary region, load some data and flush 5581 // create a secondary region, and do a put against that 5582 Path rootDir = new Path(dir + name.getMethodName()); 5583 CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir); 5584 5585 byte[][] families = 5586 new byte[][] { Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3") }; 5587 byte[] cq = Bytes.toBytes("cq"); 5588 TableDescriptorBuilder builder = 5589 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())); 5590 for (byte[] family : families) { 5591 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)); 5592 } 5593 TableDescriptor tableDescriptor = builder.build(); 5594 long time = EnvironmentEdgeManager.currentTime(); 5595 RegionInfo primaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5596 .setRegionId(time).setReplicaId(0).build(); 5597 RegionInfo secondaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5598 .setRegionId(time).setReplicaId(1).build(); 5599 5600 HRegion primaryRegion = null, secondaryRegion = null; 5601 5602 try { 5603 primaryRegion = HBaseTestingUtil.createRegionAndWAL(primaryHri, rootDir, 5604 TEST_UTIL.getConfiguration(), tableDescriptor); 5605 5606 // load some data 5607 putData(primaryRegion, 0, 1000, cq, families); 5608 5609 // flush region 5610 primaryRegion.flush(true); 5611 5612 // open secondary region 5613 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, tableDescriptor, null, CONF); 5614 5615 try { 5616 putData(secondaryRegion, 0, 1000, cq, families); 5617 fail("Should have thrown exception"); 5618 } catch (IOException ex) { 5619 // expected 5620 } 5621 } finally { 5622 if (primaryRegion != null) { 5623 HBaseTestingUtil.closeRegionAndWAL(primaryRegion); 5624 } 5625 if (secondaryRegion != null) { 5626 HBaseTestingUtil.closeRegionAndWAL(secondaryRegion); 5627 } 5628 } 5629 } 5630 5631 static WALFactory createWALFactory(Configuration conf, Path rootDir) throws IOException { 5632 Configuration confForWAL = new Configuration(conf); 5633 confForWAL.set(HConstants.HBASE_DIR, rootDir.toString()); 5634 return new WALFactory(confForWAL, "hregion-" + RandomStringUtils.randomNumeric(8)); 5635 } 5636 5637 @Test 5638 public void testCompactionFromPrimary() throws IOException { 5639 Path rootDir = new Path(dir + name.getMethodName()); 5640 CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir); 5641 5642 byte[][] families = 5643 new byte[][] { Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3") }; 5644 byte[] cq = Bytes.toBytes("cq"); 5645 TableDescriptorBuilder builder = 5646 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())); 5647 for (byte[] family : families) { 5648 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)); 5649 } 5650 TableDescriptor tableDescriptor = builder.build(); 5651 long time = EnvironmentEdgeManager.currentTime(); 5652 RegionInfo primaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5653 .setRegionId(time).setReplicaId(0).build(); 5654 RegionInfo secondaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5655 .setRegionId(time).setReplicaId(1).build(); 5656 5657 HRegion primaryRegion = null, secondaryRegion = null; 5658 5659 try { 5660 primaryRegion = HBaseTestingUtil.createRegionAndWAL(primaryHri, rootDir, 5661 TEST_UTIL.getConfiguration(), tableDescriptor); 5662 5663 // load some data 5664 putData(primaryRegion, 0, 1000, cq, families); 5665 5666 // flush region 5667 primaryRegion.flush(true); 5668 5669 // open secondary region 5670 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, tableDescriptor, null, CONF); 5671 5672 // move the file of the primary region to the archive, simulating a compaction 5673 Collection<HStoreFile> storeFiles = primaryRegion.getStore(families[0]).getStorefiles(); 5674 primaryRegion.getRegionFileSystem().removeStoreFiles(Bytes.toString(families[0]), storeFiles); 5675 Collection<StoreFileInfo> storeFileInfos = 5676 primaryRegion.getRegionFileSystem().getStoreFiles(Bytes.toString(families[0])); 5677 Assert.assertTrue(storeFileInfos == null || storeFileInfos.isEmpty()); 5678 5679 verifyData(secondaryRegion, 0, 1000, cq, families); 5680 } finally { 5681 if (primaryRegion != null) { 5682 HBaseTestingUtil.closeRegionAndWAL(primaryRegion); 5683 } 5684 if (secondaryRegion != null) { 5685 HBaseTestingUtil.closeRegionAndWAL(secondaryRegion); 5686 } 5687 } 5688 } 5689 5690 private void putData(int startRow, int numRows, byte[] qf, byte[]... families) 5691 throws IOException { 5692 putData(this.region, startRow, numRows, qf, families); 5693 } 5694 5695 private void putData(HRegion region, int startRow, int numRows, byte[] qf, byte[]... families) 5696 throws IOException { 5697 putData(region, Durability.SKIP_WAL, startRow, numRows, qf, families); 5698 } 5699 5700 static void putData(HRegion region, Durability durability, int startRow, int numRows, byte[] qf, 5701 byte[]... families) throws IOException { 5702 for (int i = startRow; i < startRow + numRows; i++) { 5703 Put put = new Put(Bytes.toBytes("" + i)); 5704 put.setDurability(durability); 5705 for (byte[] family : families) { 5706 put.addColumn(family, qf, null); 5707 } 5708 region.put(put); 5709 LOG.info(put.toString()); 5710 } 5711 } 5712 5713 static void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families) 5714 throws IOException { 5715 for (int i = startRow; i < startRow + numRows; i++) { 5716 byte[] row = Bytes.toBytes("" + i); 5717 Get get = new Get(row); 5718 for (byte[] family : families) { 5719 get.addColumn(family, qf); 5720 } 5721 Result result = newReg.get(get); 5722 Cell[] raw = result.rawCells(); 5723 assertEquals(families.length, result.size()); 5724 for (int j = 0; j < families.length; j++) { 5725 assertTrue(CellUtil.matchingRows(raw[j], row)); 5726 assertTrue(CellUtil.matchingFamily(raw[j], families[j])); 5727 assertTrue(CellUtil.matchingQualifier(raw[j], qf)); 5728 } 5729 } 5730 } 5731 5732 static void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException { 5733 // Now I have k, get values out and assert they are as expected. 5734 Get get = new Get(k).addFamily(family).readAllVersions(); 5735 Cell[] results = r.get(get).rawCells(); 5736 for (int j = 0; j < results.length; j++) { 5737 byte[] tmp = CellUtil.cloneValue(results[j]); 5738 // Row should be equal to value every time. 5739 assertTrue(Bytes.equals(k, tmp)); 5740 } 5741 } 5742 5743 /* 5744 * Assert first value in the passed region is <code>firstValue</code>. n * n * n * n 5745 */ 5746 protected void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue) 5747 throws IOException { 5748 byte[][] families = { fs }; 5749 Scan scan = new Scan(); 5750 for (int i = 0; i < families.length; i++) 5751 scan.addFamily(families[i]); 5752 InternalScanner s = r.getScanner(scan); 5753 try { 5754 List<Cell> curVals = new ArrayList<>(); 5755 boolean first = true; 5756 OUTER_LOOP: while (s.next(curVals)) { 5757 for (Cell kv : curVals) { 5758 byte[] val = CellUtil.cloneValue(kv); 5759 byte[] curval = val; 5760 if (first) { 5761 first = false; 5762 assertTrue(Bytes.compareTo(curval, firstValue) == 0); 5763 } else { 5764 // Not asserting anything. Might as well break. 5765 break OUTER_LOOP; 5766 } 5767 } 5768 } 5769 } finally { 5770 s.close(); 5771 } 5772 } 5773 5774 /** 5775 * Test that we get the expected flush results back 5776 */ 5777 @Test 5778 public void testFlushResult() throws IOException { 5779 byte[] family = Bytes.toBytes("family"); 5780 5781 this.region = initHRegion(tableName, method, family); 5782 5783 // empty memstore, flush doesn't run 5784 HRegion.FlushResult fr = region.flush(true); 5785 assertFalse(fr.isFlushSucceeded()); 5786 assertFalse(fr.isCompactionNeeded()); 5787 5788 // Flush enough files to get up to the threshold, doesn't need compactions 5789 for (int i = 0; i < 2; i++) { 5790 Put put = new Put(tableName.toBytes()).addColumn(family, family, tableName.toBytes()); 5791 region.put(put); 5792 fr = region.flush(true); 5793 assertTrue(fr.isFlushSucceeded()); 5794 assertFalse(fr.isCompactionNeeded()); 5795 } 5796 5797 // Two flushes after the threshold, compactions are needed 5798 for (int i = 0; i < 2; i++) { 5799 Put put = new Put(tableName.toBytes()).addColumn(family, family, tableName.toBytes()); 5800 region.put(put); 5801 fr = region.flush(true); 5802 assertTrue(fr.isFlushSucceeded()); 5803 assertTrue(fr.isCompactionNeeded()); 5804 } 5805 } 5806 5807 protected Configuration initSplit() { 5808 // Always compact if there is more than one store file. 5809 CONF.setInt("hbase.hstore.compactionThreshold", 2); 5810 5811 CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000); 5812 5813 // Increase the amount of time between client retries 5814 CONF.setLong("hbase.client.pause", 15 * 1000); 5815 5816 // This size should make it so we always split using the addContent 5817 // below. After adding all data, the first region is 1.3M 5818 CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128); 5819 return CONF; 5820 } 5821 5822 /** 5823 * @return A region on which you must call {@link HBaseTestingUtil#closeRegionAndWAL(HRegion)} 5824 * when done. 5825 */ 5826 protected HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf, 5827 byte[]... families) throws IOException { 5828 return initHRegion(tableName, callingMethod, conf, false, families); 5829 } 5830 5831 /** 5832 * @return A region on which you must call {@link HBaseTestingUtil#closeRegionAndWAL(HRegion)} 5833 * when done. 5834 */ 5835 protected HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf, 5836 boolean isReadOnly, byte[]... families) throws IOException { 5837 return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families); 5838 } 5839 5840 protected HRegion initHRegion(TableName tableName, byte[] startKey, byte[] stopKey, 5841 String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families) 5842 throws IOException { 5843 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(callingMethod + ".log"); 5844 RegionInfo hri = 5845 RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey).setEndKey(stopKey).build(); 5846 final WAL wal = HBaseTestingUtil.createWal(conf, logDir, hri); 5847 return initHRegion(tableName, startKey, stopKey, conf, isReadOnly, Durability.SYNC_WAL, wal, 5848 families); 5849 } 5850 5851 /** 5852 * @return A region on which you must call {@link HBaseTestingUtil#closeRegionAndWAL(HRegion)} 5853 * when done. 5854 */ 5855 public HRegion initHRegion(TableName tableName, byte[] startKey, byte[] stopKey, 5856 Configuration conf, boolean isReadOnly, Durability durability, WAL wal, byte[]... families) 5857 throws IOException { 5858 ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null, 5859 MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT); 5860 return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, conf, isReadOnly, durability, 5861 wal, families); 5862 } 5863 5864 /** 5865 * Assert that the passed in Cell has expected contents for the specified row, column & timestamp. 5866 */ 5867 private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) { 5868 String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts; 5869 assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx, 5870 Bytes.toString(CellUtil.cloneRow(kv))); 5871 assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf), 5872 Bytes.toString(CellUtil.cloneFamily(kv))); 5873 assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx, 5874 Bytes.toString(CellUtil.cloneQualifier(kv))); 5875 assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp()); 5876 assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts, 5877 Bytes.toString(CellUtil.cloneValue(kv))); 5878 } 5879 5880 @Test 5881 public void testReverseScanner_FromMemStore_SingleCF_Normal() throws IOException { 5882 byte[] rowC = Bytes.toBytes("rowC"); 5883 byte[] rowA = Bytes.toBytes("rowA"); 5884 byte[] rowB = Bytes.toBytes("rowB"); 5885 byte[] cf = Bytes.toBytes("CF"); 5886 byte[][] families = { cf }; 5887 byte[] col = Bytes.toBytes("C"); 5888 long ts = 1; 5889 this.region = initHRegion(tableName, method, families); 5890 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null); 5891 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, null); 5892 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null); 5893 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null); 5894 Put put = null; 5895 put = new Put(rowC); 5896 put.add(kv1); 5897 put.add(kv11); 5898 region.put(put); 5899 put = new Put(rowA); 5900 put.add(kv2); 5901 region.put(put); 5902 put = new Put(rowB); 5903 put.add(kv3); 5904 region.put(put); 5905 5906 Scan scan = new Scan().withStartRow(rowC); 5907 scan.readVersions(5); 5908 scan.setReversed(true); 5909 InternalScanner scanner = region.getScanner(scan); 5910 List<Cell> currRow = new ArrayList<>(); 5911 boolean hasNext = scanner.next(currRow); 5912 assertEquals(2, currRow.size()); 5913 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5914 currRow.get(0).getRowLength(), rowC, 0, rowC.length)); 5915 assertTrue(hasNext); 5916 currRow.clear(); 5917 hasNext = scanner.next(currRow); 5918 assertEquals(1, currRow.size()); 5919 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5920 currRow.get(0).getRowLength(), rowB, 0, rowB.length)); 5921 assertTrue(hasNext); 5922 currRow.clear(); 5923 hasNext = scanner.next(currRow); 5924 assertEquals(1, currRow.size()); 5925 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5926 currRow.get(0).getRowLength(), rowA, 0, rowA.length)); 5927 assertFalse(hasNext); 5928 scanner.close(); 5929 } 5930 5931 @Test 5932 public void testReverseScanner_FromMemStore_SingleCF_LargerKey() throws IOException { 5933 byte[] rowC = Bytes.toBytes("rowC"); 5934 byte[] rowA = Bytes.toBytes("rowA"); 5935 byte[] rowB = Bytes.toBytes("rowB"); 5936 byte[] rowD = Bytes.toBytes("rowD"); 5937 byte[] cf = Bytes.toBytes("CF"); 5938 byte[][] families = { cf }; 5939 byte[] col = Bytes.toBytes("C"); 5940 long ts = 1; 5941 this.region = initHRegion(tableName, method, families); 5942 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null); 5943 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, null); 5944 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null); 5945 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null); 5946 Put put = null; 5947 put = new Put(rowC); 5948 put.add(kv1); 5949 put.add(kv11); 5950 region.put(put); 5951 put = new Put(rowA); 5952 put.add(kv2); 5953 region.put(put); 5954 put = new Put(rowB); 5955 put.add(kv3); 5956 region.put(put); 5957 5958 Scan scan = new Scan().withStartRow(rowD); 5959 List<Cell> currRow = new ArrayList<>(); 5960 scan.setReversed(true); 5961 scan.readVersions(5); 5962 InternalScanner scanner = region.getScanner(scan); 5963 boolean hasNext = scanner.next(currRow); 5964 assertEquals(2, currRow.size()); 5965 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5966 currRow.get(0).getRowLength(), rowC, 0, rowC.length)); 5967 assertTrue(hasNext); 5968 currRow.clear(); 5969 hasNext = scanner.next(currRow); 5970 assertEquals(1, currRow.size()); 5971 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5972 currRow.get(0).getRowLength(), rowB, 0, rowB.length)); 5973 assertTrue(hasNext); 5974 currRow.clear(); 5975 hasNext = scanner.next(currRow); 5976 assertEquals(1, currRow.size()); 5977 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5978 currRow.get(0).getRowLength(), rowA, 0, rowA.length)); 5979 assertFalse(hasNext); 5980 scanner.close(); 5981 } 5982 5983 @Test 5984 public void testReverseScanner_FromMemStore_SingleCF_FullScan() throws IOException { 5985 byte[] rowC = Bytes.toBytes("rowC"); 5986 byte[] rowA = Bytes.toBytes("rowA"); 5987 byte[] rowB = Bytes.toBytes("rowB"); 5988 byte[] cf = Bytes.toBytes("CF"); 5989 byte[][] families = { cf }; 5990 byte[] col = Bytes.toBytes("C"); 5991 long ts = 1; 5992 this.region = initHRegion(tableName, method, families); 5993 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null); 5994 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, null); 5995 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null); 5996 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null); 5997 Put put = null; 5998 put = new Put(rowC); 5999 put.add(kv1); 6000 put.add(kv11); 6001 region.put(put); 6002 put = new Put(rowA); 6003 put.add(kv2); 6004 region.put(put); 6005 put = new Put(rowB); 6006 put.add(kv3); 6007 region.put(put); 6008 Scan scan = new Scan(); 6009 List<Cell> currRow = new ArrayList<>(); 6010 scan.setReversed(true); 6011 InternalScanner scanner = region.getScanner(scan); 6012 boolean hasNext = scanner.next(currRow); 6013 assertEquals(1, currRow.size()); 6014 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6015 currRow.get(0).getRowLength(), rowC, 0, rowC.length)); 6016 assertTrue(hasNext); 6017 currRow.clear(); 6018 hasNext = scanner.next(currRow); 6019 assertEquals(1, currRow.size()); 6020 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6021 currRow.get(0).getRowLength(), rowB, 0, rowB.length)); 6022 assertTrue(hasNext); 6023 currRow.clear(); 6024 hasNext = scanner.next(currRow); 6025 assertEquals(1, currRow.size()); 6026 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6027 currRow.get(0).getRowLength(), rowA, 0, rowA.length)); 6028 assertFalse(hasNext); 6029 scanner.close(); 6030 } 6031 6032 @Test 6033 public void testReverseScanner_moreRowsMayExistAfter() throws IOException { 6034 // case for "INCLUDE_AND_SEEK_NEXT_ROW & SEEK_NEXT_ROW" endless loop 6035 byte[] rowA = Bytes.toBytes("rowA"); 6036 byte[] rowB = Bytes.toBytes("rowB"); 6037 byte[] rowC = Bytes.toBytes("rowC"); 6038 byte[] rowD = Bytes.toBytes("rowD"); 6039 byte[] rowE = Bytes.toBytes("rowE"); 6040 byte[] cf = Bytes.toBytes("CF"); 6041 byte[][] families = { cf }; 6042 byte[] col1 = Bytes.toBytes("col1"); 6043 byte[] col2 = Bytes.toBytes("col2"); 6044 long ts = 1; 6045 this.region = initHRegion(tableName, method, families); 6046 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null); 6047 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null); 6048 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null); 6049 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null); 6050 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null); 6051 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null); 6052 Put put = null; 6053 put = new Put(rowA); 6054 put.add(kv1); 6055 region.put(put); 6056 put = new Put(rowB); 6057 put.add(kv2); 6058 region.put(put); 6059 put = new Put(rowC); 6060 put.add(kv3); 6061 region.put(put); 6062 put = new Put(rowD); 6063 put.add(kv4_1); 6064 region.put(put); 6065 put = new Put(rowD); 6066 put.add(kv4_2); 6067 region.put(put); 6068 put = new Put(rowE); 6069 put.add(kv5); 6070 region.put(put); 6071 region.flush(true); 6072 Scan scan = new Scan().withStartRow(rowD).withStopRow(rowA); 6073 scan.addColumn(families[0], col1); 6074 scan.setReversed(true); 6075 List<Cell> currRow = new ArrayList<>(); 6076 InternalScanner scanner = region.getScanner(scan); 6077 boolean hasNext = scanner.next(currRow); 6078 assertEquals(1, currRow.size()); 6079 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6080 currRow.get(0).getRowLength(), rowD, 0, rowD.length)); 6081 assertTrue(hasNext); 6082 currRow.clear(); 6083 hasNext = scanner.next(currRow); 6084 assertEquals(1, currRow.size()); 6085 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6086 currRow.get(0).getRowLength(), rowC, 0, rowC.length)); 6087 assertTrue(hasNext); 6088 currRow.clear(); 6089 hasNext = scanner.next(currRow); 6090 assertEquals(1, currRow.size()); 6091 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6092 currRow.get(0).getRowLength(), rowB, 0, rowB.length)); 6093 assertFalse(hasNext); 6094 scanner.close(); 6095 6096 scan = new Scan().withStartRow(rowD).withStopRow(rowA); 6097 scan.addColumn(families[0], col2); 6098 scan.setReversed(true); 6099 currRow.clear(); 6100 scanner = region.getScanner(scan); 6101 hasNext = scanner.next(currRow); 6102 assertEquals(1, currRow.size()); 6103 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6104 currRow.get(0).getRowLength(), rowD, 0, rowD.length)); 6105 scanner.close(); 6106 } 6107 6108 @Test 6109 public void testReverseScanner_smaller_blocksize() throws IOException { 6110 // case to ensure no conflict with HFile index optimization 6111 byte[] rowA = Bytes.toBytes("rowA"); 6112 byte[] rowB = Bytes.toBytes("rowB"); 6113 byte[] rowC = Bytes.toBytes("rowC"); 6114 byte[] rowD = Bytes.toBytes("rowD"); 6115 byte[] rowE = Bytes.toBytes("rowE"); 6116 byte[] cf = Bytes.toBytes("CF"); 6117 byte[][] families = { cf }; 6118 byte[] col1 = Bytes.toBytes("col1"); 6119 byte[] col2 = Bytes.toBytes("col2"); 6120 long ts = 1; 6121 Configuration conf = new Configuration(CONF); 6122 conf.setInt("test.block.size", 1); 6123 this.region = initHRegion(tableName, method, conf, families); 6124 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null); 6125 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null); 6126 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null); 6127 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null); 6128 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null); 6129 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null); 6130 Put put = null; 6131 put = new Put(rowA); 6132 put.add(kv1); 6133 region.put(put); 6134 put = new Put(rowB); 6135 put.add(kv2); 6136 region.put(put); 6137 put = new Put(rowC); 6138 put.add(kv3); 6139 region.put(put); 6140 put = new Put(rowD); 6141 put.add(kv4_1); 6142 region.put(put); 6143 put = new Put(rowD); 6144 put.add(kv4_2); 6145 region.put(put); 6146 put = new Put(rowE); 6147 put.add(kv5); 6148 region.put(put); 6149 region.flush(true); 6150 Scan scan = new Scan().withStartRow(rowD).withStopRow(rowA); 6151 scan.addColumn(families[0], col1); 6152 scan.setReversed(true); 6153 List<Cell> currRow = new ArrayList<>(); 6154 InternalScanner scanner = region.getScanner(scan); 6155 boolean hasNext = scanner.next(currRow); 6156 assertEquals(1, currRow.size()); 6157 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6158 currRow.get(0).getRowLength(), rowD, 0, rowD.length)); 6159 assertTrue(hasNext); 6160 currRow.clear(); 6161 hasNext = scanner.next(currRow); 6162 assertEquals(1, currRow.size()); 6163 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6164 currRow.get(0).getRowLength(), rowC, 0, rowC.length)); 6165 assertTrue(hasNext); 6166 currRow.clear(); 6167 hasNext = scanner.next(currRow); 6168 assertEquals(1, currRow.size()); 6169 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6170 currRow.get(0).getRowLength(), rowB, 0, rowB.length)); 6171 assertFalse(hasNext); 6172 scanner.close(); 6173 6174 scan = new Scan().withStartRow(rowD).withStopRow(rowA); 6175 scan.addColumn(families[0], col2); 6176 scan.setReversed(true); 6177 currRow.clear(); 6178 scanner = region.getScanner(scan); 6179 hasNext = scanner.next(currRow); 6180 assertEquals(1, currRow.size()); 6181 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6182 currRow.get(0).getRowLength(), rowD, 0, rowD.length)); 6183 scanner.close(); 6184 } 6185 6186 @Test 6187 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1() throws IOException { 6188 byte[] row0 = Bytes.toBytes("row0"); // 1 kv 6189 byte[] row1 = Bytes.toBytes("row1"); // 2 kv 6190 byte[] row2 = Bytes.toBytes("row2"); // 4 kv 6191 byte[] row3 = Bytes.toBytes("row3"); // 2 kv 6192 byte[] row4 = Bytes.toBytes("row4"); // 5 kv 6193 byte[] row5 = Bytes.toBytes("row5"); // 2 kv 6194 byte[] cf1 = Bytes.toBytes("CF1"); 6195 byte[] cf2 = Bytes.toBytes("CF2"); 6196 byte[] cf3 = Bytes.toBytes("CF3"); 6197 byte[][] families = { cf1, cf2, cf3 }; 6198 byte[] col = Bytes.toBytes("C"); 6199 long ts = 1; 6200 Configuration conf = new Configuration(CONF); 6201 // disable compactions in this test. 6202 conf.setInt("hbase.hstore.compactionThreshold", 10000); 6203 this.region = initHRegion(tableName, method, conf, families); 6204 // kv naming style: kv(row number) totalKvCountInThisRow seq no 6205 KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put, null); 6206 KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put, null); 6207 KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1, KeyValue.Type.Put, null); 6208 KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null); 6209 KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put, null); 6210 KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put, null); 6211 KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4, KeyValue.Type.Put, null); 6212 KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put, null); 6213 KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4, KeyValue.Type.Put, null); 6214 KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put, null); 6215 KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put, null); 6216 KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5, KeyValue.Type.Put, null); 6217 KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put, null); 6218 KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3, KeyValue.Type.Put, null); 6219 KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put, null); 6220 KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put, null); 6221 // hfiles(cf1/cf2) :"row1"(1 kv) / "row2"(1 kv) / "row4"(2 kv) 6222 Put put = null; 6223 put = new Put(row1); 6224 put.add(kv1_2_1); 6225 region.put(put); 6226 put = new Put(row2); 6227 put.add(kv2_4_1); 6228 region.put(put); 6229 put = new Put(row4); 6230 put.add(kv4_5_4); 6231 put.add(kv4_5_5); 6232 region.put(put); 6233 region.flush(true); 6234 // hfiles(cf1/cf3) : "row1" (1 kvs) / "row2" (1 kv) / "row4" (2 kv) 6235 put = new Put(row4); 6236 put.add(kv4_5_1); 6237 put.add(kv4_5_3); 6238 region.put(put); 6239 put = new Put(row1); 6240 put.add(kv1_2_2); 6241 region.put(put); 6242 put = new Put(row2); 6243 put.add(kv2_4_4); 6244 region.put(put); 6245 region.flush(true); 6246 // hfiles(cf1/cf3) : "row2"(2 kv) / "row3"(1 kvs) / "row4" (1 kv) 6247 put = new Put(row4); 6248 put.add(kv4_5_2); 6249 region.put(put); 6250 put = new Put(row2); 6251 put.add(kv2_4_2); 6252 put.add(kv2_4_3); 6253 region.put(put); 6254 put = new Put(row3); 6255 put.add(kv3_2_2); 6256 region.put(put); 6257 region.flush(true); 6258 // memstore(cf1/cf2/cf3) : "row0" (1 kvs) / "row3" ( 1 kv) / "row5" (max) 6259 // ( 2 kv) 6260 put = new Put(row0); 6261 put.add(kv0_1_1); 6262 region.put(put); 6263 put = new Put(row3); 6264 put.add(kv3_2_1); 6265 region.put(put); 6266 put = new Put(row5); 6267 put.add(kv5_2_1); 6268 put.add(kv5_2_2); 6269 region.put(put); 6270 // scan range = ["row4", min), skip the max "row5" 6271 Scan scan = new Scan().withStartRow(row4); 6272 scan.readVersions(5); 6273 scan.setBatch(3); 6274 scan.setReversed(true); 6275 InternalScanner scanner = region.getScanner(scan); 6276 List<Cell> currRow = new ArrayList<>(); 6277 boolean hasNext = false; 6278 // 1. scan out "row4" (5 kvs), "row5" can't be scanned out since not 6279 // included in scan range 6280 // "row4" takes 2 next() calls since batch=3 6281 hasNext = scanner.next(currRow); 6282 assertEquals(3, currRow.size()); 6283 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6284 currRow.get(0).getRowLength(), row4, 0, row4.length)); 6285 assertTrue(hasNext); 6286 currRow.clear(); 6287 hasNext = scanner.next(currRow); 6288 assertEquals(2, currRow.size()); 6289 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6290 currRow.get(0).getRowLength(), row4, 0, row4.length)); 6291 assertTrue(hasNext); 6292 // 2. scan out "row3" (2 kv) 6293 currRow.clear(); 6294 hasNext = scanner.next(currRow); 6295 assertEquals(2, currRow.size()); 6296 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6297 currRow.get(0).getRowLength(), row3, 0, row3.length)); 6298 assertTrue(hasNext); 6299 // 3. scan out "row2" (4 kvs) 6300 // "row2" takes 2 next() calls since batch=3 6301 currRow.clear(); 6302 hasNext = scanner.next(currRow); 6303 assertEquals(3, currRow.size()); 6304 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6305 currRow.get(0).getRowLength(), row2, 0, row2.length)); 6306 assertTrue(hasNext); 6307 currRow.clear(); 6308 hasNext = scanner.next(currRow); 6309 assertEquals(1, currRow.size()); 6310 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6311 currRow.get(0).getRowLength(), row2, 0, row2.length)); 6312 assertTrue(hasNext); 6313 // 4. scan out "row1" (2 kv) 6314 currRow.clear(); 6315 hasNext = scanner.next(currRow); 6316 assertEquals(2, currRow.size()); 6317 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6318 currRow.get(0).getRowLength(), row1, 0, row1.length)); 6319 assertTrue(hasNext); 6320 // 5. scan out "row0" (1 kv) 6321 currRow.clear(); 6322 hasNext = scanner.next(currRow); 6323 assertEquals(1, currRow.size()); 6324 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6325 currRow.get(0).getRowLength(), row0, 0, row0.length)); 6326 assertFalse(hasNext); 6327 6328 scanner.close(); 6329 } 6330 6331 @Test 6332 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2() throws IOException { 6333 byte[] row1 = Bytes.toBytes("row1"); 6334 byte[] row2 = Bytes.toBytes("row2"); 6335 byte[] row3 = Bytes.toBytes("row3"); 6336 byte[] row4 = Bytes.toBytes("row4"); 6337 byte[] cf1 = Bytes.toBytes("CF1"); 6338 byte[] cf2 = Bytes.toBytes("CF2"); 6339 byte[] cf3 = Bytes.toBytes("CF3"); 6340 byte[] cf4 = Bytes.toBytes("CF4"); 6341 byte[][] families = { cf1, cf2, cf3, cf4 }; 6342 byte[] col = Bytes.toBytes("C"); 6343 long ts = 1; 6344 Configuration conf = new Configuration(CONF); 6345 // disable compactions in this test. 6346 conf.setInt("hbase.hstore.compactionThreshold", 10000); 6347 this.region = initHRegion(tableName, method, conf, families); 6348 KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null); 6349 KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null); 6350 KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null); 6351 KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null); 6352 // storefile1 6353 Put put = new Put(row1); 6354 put.add(kv1); 6355 region.put(put); 6356 region.flush(true); 6357 // storefile2 6358 put = new Put(row2); 6359 put.add(kv2); 6360 region.put(put); 6361 region.flush(true); 6362 // storefile3 6363 put = new Put(row3); 6364 put.add(kv3); 6365 region.put(put); 6366 region.flush(true); 6367 // memstore 6368 put = new Put(row4); 6369 put.add(kv4); 6370 region.put(put); 6371 // scan range = ["row4", min) 6372 Scan scan = new Scan().withStartRow(row4); 6373 scan.setReversed(true); 6374 scan.setBatch(10); 6375 InternalScanner scanner = region.getScanner(scan); 6376 List<Cell> currRow = new ArrayList<>(); 6377 boolean hasNext = scanner.next(currRow); 6378 assertEquals(1, currRow.size()); 6379 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6380 currRow.get(0).getRowLength(), row4, 0, row4.length)); 6381 assertTrue(hasNext); 6382 currRow.clear(); 6383 hasNext = scanner.next(currRow); 6384 assertEquals(1, currRow.size()); 6385 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6386 currRow.get(0).getRowLength(), row3, 0, row3.length)); 6387 assertTrue(hasNext); 6388 currRow.clear(); 6389 hasNext = scanner.next(currRow); 6390 assertEquals(1, currRow.size()); 6391 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6392 currRow.get(0).getRowLength(), row2, 0, row2.length)); 6393 assertTrue(hasNext); 6394 currRow.clear(); 6395 hasNext = scanner.next(currRow); 6396 assertEquals(1, currRow.size()); 6397 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6398 currRow.get(0).getRowLength(), row1, 0, row1.length)); 6399 assertFalse(hasNext); 6400 } 6401 6402 /** 6403 * Test for HBASE-14497: Reverse Scan threw StackOverflow caused by readPt checking 6404 */ 6405 @Test 6406 public void testReverseScanner_StackOverflow() throws IOException { 6407 byte[] cf1 = Bytes.toBytes("CF1"); 6408 byte[][] families = { cf1 }; 6409 byte[] col = Bytes.toBytes("C"); 6410 Configuration conf = new Configuration(CONF); 6411 this.region = initHRegion(tableName, method, conf, families); 6412 // setup with one storefile and one memstore, to create scanner and get an earlier readPt 6413 Put put = new Put(Bytes.toBytes("19998")); 6414 put.addColumn(cf1, col, Bytes.toBytes("val")); 6415 region.put(put); 6416 region.flushcache(true, true, FlushLifeCycleTracker.DUMMY); 6417 Put put2 = new Put(Bytes.toBytes("19997")); 6418 put2.addColumn(cf1, col, Bytes.toBytes("val")); 6419 region.put(put2); 6420 6421 Scan scan = new Scan().withStartRow(Bytes.toBytes("19998")); 6422 scan.setReversed(true); 6423 InternalScanner scanner = region.getScanner(scan); 6424 6425 // create one storefile contains many rows will be skipped 6426 // to check StoreFileScanner.seekToPreviousRow 6427 for (int i = 10000; i < 20000; i++) { 6428 Put p = new Put(Bytes.toBytes("" + i)); 6429 p.addColumn(cf1, col, Bytes.toBytes("" + i)); 6430 region.put(p); 6431 } 6432 region.flushcache(true, true, FlushLifeCycleTracker.DUMMY); 6433 6434 // create one memstore contains many rows will be skipped 6435 // to check MemStoreScanner.seekToPreviousRow 6436 for (int i = 10000; i < 20000; i++) { 6437 Put p = new Put(Bytes.toBytes("" + i)); 6438 p.addColumn(cf1, col, Bytes.toBytes("" + i)); 6439 region.put(p); 6440 } 6441 6442 List<Cell> currRow = new ArrayList<>(); 6443 boolean hasNext; 6444 do { 6445 hasNext = scanner.next(currRow); 6446 } while (hasNext); 6447 assertEquals(2, currRow.size()); 6448 assertEquals("19998", Bytes.toString(currRow.get(0).getRowArray(), 6449 currRow.get(0).getRowOffset(), currRow.get(0).getRowLength())); 6450 assertEquals("19997", Bytes.toString(currRow.get(1).getRowArray(), 6451 currRow.get(1).getRowOffset(), currRow.get(1).getRowLength())); 6452 } 6453 6454 @Test 6455 public void testReverseScanShouldNotScanMemstoreIfReadPtLesser() throws Exception { 6456 byte[] cf1 = Bytes.toBytes("CF1"); 6457 byte[][] families = { cf1 }; 6458 byte[] col = Bytes.toBytes("C"); 6459 this.region = initHRegion(tableName, method, CONF, families); 6460 // setup with one storefile and one memstore, to create scanner and get an earlier readPt 6461 Put put = new Put(Bytes.toBytes("19996")); 6462 put.addColumn(cf1, col, Bytes.toBytes("val")); 6463 region.put(put); 6464 Put put2 = new Put(Bytes.toBytes("19995")); 6465 put2.addColumn(cf1, col, Bytes.toBytes("val")); 6466 region.put(put2); 6467 // create a reverse scan 6468 Scan scan = new Scan().withStartRow(Bytes.toBytes("19996")); 6469 scan.setReversed(true); 6470 RegionScannerImpl scanner = region.getScanner(scan); 6471 6472 // flush the cache. This will reset the store scanner 6473 region.flushcache(true, true, FlushLifeCycleTracker.DUMMY); 6474 6475 // create one memstore contains many rows will be skipped 6476 // to check MemStoreScanner.seekToPreviousRow 6477 for (int i = 10000; i < 20000; i++) { 6478 Put p = new Put(Bytes.toBytes("" + i)); 6479 p.addColumn(cf1, col, Bytes.toBytes("" + i)); 6480 region.put(p); 6481 } 6482 List<Cell> currRow = new ArrayList<>(); 6483 boolean hasNext; 6484 boolean assertDone = false; 6485 do { 6486 hasNext = scanner.next(currRow); 6487 // With HBASE-15871, after the scanner is reset the memstore scanner should not be 6488 // added here 6489 if (!assertDone) { 6490 StoreScanner current = (StoreScanner) (scanner.storeHeap).getCurrentForTesting(); 6491 List<KeyValueScanner> scanners = current.getAllScannersForTesting(); 6492 assertEquals("There should be only one scanner the store file scanner", 1, scanners.size()); 6493 assertDone = true; 6494 } 6495 } while (hasNext); 6496 assertEquals(2, currRow.size()); 6497 assertEquals("19996", Bytes.toString(currRow.get(0).getRowArray(), 6498 currRow.get(0).getRowOffset(), currRow.get(0).getRowLength())); 6499 assertEquals("19995", Bytes.toString(currRow.get(1).getRowArray(), 6500 currRow.get(1).getRowOffset(), currRow.get(1).getRowLength())); 6501 } 6502 6503 @Test 6504 public void testReverseScanWhenPutCellsAfterOpenReverseScan() throws Exception { 6505 byte[] cf1 = Bytes.toBytes("CF1"); 6506 byte[][] families = { cf1 }; 6507 byte[] col = Bytes.toBytes("C"); 6508 6509 this.region = initHRegion(tableName, method, CONF, families); 6510 6511 Put put = new Put(Bytes.toBytes("199996")); 6512 put.addColumn(cf1, col, Bytes.toBytes("val")); 6513 region.put(put); 6514 Put put2 = new Put(Bytes.toBytes("199995")); 6515 put2.addColumn(cf1, col, Bytes.toBytes("val")); 6516 region.put(put2); 6517 6518 // Create a reverse scan 6519 Scan scan = new Scan().withStartRow(Bytes.toBytes("199996")); 6520 scan.setReversed(true); 6521 RegionScannerImpl scanner = region.getScanner(scan); 6522 6523 // Put a lot of cells that have sequenceIDs grater than the readPt of the reverse scan 6524 for (int i = 100000; i < 200000; i++) { 6525 Put p = new Put(Bytes.toBytes("" + i)); 6526 p.addColumn(cf1, col, Bytes.toBytes("" + i)); 6527 region.put(p); 6528 } 6529 List<Cell> currRow = new ArrayList<>(); 6530 boolean hasNext; 6531 do { 6532 hasNext = scanner.next(currRow); 6533 } while (hasNext); 6534 6535 assertEquals(2, currRow.size()); 6536 assertEquals("199996", Bytes.toString(currRow.get(0).getRowArray(), 6537 currRow.get(0).getRowOffset(), currRow.get(0).getRowLength())); 6538 assertEquals("199995", Bytes.toString(currRow.get(1).getRowArray(), 6539 currRow.get(1).getRowOffset(), currRow.get(1).getRowLength())); 6540 } 6541 6542 @Test 6543 public void testWriteRequestsCounter() throws IOException { 6544 byte[] fam = Bytes.toBytes("info"); 6545 byte[][] families = { fam }; 6546 this.region = initHRegion(tableName, method, CONF, families); 6547 6548 Assert.assertEquals(0L, region.getWriteRequestsCount()); 6549 6550 Put put = new Put(row); 6551 put.addColumn(fam, fam, fam); 6552 6553 Assert.assertEquals(0L, region.getWriteRequestsCount()); 6554 region.put(put); 6555 Assert.assertEquals(1L, region.getWriteRequestsCount()); 6556 region.put(put); 6557 Assert.assertEquals(2L, region.getWriteRequestsCount()); 6558 region.put(put); 6559 Assert.assertEquals(3L, region.getWriteRequestsCount()); 6560 6561 region.delete(new Delete(row)); 6562 Assert.assertEquals(4L, region.getWriteRequestsCount()); 6563 } 6564 6565 @Test 6566 public void testOpenRegionWrittenToWAL() throws Exception { 6567 final ServerName serverName = ServerName.valueOf(name.getMethodName(), 100, 42); 6568 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName)); 6569 6570 TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) 6571 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)) 6572 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam2)).build(); 6573 RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build(); 6574 6575 // open the region w/o rss and wal and flush some files 6576 region = HBaseTestingUtil.createRegionAndWAL(hri, TEST_UTIL.getDataTestDir(), 6577 TEST_UTIL.getConfiguration(), htd); 6578 assertNotNull(region); 6579 6580 // create a file in fam1 for the region before opening in OpenRegionHandler 6581 region.put(new Put(Bytes.toBytes("a")).addColumn(fam1, fam1, fam1)); 6582 region.flush(true); 6583 HBaseTestingUtil.closeRegionAndWAL(region); 6584 6585 ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class); 6586 6587 // capture append() calls 6588 WAL wal = mockWAL(); 6589 when(rss.getWAL(any(RegionInfo.class))).thenReturn(wal); 6590 6591 region = 6592 HRegion.openHRegion(hri, htd, rss.getWAL(hri), TEST_UTIL.getConfiguration(), rss, null); 6593 6594 verify(wal, times(1)).appendMarker(any(RegionInfo.class), any(WALKeyImpl.class), 6595 editCaptor.capture()); 6596 6597 WALEdit edit = editCaptor.getValue(); 6598 assertNotNull(edit); 6599 assertNotNull(edit.getCells()); 6600 assertEquals(1, edit.getCells().size()); 6601 RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0)); 6602 assertNotNull(desc); 6603 6604 LOG.info("RegionEventDescriptor from WAL: " + desc); 6605 6606 assertEquals(RegionEventDescriptor.EventType.REGION_OPEN, desc.getEventType()); 6607 assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getTableName().toBytes())); 6608 assertTrue( 6609 Bytes.equals(desc.getEncodedRegionName().toByteArray(), hri.getEncodedNameAsBytes())); 6610 assertTrue(desc.getLogSequenceNumber() > 0); 6611 assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer())); 6612 assertEquals(2, desc.getStoresCount()); 6613 6614 StoreDescriptor store = desc.getStores(0); 6615 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1)); 6616 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1)); 6617 assertEquals(1, store.getStoreFileCount()); // 1store file 6618 assertFalse(store.getStoreFile(0).contains("/")); // ensure path is relative 6619 6620 store = desc.getStores(1); 6621 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2)); 6622 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2)); 6623 assertEquals(0, store.getStoreFileCount()); // no store files 6624 } 6625 6626 // Helper for test testOpenRegionWrittenToWALForLogReplay 6627 static class HRegionWithSeqId extends HRegion { 6628 public HRegionWithSeqId(final Path tableDir, final WAL wal, final FileSystem fs, 6629 final Configuration confParam, final RegionInfo regionInfo, final TableDescriptor htd, 6630 final RegionServerServices rsServices) { 6631 super(tableDir, wal, fs, confParam, regionInfo, htd, rsServices); 6632 } 6633 6634 @Override 6635 protected long getNextSequenceId(WAL wal) throws IOException { 6636 return 42; 6637 } 6638 } 6639 6640 @Test 6641 public void testFlushedFileWithNoTags() throws Exception { 6642 final TableName tableName = TableName.valueOf(name.getMethodName()); 6643 TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName) 6644 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)).build(); 6645 RegionInfo info = RegionInfoBuilder.newBuilder(tableName).build(); 6646 Path path = TEST_UTIL.getDataTestDir(getClass().getSimpleName()); 6647 region = HBaseTestingUtil.createRegionAndWAL(info, path, TEST_UTIL.getConfiguration(), 6648 tableDescriptor); 6649 Put put = new Put(Bytes.toBytes("a-b-0-0")); 6650 put.addColumn(fam1, qual1, Bytes.toBytes("c1-value")); 6651 region.put(put); 6652 region.flush(true); 6653 HStore store = region.getStore(fam1); 6654 Collection<HStoreFile> storefiles = store.getStorefiles(); 6655 for (HStoreFile sf : storefiles) { 6656 assertFalse("Tags should not be present ", 6657 sf.getReader().getHFileReader().getFileContext().isIncludesTags()); 6658 } 6659 } 6660 6661 /** 6662 * Utility method to setup a WAL mock. 6663 * <p/> 6664 * Needs to do the bit where we close latch on the WALKeyImpl on append else test hangs. 6665 * @return a mock WAL 6666 */ 6667 private WAL mockWAL() throws IOException { 6668 WAL wal = mock(WAL.class); 6669 when(wal.appendData(any(RegionInfo.class), any(WALKeyImpl.class), any(WALEdit.class))) 6670 .thenAnswer(new Answer<Long>() { 6671 @Override 6672 public Long answer(InvocationOnMock invocation) throws Throwable { 6673 WALKeyImpl key = invocation.getArgument(1); 6674 MultiVersionConcurrencyControl.WriteEntry we = key.getMvcc().begin(); 6675 key.setWriteEntry(we); 6676 return 1L; 6677 } 6678 }); 6679 when(wal.appendMarker(any(RegionInfo.class), any(WALKeyImpl.class), any(WALEdit.class))) 6680 .thenAnswer(new Answer<Long>() { 6681 @Override 6682 public Long answer(InvocationOnMock invocation) throws Throwable { 6683 WALKeyImpl key = invocation.getArgument(1); 6684 MultiVersionConcurrencyControl.WriteEntry we = key.getMvcc().begin(); 6685 key.setWriteEntry(we); 6686 return 1L; 6687 } 6688 }); 6689 return wal; 6690 } 6691 6692 @Test 6693 public void testCloseRegionWrittenToWAL() throws Exception { 6694 Path rootDir = new Path(dir + name.getMethodName()); 6695 CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir); 6696 6697 final ServerName serverName = ServerName.valueOf("testCloseRegionWrittenToWAL", 100, 42); 6698 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName)); 6699 6700 TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) 6701 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)) 6702 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam2)).build(); 6703 RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build(); 6704 6705 ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class); 6706 6707 // capture append() calls 6708 WAL wal = mockWAL(); 6709 when(rss.getWAL(any(RegionInfo.class))).thenReturn(wal); 6710 6711 // create and then open a region first so that it can be closed later 6712 region = 6713 HRegion.createHRegion(hri, rootDir, TEST_UTIL.getConfiguration(), htd, rss.getWAL(hri)); 6714 region = 6715 HRegion.openHRegion(hri, htd, rss.getWAL(hri), TEST_UTIL.getConfiguration(), rss, null); 6716 6717 // close the region 6718 region.close(false); 6719 6720 // 2 times, one for region open, the other close region 6721 verify(wal, times(2)).appendMarker(any(RegionInfo.class), (WALKeyImpl) any(WALKeyImpl.class), 6722 editCaptor.capture()); 6723 6724 WALEdit edit = editCaptor.getAllValues().get(1); 6725 assertNotNull(edit); 6726 assertNotNull(edit.getCells()); 6727 assertEquals(1, edit.getCells().size()); 6728 RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0)); 6729 assertNotNull(desc); 6730 6731 LOG.info("RegionEventDescriptor from WAL: " + desc); 6732 6733 assertEquals(RegionEventDescriptor.EventType.REGION_CLOSE, desc.getEventType()); 6734 assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getTableName().toBytes())); 6735 assertTrue( 6736 Bytes.equals(desc.getEncodedRegionName().toByteArray(), hri.getEncodedNameAsBytes())); 6737 assertTrue(desc.getLogSequenceNumber() > 0); 6738 assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer())); 6739 assertEquals(2, desc.getStoresCount()); 6740 6741 StoreDescriptor store = desc.getStores(0); 6742 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1)); 6743 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1)); 6744 assertEquals(0, store.getStoreFileCount()); // no store files 6745 6746 store = desc.getStores(1); 6747 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2)); 6748 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2)); 6749 assertEquals(0, store.getStoreFileCount()); // no store files 6750 } 6751 6752 /** 6753 * Test RegionTooBusyException thrown when region is busy 6754 */ 6755 @Test 6756 public void testRegionTooBusy() throws IOException { 6757 byte[] family = Bytes.toBytes("family"); 6758 long defaultBusyWaitDuration = 6759 CONF.getLong("hbase.busy.wait.duration", HRegion.DEFAULT_BUSY_WAIT_DURATION); 6760 CONF.setLong("hbase.busy.wait.duration", 1000); 6761 region = initHRegion(tableName, method, CONF, family); 6762 final AtomicBoolean stopped = new AtomicBoolean(true); 6763 Thread t = new Thread(new Runnable() { 6764 @Override 6765 public void run() { 6766 try { 6767 region.lock.writeLock().lock(); 6768 stopped.set(false); 6769 while (!stopped.get()) { 6770 Thread.sleep(100); 6771 } 6772 } catch (InterruptedException ie) { 6773 } finally { 6774 region.lock.writeLock().unlock(); 6775 } 6776 } 6777 }); 6778 t.start(); 6779 Get get = new Get(row); 6780 try { 6781 while (stopped.get()) { 6782 Thread.sleep(100); 6783 } 6784 region.get(get); 6785 fail("Should throw RegionTooBusyException"); 6786 } catch (InterruptedException ie) { 6787 fail("test interrupted"); 6788 } catch (RegionTooBusyException e) { 6789 // Good, expected 6790 } finally { 6791 stopped.set(true); 6792 try { 6793 t.join(); 6794 } catch (Throwable e) { 6795 } 6796 6797 HBaseTestingUtil.closeRegionAndWAL(region); 6798 region = null; 6799 CONF.setLong("hbase.busy.wait.duration", defaultBusyWaitDuration); 6800 } 6801 } 6802 6803 @Test 6804 public void testCellTTLs() throws IOException { 6805 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge(); 6806 EnvironmentEdgeManager.injectEdge(edge); 6807 6808 final byte[] row = Bytes.toBytes("testRow"); 6809 final byte[] q1 = Bytes.toBytes("q1"); 6810 final byte[] q2 = Bytes.toBytes("q2"); 6811 final byte[] q3 = Bytes.toBytes("q3"); 6812 final byte[] q4 = Bytes.toBytes("q4"); 6813 6814 // 10 seconds 6815 TableDescriptor tableDescriptor = 6816 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) 6817 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam1).setTimeToLive(10).build()) 6818 .build(); 6819 6820 Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); 6821 conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS); 6822 6823 region = HBaseTestingUtil.createRegionAndWAL( 6824 RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(), 6825 TEST_UTIL.getDataTestDir(), conf, tableDescriptor); 6826 assertNotNull(region); 6827 long now = EnvironmentEdgeManager.currentTime(); 6828 // Add a cell that will expire in 5 seconds via cell TTL 6829 region.put(new Put(row).add(new KeyValue(row, fam1, q1, now, HConstants.EMPTY_BYTE_ARRAY, 6830 new ArrayBackedTag[] { 6831 // TTL tags specify ts in milliseconds 6832 new ArrayBackedTag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) }))); 6833 // Add a cell that will expire after 10 seconds via family setting 6834 region.put(new Put(row).addColumn(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY)); 6835 // Add a cell that will expire in 15 seconds via cell TTL 6836 region.put(new Put(row).add(new KeyValue(row, fam1, q3, now + 10000 - 1, 6837 HConstants.EMPTY_BYTE_ARRAY, new ArrayBackedTag[] { 6838 // TTL tags specify ts in milliseconds 6839 new ArrayBackedTag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) }))); 6840 // Add a cell that will expire in 20 seconds via family setting 6841 region.put(new Put(row).addColumn(fam1, q4, now + 10000 - 1, HConstants.EMPTY_BYTE_ARRAY)); 6842 6843 // Flush so we are sure store scanning gets this right 6844 region.flush(true); 6845 6846 // A query at time T+0 should return all cells 6847 Result r = region.get(new Get(row)); 6848 assertNotNull(r.getValue(fam1, q1)); 6849 assertNotNull(r.getValue(fam1, q2)); 6850 assertNotNull(r.getValue(fam1, q3)); 6851 assertNotNull(r.getValue(fam1, q4)); 6852 6853 // Increment time to T+5 seconds 6854 edge.incrementTime(5000); 6855 6856 r = region.get(new Get(row)); 6857 assertNull(r.getValue(fam1, q1)); 6858 assertNotNull(r.getValue(fam1, q2)); 6859 assertNotNull(r.getValue(fam1, q3)); 6860 assertNotNull(r.getValue(fam1, q4)); 6861 6862 // Increment time to T+10 seconds 6863 edge.incrementTime(5000); 6864 6865 r = region.get(new Get(row)); 6866 assertNull(r.getValue(fam1, q1)); 6867 assertNull(r.getValue(fam1, q2)); 6868 assertNotNull(r.getValue(fam1, q3)); 6869 assertNotNull(r.getValue(fam1, q4)); 6870 6871 // Increment time to T+15 seconds 6872 edge.incrementTime(5000); 6873 6874 r = region.get(new Get(row)); 6875 assertNull(r.getValue(fam1, q1)); 6876 assertNull(r.getValue(fam1, q2)); 6877 assertNull(r.getValue(fam1, q3)); 6878 assertNotNull(r.getValue(fam1, q4)); 6879 6880 // Increment time to T+20 seconds 6881 edge.incrementTime(10000); 6882 6883 r = region.get(new Get(row)); 6884 assertNull(r.getValue(fam1, q1)); 6885 assertNull(r.getValue(fam1, q2)); 6886 assertNull(r.getValue(fam1, q3)); 6887 assertNull(r.getValue(fam1, q4)); 6888 6889 // Fun with disappearing increments 6890 6891 // Start at 1 6892 region.put(new Put(row).addColumn(fam1, q1, Bytes.toBytes(1L))); 6893 r = region.get(new Get(row)); 6894 byte[] val = r.getValue(fam1, q1); 6895 assertNotNull(val); 6896 assertEquals(1L, Bytes.toLong(val)); 6897 6898 // Increment with a TTL of 5 seconds 6899 Increment incr = new Increment(row).addColumn(fam1, q1, 1L); 6900 incr.setTTL(5000); 6901 region.increment(incr); // 2 6902 6903 // New value should be 2 6904 r = region.get(new Get(row)); 6905 val = r.getValue(fam1, q1); 6906 assertNotNull(val); 6907 assertEquals(2L, Bytes.toLong(val)); 6908 6909 // Increment time to T+25 seconds 6910 edge.incrementTime(5000); 6911 6912 // Value should be back to 1 6913 r = region.get(new Get(row)); 6914 val = r.getValue(fam1, q1); 6915 assertNotNull(val); 6916 assertEquals(1L, Bytes.toLong(val)); 6917 6918 // Increment time to T+30 seconds 6919 edge.incrementTime(5000); 6920 6921 // Original value written at T+20 should be gone now via family TTL 6922 r = region.get(new Get(row)); 6923 assertNull(r.getValue(fam1, q1)); 6924 } 6925 6926 @Test 6927 public void testIncrementTimestampsAreMonotonic() throws IOException { 6928 region = initHRegion(tableName, method, CONF, fam1); 6929 ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); 6930 EnvironmentEdgeManager.injectEdge(edge); 6931 6932 edge.setValue(10); 6933 Increment inc = new Increment(row); 6934 inc.setDurability(Durability.SKIP_WAL); 6935 inc.addColumn(fam1, qual1, 1L); 6936 region.increment(inc); 6937 6938 Result result = region.get(new Get(row)); 6939 Cell c = result.getColumnLatestCell(fam1, qual1); 6940 assertNotNull(c); 6941 assertEquals(10L, c.getTimestamp()); 6942 6943 edge.setValue(1); // clock goes back 6944 region.increment(inc); 6945 result = region.get(new Get(row)); 6946 c = result.getColumnLatestCell(fam1, qual1); 6947 assertEquals(11L, c.getTimestamp()); 6948 assertEquals(2L, Bytes.toLong(c.getValueArray(), c.getValueOffset(), c.getValueLength())); 6949 } 6950 6951 @Test 6952 public void testAppendTimestampsAreMonotonic() throws IOException { 6953 region = initHRegion(tableName, method, CONF, fam1); 6954 ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); 6955 EnvironmentEdgeManager.injectEdge(edge); 6956 6957 edge.setValue(10); 6958 Append a = new Append(row); 6959 a.setDurability(Durability.SKIP_WAL); 6960 a.addColumn(fam1, qual1, qual1); 6961 region.append(a); 6962 6963 Result result = region.get(new Get(row)); 6964 Cell c = result.getColumnLatestCell(fam1, qual1); 6965 assertNotNull(c); 6966 assertEquals(10L, c.getTimestamp()); 6967 6968 edge.setValue(1); // clock goes back 6969 region.append(a); 6970 result = region.get(new Get(row)); 6971 c = result.getColumnLatestCell(fam1, qual1); 6972 assertEquals(11L, c.getTimestamp()); 6973 6974 byte[] expected = new byte[qual1.length * 2]; 6975 System.arraycopy(qual1, 0, expected, 0, qual1.length); 6976 System.arraycopy(qual1, 0, expected, qual1.length, qual1.length); 6977 6978 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(), expected, 0, 6979 expected.length)); 6980 } 6981 6982 @Test 6983 public void testCheckAndMutateTimestampsAreMonotonic() throws IOException { 6984 region = initHRegion(tableName, method, CONF, fam1); 6985 ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); 6986 EnvironmentEdgeManager.injectEdge(edge); 6987 6988 edge.setValue(10); 6989 Put p = new Put(row); 6990 p.setDurability(Durability.SKIP_WAL); 6991 p.addColumn(fam1, qual1, qual1); 6992 region.put(p); 6993 6994 Result result = region.get(new Get(row)); 6995 Cell c = result.getColumnLatestCell(fam1, qual1); 6996 assertNotNull(c); 6997 assertEquals(10L, c.getTimestamp()); 6998 6999 edge.setValue(1); // clock goes back 7000 p = new Put(row); 7001 p.setDurability(Durability.SKIP_WAL); 7002 p.addColumn(fam1, qual1, qual2); 7003 region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(qual1), p); 7004 result = region.get(new Get(row)); 7005 c = result.getColumnLatestCell(fam1, qual1); 7006 assertEquals(10L, c.getTimestamp()); 7007 7008 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(), qual2, 0, 7009 qual2.length)); 7010 } 7011 7012 @Test 7013 public void testBatchMutateWithWrongRegionException() throws Exception { 7014 final byte[] a = Bytes.toBytes("a"); 7015 final byte[] b = Bytes.toBytes("b"); 7016 final byte[] c = Bytes.toBytes("c"); // exclusive 7017 7018 int prevLockTimeout = CONF.getInt("hbase.rowlock.wait.duration", 30000); 7019 CONF.setInt("hbase.rowlock.wait.duration", 1000); 7020 region = initHRegion(tableName, a, c, method, CONF, false, fam1); 7021 7022 Mutation[] mutations = new Mutation[] { 7023 new Put(a).add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a) 7024 .setFamily(fam1).setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), 7025 // this is outside the region boundary 7026 new Put(c).add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(c) 7027 .setFamily(fam1).setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Type.Put).build()), 7028 new Put(b) 7029 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1) 7030 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()) }; 7031 7032 OperationStatus[] status = region.batchMutate(mutations); 7033 assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode()); 7034 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, status[1].getOperationStatusCode()); 7035 assertEquals(OperationStatusCode.SUCCESS, status[2].getOperationStatusCode()); 7036 7037 // test with a row lock held for a long time 7038 final CountDownLatch obtainedRowLock = new CountDownLatch(1); 7039 ExecutorService exec = Executors.newFixedThreadPool(2); 7040 Future<Void> f1 = exec.submit(new Callable<Void>() { 7041 @Override 7042 public Void call() throws Exception { 7043 LOG.info("Acquiring row lock"); 7044 RowLock rl = region.getRowLock(b); 7045 obtainedRowLock.countDown(); 7046 LOG.info("Waiting for 5 seconds before releasing lock"); 7047 Threads.sleep(5000); 7048 LOG.info("Releasing row lock"); 7049 rl.release(); 7050 return null; 7051 } 7052 }); 7053 obtainedRowLock.await(30, TimeUnit.SECONDS); 7054 7055 Future<Void> f2 = exec.submit(new Callable<Void>() { 7056 @Override 7057 public Void call() throws Exception { 7058 Mutation[] mutations = new Mutation[] { 7059 new Put(a) 7060 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a).setFamily(fam1) 7061 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), 7062 new Put(b) 7063 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1) 7064 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), }; 7065 7066 // this will wait for the row lock, and it will eventually succeed 7067 OperationStatus[] status = region.batchMutate(mutations); 7068 assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode()); 7069 assertEquals(OperationStatusCode.SUCCESS, status[1].getOperationStatusCode()); 7070 return null; 7071 } 7072 }); 7073 7074 f1.get(); 7075 f2.get(); 7076 7077 CONF.setInt("hbase.rowlock.wait.duration", prevLockTimeout); 7078 } 7079 7080 @Test 7081 public void testBatchMutateWithZeroRowLockWait() throws Exception { 7082 final byte[] a = Bytes.toBytes("a"); 7083 final byte[] b = Bytes.toBytes("b"); 7084 final byte[] c = Bytes.toBytes("c"); // exclusive 7085 7086 Configuration conf = new Configuration(CONF); 7087 conf.setInt("hbase.rowlock.wait.duration", 0); 7088 final RegionInfo hri = 7089 RegionInfoBuilder.newBuilder(tableName).setStartKey(a).setEndKey(c).build(); 7090 final TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName) 7091 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)).build(); 7092 region = HRegion.createHRegion(hri, TEST_UTIL.getDataTestDir(), conf, htd, 7093 HBaseTestingUtil.createWal(conf, TEST_UTIL.getDataTestDirOnTestFS(method + ".log"), hri)); 7094 7095 Mutation[] mutations = new Mutation[] { 7096 new Put(a).add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a) 7097 .setFamily(fam1).setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), 7098 new Put(b) 7099 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1) 7100 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()) }; 7101 7102 OperationStatus[] status = region.batchMutate(mutations); 7103 assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode()); 7104 assertEquals(OperationStatusCode.SUCCESS, status[1].getOperationStatusCode()); 7105 7106 // test with a row lock held for a long time 7107 final CountDownLatch obtainedRowLock = new CountDownLatch(1); 7108 ExecutorService exec = Executors.newFixedThreadPool(2); 7109 Future<Void> f1 = exec.submit(new Callable<Void>() { 7110 @Override 7111 public Void call() throws Exception { 7112 LOG.info("Acquiring row lock"); 7113 RowLock rl = region.getRowLock(b); 7114 obtainedRowLock.countDown(); 7115 LOG.info("Waiting for 5 seconds before releasing lock"); 7116 Threads.sleep(5000); 7117 LOG.info("Releasing row lock"); 7118 rl.release(); 7119 return null; 7120 } 7121 }); 7122 obtainedRowLock.await(30, TimeUnit.SECONDS); 7123 7124 Future<Void> f2 = exec.submit(new Callable<Void>() { 7125 @Override 7126 public Void call() throws Exception { 7127 Mutation[] mutations = new Mutation[] { 7128 new Put(a) 7129 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a).setFamily(fam1) 7130 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), 7131 new Put(b) 7132 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1) 7133 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), }; 7134 // when handling row b we are going to spin on the failure to get the row lock 7135 // until the lock above is released, but we will still succeed so long as that 7136 // takes less time then the test time out. 7137 OperationStatus[] status = region.batchMutate(mutations); 7138 assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode()); 7139 assertEquals(OperationStatusCode.SUCCESS, status[1].getOperationStatusCode()); 7140 return null; 7141 } 7142 }); 7143 7144 f1.get(); 7145 f2.get(); 7146 } 7147 7148 @Test 7149 public void testCheckAndRowMutateTimestampsAreMonotonic() throws IOException { 7150 region = initHRegion(tableName, method, CONF, fam1); 7151 ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); 7152 EnvironmentEdgeManager.injectEdge(edge); 7153 7154 edge.setValue(10); 7155 Put p = new Put(row); 7156 p.setDurability(Durability.SKIP_WAL); 7157 p.addColumn(fam1, qual1, qual1); 7158 region.put(p); 7159 7160 Result result = region.get(new Get(row)); 7161 Cell c = result.getColumnLatestCell(fam1, qual1); 7162 assertNotNull(c); 7163 assertEquals(10L, c.getTimestamp()); 7164 7165 edge.setValue(1); // clock goes back 7166 p = new Put(row); 7167 p.setDurability(Durability.SKIP_WAL); 7168 p.addColumn(fam1, qual1, qual2); 7169 RowMutations rm = new RowMutations(row); 7170 rm.add(p); 7171 assertTrue(region.checkAndRowMutate(row, fam1, qual1, CompareOperator.EQUAL, 7172 new BinaryComparator(qual1), rm)); 7173 result = region.get(new Get(row)); 7174 c = result.getColumnLatestCell(fam1, qual1); 7175 assertEquals(10L, c.getTimestamp()); 7176 LOG.info( 7177 "c value " + Bytes.toStringBinary(c.getValueArray(), c.getValueOffset(), c.getValueLength())); 7178 7179 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(), qual2, 0, 7180 qual2.length)); 7181 } 7182 7183 HRegion initHRegion(TableName tableName, String callingMethod, byte[]... families) 7184 throws IOException { 7185 return initHRegion(tableName, callingMethod, HBaseConfiguration.create(), families); 7186 } 7187 7188 /** 7189 * HBASE-16429 Make sure no stuck if roll writer when ring buffer is filled with appends 7190 * @throws IOException if IO error occurred during test 7191 */ 7192 @Test 7193 public void testWritesWhileRollWriter() throws IOException { 7194 int testCount = 10; 7195 int numRows = 1024; 7196 int numFamilies = 2; 7197 int numQualifiers = 2; 7198 final byte[][] families = new byte[numFamilies][]; 7199 for (int i = 0; i < numFamilies; i++) { 7200 families[i] = Bytes.toBytes("family" + i); 7201 } 7202 final byte[][] qualifiers = new byte[numQualifiers][]; 7203 for (int i = 0; i < numQualifiers; i++) { 7204 qualifiers[i] = Bytes.toBytes("qual" + i); 7205 } 7206 7207 CONF.setInt("hbase.regionserver.wal.disruptor.event.count", 2); 7208 this.region = initHRegion(tableName, method, CONF, families); 7209 try { 7210 List<Thread> threads = new ArrayList<>(); 7211 for (int i = 0; i < numRows; i++) { 7212 final int count = i; 7213 Thread t = new Thread(new Runnable() { 7214 7215 @Override 7216 public void run() { 7217 byte[] row = Bytes.toBytes("row" + count); 7218 Put put = new Put(row); 7219 put.setDurability(Durability.SYNC_WAL); 7220 byte[] value = Bytes.toBytes(String.valueOf(count)); 7221 for (byte[] family : families) { 7222 for (byte[] qualifier : qualifiers) { 7223 put.addColumn(family, qualifier, count, value); 7224 } 7225 } 7226 try { 7227 region.put(put); 7228 } catch (IOException e) { 7229 throw new RuntimeException(e); 7230 } 7231 } 7232 }); 7233 threads.add(t); 7234 } 7235 for (Thread t : threads) { 7236 t.start(); 7237 } 7238 7239 for (int i = 0; i < testCount; i++) { 7240 region.getWAL().rollWriter(); 7241 Thread.yield(); 7242 } 7243 } finally { 7244 try { 7245 HBaseTestingUtil.closeRegionAndWAL(this.region); 7246 CONF.setInt("hbase.regionserver.wal.disruptor.event.count", 16 * 1024); 7247 } catch (DroppedSnapshotException dse) { 7248 // We could get this on way out because we interrupt the background flusher and it could 7249 // fail anywhere causing a DSE over in the background flusher... only it is not properly 7250 // dealt with so could still be memory hanging out when we get to here -- memory we can't 7251 // flush because the accounting is 'off' since original DSE. 7252 } 7253 this.region = null; 7254 } 7255 } 7256 7257 @Test 7258 public void testMutateRow() throws Exception { 7259 final byte[] row = Bytes.toBytes("row"); 7260 final byte[] q1 = Bytes.toBytes("q1"); 7261 final byte[] q2 = Bytes.toBytes("q2"); 7262 final byte[] q3 = Bytes.toBytes("q3"); 7263 final byte[] q4 = Bytes.toBytes("q4"); 7264 final String v1 = "v1"; 7265 7266 region = initHRegion(tableName, method, CONF, fam1); 7267 7268 // Initial values 7269 region 7270 .batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q2, Bytes.toBytes("toBeDeleted")), 7271 new Put(row).addColumn(fam1, q3, Bytes.toBytes(5L)), 7272 new Put(row).addColumn(fam1, q4, Bytes.toBytes("a")), }); 7273 7274 // Do mutateRow 7275 Result result = region.mutateRow( 7276 new RowMutations(row).add(Arrays.asList(new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1)), 7277 new Delete(row).addColumns(fam1, q2), new Increment(row).addColumn(fam1, q3, 1), 7278 new Append(row).addColumn(fam1, q4, Bytes.toBytes("b"))))); 7279 7280 assertNotNull(result); 7281 assertEquals(6L, Bytes.toLong(result.getValue(fam1, q3))); 7282 assertEquals("ab", Bytes.toString(result.getValue(fam1, q4))); 7283 7284 // Verify the value 7285 result = region.get(new Get(row)); 7286 assertEquals(v1, Bytes.toString(result.getValue(fam1, q1))); 7287 assertNull(result.getValue(fam1, q2)); 7288 assertEquals(6L, Bytes.toLong(result.getValue(fam1, q3))); 7289 assertEquals("ab", Bytes.toString(result.getValue(fam1, q4))); 7290 } 7291 7292 @Test 7293 public void testMutateRowInParallel() throws Exception { 7294 final int numReaderThreads = 100; 7295 final CountDownLatch latch = new CountDownLatch(numReaderThreads); 7296 7297 final byte[] row = Bytes.toBytes("row"); 7298 final byte[] q1 = Bytes.toBytes("q1"); 7299 final byte[] q2 = Bytes.toBytes("q2"); 7300 final byte[] q3 = Bytes.toBytes("q3"); 7301 final byte[] q4 = Bytes.toBytes("q4"); 7302 final String v1 = "v1"; 7303 final String v2 = "v2"; 7304 7305 // We need to ensure the timestamp of the delete operation is more than the previous one 7306 final AtomicLong deleteTimestamp = new AtomicLong(); 7307 7308 region = initHRegion(tableName, method, CONF, fam1); 7309 7310 // Initial values 7311 region.batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1)) 7312 .addColumn(fam1, q2, deleteTimestamp.getAndIncrement(), Bytes.toBytes(v2)) 7313 .addColumn(fam1, q3, Bytes.toBytes(1L)).addColumn(fam1, q4, Bytes.toBytes("a")) }); 7314 7315 final AtomicReference<AssertionError> assertionError = new AtomicReference<>(); 7316 7317 // Writer thread 7318 Thread writerThread = new Thread(() -> { 7319 try { 7320 while (true) { 7321 // If all the reader threads finish, then stop the writer thread 7322 if (latch.await(0, TimeUnit.MILLISECONDS)) { 7323 return; 7324 } 7325 7326 // Execute the mutations. This should be done atomically 7327 region.mutateRow(new RowMutations(row) 7328 .add(Arrays.asList(new Put(row).addColumn(fam1, q1, Bytes.toBytes(v2)), 7329 new Delete(row).addColumns(fam1, q2, deleteTimestamp.getAndIncrement()), 7330 new Increment(row).addColumn(fam1, q3, 1L), 7331 new Append(row).addColumn(fam1, q4, Bytes.toBytes("b"))))); 7332 7333 // We need to ensure the timestamps of the Increment/Append operations are more than the 7334 // previous ones 7335 Result result = region.get(new Get(row).addColumn(fam1, q3).addColumn(fam1, q4)); 7336 long tsIncrement = result.getColumnLatestCell(fam1, q3).getTimestamp(); 7337 long tsAppend = result.getColumnLatestCell(fam1, q4).getTimestamp(); 7338 7339 // Put the initial values 7340 region.batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1)) 7341 .addColumn(fam1, q2, deleteTimestamp.getAndIncrement(), Bytes.toBytes(v2)) 7342 .addColumn(fam1, q3, tsIncrement + 1, Bytes.toBytes(1L)) 7343 .addColumn(fam1, q4, tsAppend + 1, Bytes.toBytes("a")) }); 7344 } 7345 } catch (Exception e) { 7346 assertionError.set(new AssertionError(e)); 7347 } 7348 }); 7349 writerThread.start(); 7350 7351 // Reader threads 7352 for (int i = 0; i < numReaderThreads; i++) { 7353 new Thread(() -> { 7354 try { 7355 for (int j = 0; j < 10000; j++) { 7356 // Verify the values 7357 Result result = region.get(new Get(row)); 7358 7359 // The values should be equals to either the initial values or the values after 7360 // executing the mutations 7361 String q1Value = Bytes.toString(result.getValue(fam1, q1)); 7362 if (v1.equals(q1Value)) { 7363 assertEquals(v2, Bytes.toString(result.getValue(fam1, q2))); 7364 assertEquals(1L, Bytes.toLong(result.getValue(fam1, q3))); 7365 assertEquals("a", Bytes.toString(result.getValue(fam1, q4))); 7366 } else if (v2.equals(q1Value)) { 7367 assertNull(Bytes.toString(result.getValue(fam1, q2))); 7368 assertEquals(2L, Bytes.toLong(result.getValue(fam1, q3))); 7369 assertEquals("ab", Bytes.toString(result.getValue(fam1, q4))); 7370 } else { 7371 fail("the qualifier " + Bytes.toString(q1) + " should be " + v1 + " or " + v2 7372 + ", but " + q1Value); 7373 } 7374 } 7375 } catch (Exception e) { 7376 assertionError.set(new AssertionError(e)); 7377 } catch (AssertionError e) { 7378 assertionError.set(e); 7379 } 7380 7381 latch.countDown(); 7382 }).start(); 7383 } 7384 7385 writerThread.join(); 7386 7387 if (assertionError.get() != null) { 7388 throw assertionError.get(); 7389 } 7390 } 7391 7392 @Test 7393 public void testMutateRow_WriteRequestCount() throws Exception { 7394 byte[] row1 = Bytes.toBytes("row1"); 7395 byte[] fam1 = Bytes.toBytes("fam1"); 7396 byte[] qf1 = Bytes.toBytes("qualifier"); 7397 byte[] val1 = Bytes.toBytes("value1"); 7398 7399 RowMutations rm = new RowMutations(row1); 7400 Put put = new Put(row1); 7401 put.addColumn(fam1, qf1, val1); 7402 rm.add(put); 7403 7404 this.region = initHRegion(tableName, method, CONF, fam1); 7405 long wrcBeforeMutate = this.region.writeRequestsCount.longValue(); 7406 this.region.mutateRow(rm); 7407 long wrcAfterMutate = this.region.writeRequestsCount.longValue(); 7408 Assert.assertEquals(wrcBeforeMutate + rm.getMutations().size(), wrcAfterMutate); 7409 } 7410 7411 @Test 7412 public void testBulkLoadReplicationEnabled() throws IOException { 7413 TEST_UTIL.getConfiguration().setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true); 7414 final ServerName serverName = ServerName.valueOf(name.getMethodName(), 100, 42); 7415 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName)); 7416 7417 TableDescriptor tableDescriptor = 7418 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) 7419 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)).build(); 7420 RegionInfo hri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 7421 region = HRegion.openHRegion(hri, tableDescriptor, rss.getWAL(hri), 7422 TEST_UTIL.getConfiguration(), rss, null); 7423 7424 assertTrue(region.conf.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, false)); 7425 String plugins = region.conf.get(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, ""); 7426 String replicationCoprocessorClass = ReplicationObserver.class.getCanonicalName(); 7427 assertTrue(plugins.contains(replicationCoprocessorClass)); 7428 assertTrue(region.getCoprocessorHost().getCoprocessors() 7429 .contains(ReplicationObserver.class.getSimpleName())); 7430 } 7431 7432 /** 7433 * The same as HRegion class, the only difference is that instantiateHStore will create a 7434 * different HStore - HStoreForTesting. [HBASE-8518] 7435 */ 7436 public static class HRegionForTesting extends HRegion { 7437 7438 public HRegionForTesting(final Path tableDir, final WAL wal, final FileSystem fs, 7439 final Configuration confParam, final RegionInfo regionInfo, final TableDescriptor htd, 7440 final RegionServerServices rsServices) { 7441 this(new HRegionFileSystem(confParam, fs, tableDir, regionInfo), wal, confParam, htd, 7442 rsServices); 7443 } 7444 7445 public HRegionForTesting(HRegionFileSystem fs, WAL wal, Configuration confParam, 7446 TableDescriptor htd, RegionServerServices rsServices) { 7447 super(fs, wal, confParam, htd, rsServices); 7448 } 7449 7450 /** 7451 * Create HStore instance. 7452 * @return If Mob is enabled, return HMobStore, otherwise return HStoreForTesting. 7453 */ 7454 @Override 7455 protected HStore instantiateHStore(final ColumnFamilyDescriptor family, boolean warmup) 7456 throws IOException { 7457 if (family.isMobEnabled()) { 7458 if (HFile.getFormatVersion(this.conf) < HFile.MIN_FORMAT_VERSION_WITH_TAGS) { 7459 throw new IOException("A minimum HFile version of " + HFile.MIN_FORMAT_VERSION_WITH_TAGS 7460 + " is required for MOB feature. Consider setting " + HFile.FORMAT_VERSION_KEY 7461 + " accordingly."); 7462 } 7463 return new HMobStore(this, family, this.conf, warmup); 7464 } 7465 return new HStoreForTesting(this, family, this.conf, warmup); 7466 } 7467 } 7468 7469 /** 7470 * HStoreForTesting is merely the same as HStore, the difference is in the doCompaction method of 7471 * HStoreForTesting there is a checkpoint "hbase.hstore.compaction.complete" which doesn't let 7472 * hstore compaction complete. In the former edition, this config is set in HStore class inside 7473 * compact method, though this is just for testing, otherwise it doesn't do any help. In 7474 * HBASE-8518, we try to get rid of all "hbase.hstore.compaction.complete" config (except for 7475 * testing code). 7476 */ 7477 public static class HStoreForTesting extends HStore { 7478 7479 protected HStoreForTesting(final HRegion region, final ColumnFamilyDescriptor family, 7480 final Configuration confParam, boolean warmup) throws IOException { 7481 super(region, family, confParam, warmup); 7482 } 7483 7484 @Override 7485 protected List<HStoreFile> doCompaction(CompactionRequestImpl cr, 7486 Collection<HStoreFile> filesToCompact, User user, long compactionStartTime, 7487 List<Path> newFiles) throws IOException { 7488 // let compaction incomplete. 7489 if (!this.conf.getBoolean("hbase.hstore.compaction.complete", true)) { 7490 LOG.warn("hbase.hstore.compaction.complete is set to false"); 7491 List<HStoreFile> sfs = new ArrayList<>(newFiles.size()); 7492 final boolean evictOnClose = 7493 getCacheConfig() != null ? getCacheConfig().shouldEvictOnClose() : true; 7494 for (Path newFile : newFiles) { 7495 // Create storefile around what we wrote with a reader on it. 7496 HStoreFile sf = storeEngine.createStoreFileAndReader(newFile); 7497 sf.closeStoreFile(evictOnClose); 7498 sfs.add(sf); 7499 } 7500 return sfs; 7501 } 7502 return super.doCompaction(cr, filesToCompact, user, compactionStartTime, newFiles); 7503 } 7504 } 7505 7506 @Test 7507 public void testCloseNoInterrupt() throws Exception { 7508 byte[] cf1 = Bytes.toBytes("CF1"); 7509 byte[][] families = { cf1 }; 7510 final int SLEEP_TIME = 10 * 1000; 7511 7512 Configuration conf = new Configuration(CONF); 7513 // Disable close thread interrupt and server abort behavior 7514 conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, false); 7515 conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, 1000); 7516 region = initHRegion(tableName, method, conf, families); 7517 7518 final CountDownLatch latch = new CountDownLatch(1); 7519 final AtomicBoolean holderInterrupted = new AtomicBoolean(); 7520 Thread holder = new Thread(new Runnable() { 7521 @Override 7522 public void run() { 7523 try { 7524 LOG.info("Starting region operation holder"); 7525 region.startRegionOperation(Operation.SCAN); 7526 latch.countDown(); 7527 try { 7528 Thread.sleep(SLEEP_TIME); 7529 } catch (InterruptedException e) { 7530 LOG.info("Interrupted"); 7531 holderInterrupted.set(true); 7532 } 7533 } catch (Exception e) { 7534 throw new RuntimeException(e); 7535 } finally { 7536 try { 7537 region.closeRegionOperation(); 7538 } catch (IOException e) { 7539 } 7540 LOG.info("Stopped region operation holder"); 7541 } 7542 } 7543 }); 7544 7545 holder.start(); 7546 latch.await(); 7547 region.close(); 7548 region = null; 7549 holder.join(); 7550 7551 assertFalse("Region lock holder should not have been interrupted", holderInterrupted.get()); 7552 } 7553 7554 @Test 7555 public void testCloseInterrupt() throws Exception { 7556 byte[] cf1 = Bytes.toBytes("CF1"); 7557 byte[][] families = { cf1 }; 7558 final int SLEEP_TIME = 10 * 1000; 7559 7560 Configuration conf = new Configuration(CONF); 7561 // Enable close thread interrupt and server abort behavior 7562 conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, true); 7563 // Speed up the unit test, no need to wait default 10 seconds. 7564 conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, 1000); 7565 region = initHRegion(tableName, method, conf, families); 7566 7567 final CountDownLatch latch = new CountDownLatch(1); 7568 final AtomicBoolean holderInterrupted = new AtomicBoolean(); 7569 Thread holder = new Thread(new Runnable() { 7570 @Override 7571 public void run() { 7572 try { 7573 LOG.info("Starting region operation holder"); 7574 region.startRegionOperation(Operation.SCAN); 7575 latch.countDown(); 7576 try { 7577 Thread.sleep(SLEEP_TIME); 7578 } catch (InterruptedException e) { 7579 LOG.info("Interrupted"); 7580 holderInterrupted.set(true); 7581 } 7582 } catch (Exception e) { 7583 throw new RuntimeException(e); 7584 } finally { 7585 try { 7586 region.closeRegionOperation(); 7587 } catch (IOException e) { 7588 } 7589 LOG.info("Stopped region operation holder"); 7590 } 7591 } 7592 }); 7593 7594 holder.start(); 7595 latch.await(); 7596 region.close(); 7597 region = null; 7598 holder.join(); 7599 7600 assertTrue("Region lock holder was not interrupted", holderInterrupted.get()); 7601 } 7602 7603 @Test 7604 public void testCloseAbort() throws Exception { 7605 byte[] cf1 = Bytes.toBytes("CF1"); 7606 byte[][] families = { cf1 }; 7607 final int SLEEP_TIME = 10 * 1000; 7608 7609 Configuration conf = new Configuration(CONF); 7610 // Enable close thread interrupt and server abort behavior. 7611 conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, true); 7612 // Set the abort interval to a fraction of sleep time so we are guaranteed to be aborted. 7613 conf.setInt(HRegion.CLOSE_WAIT_TIME, SLEEP_TIME / 2); 7614 // Set the wait interval to a fraction of sleep time so we are guaranteed to be interrupted. 7615 conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, SLEEP_TIME / 4); 7616 region = initHRegion(tableName, method, conf, families); 7617 RegionServerServices rsServices = mock(RegionServerServices.class); 7618 when(rsServices.getServerName()).thenReturn(ServerName.valueOf("localhost", 1000, 1000)); 7619 region.rsServices = rsServices; 7620 7621 final CountDownLatch latch = new CountDownLatch(1); 7622 Thread holder = new Thread(new Runnable() { 7623 @Override 7624 public void run() { 7625 try { 7626 LOG.info("Starting region operation holder"); 7627 region.startRegionOperation(Operation.SCAN); 7628 latch.countDown(); 7629 // Hold the lock for SLEEP_TIME seconds no matter how many times we are interrupted. 7630 int timeRemaining = SLEEP_TIME; 7631 while (timeRemaining > 0) { 7632 long start = EnvironmentEdgeManager.currentTime(); 7633 try { 7634 Thread.sleep(timeRemaining); 7635 } catch (InterruptedException e) { 7636 LOG.info("Interrupted"); 7637 } 7638 long end = EnvironmentEdgeManager.currentTime(); 7639 timeRemaining -= end - start; 7640 if (timeRemaining < 0) { 7641 timeRemaining = 0; 7642 } 7643 if (timeRemaining > 0) { 7644 LOG.info("Sleeping again, remaining time " + timeRemaining + " ms"); 7645 } 7646 } 7647 } catch (Exception e) { 7648 throw new RuntimeException(e); 7649 } finally { 7650 try { 7651 region.closeRegionOperation(); 7652 } catch (IOException e) { 7653 } 7654 LOG.info("Stopped region operation holder"); 7655 } 7656 } 7657 }); 7658 7659 holder.start(); 7660 latch.await(); 7661 try { 7662 region.close(); 7663 } catch (IOException e) { 7664 LOG.info("Caught expected exception", e); 7665 } 7666 region = null; 7667 holder.join(); 7668 7669 // Verify the region tried to abort the server 7670 verify(rsServices, atLeast(1)).abort(anyString(), any()); 7671 } 7672 7673 @Test 7674 public void testInterruptProtection() throws Exception { 7675 byte[] cf1 = Bytes.toBytes("CF1"); 7676 byte[][] families = { cf1 }; 7677 final int SLEEP_TIME = 10 * 1000; 7678 7679 Configuration conf = new Configuration(CONF); 7680 // Enable close thread interrupt and server abort behavior. 7681 conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, true); 7682 conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, 1000); 7683 region = initHRegion(tableName, method, conf, families); 7684 7685 final CountDownLatch latch = new CountDownLatch(1); 7686 final AtomicBoolean holderInterrupted = new AtomicBoolean(); 7687 Thread holder = new Thread(new Runnable() { 7688 @Override 7689 public void run() { 7690 try { 7691 LOG.info("Starting region operation holder"); 7692 region.startRegionOperation(Operation.SCAN); 7693 LOG.info("Protecting against interrupts"); 7694 region.disableInterrupts(); 7695 try { 7696 latch.countDown(); 7697 try { 7698 Thread.sleep(SLEEP_TIME); 7699 } catch (InterruptedException e) { 7700 LOG.info("Interrupted"); 7701 holderInterrupted.set(true); 7702 } 7703 } finally { 7704 region.enableInterrupts(); 7705 } 7706 } catch (Exception e) { 7707 throw new RuntimeException(e); 7708 } finally { 7709 try { 7710 region.closeRegionOperation(); 7711 } catch (IOException e) { 7712 } 7713 LOG.info("Stopped region operation holder"); 7714 } 7715 } 7716 }); 7717 7718 holder.start(); 7719 latch.await(); 7720 region.close(); 7721 region = null; 7722 holder.join(); 7723 7724 assertFalse("Region lock holder should not have been interrupted", holderInterrupted.get()); 7725 } 7726 7727 @Test 7728 public void testRegionOnCoprocessorsChange() throws IOException { 7729 byte[] cf1 = Bytes.toBytes("CF1"); 7730 byte[][] families = { cf1 }; 7731 7732 Configuration conf = new Configuration(CONF); 7733 region = initHRegion(tableName, method, conf, families); 7734 assertNull(region.getCoprocessorHost()); 7735 7736 // set and verify the system coprocessors for region and user region 7737 Configuration newConf = new Configuration(conf); 7738 newConf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, MetaTableMetrics.class.getName()); 7739 newConf.set(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, 7740 NoOpRegionCoprocessor.class.getName()); 7741 // trigger configuration change 7742 region.onConfigurationChange(newConf); 7743 assertTrue(region.getCoprocessorHost() != null); 7744 Set<String> coprocessors = region.getCoprocessorHost().getCoprocessors(); 7745 assertTrue(coprocessors.size() == 2); 7746 assertTrue(region.getCoprocessorHost().getCoprocessors() 7747 .contains(MetaTableMetrics.class.getSimpleName())); 7748 assertTrue(region.getCoprocessorHost().getCoprocessors() 7749 .contains(NoOpRegionCoprocessor.class.getSimpleName())); 7750 7751 // remove region coprocessor and keep only user region coprocessor 7752 newConf.unset(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY); 7753 region.onConfigurationChange(newConf); 7754 assertTrue(region.getCoprocessorHost() != null); 7755 coprocessors = region.getCoprocessorHost().getCoprocessors(); 7756 assertTrue(coprocessors.size() == 1); 7757 assertTrue(region.getCoprocessorHost().getCoprocessors() 7758 .contains(NoOpRegionCoprocessor.class.getSimpleName())); 7759 } 7760 7761 @Test 7762 public void testRegionOnCoprocessorsWithoutChange() throws IOException { 7763 byte[] cf1 = Bytes.toBytes("CF1"); 7764 byte[][] families = { cf1 }; 7765 7766 Configuration conf = new Configuration(CONF); 7767 conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, 7768 MetaTableMetrics.class.getCanonicalName()); 7769 region = initHRegion(tableName, method, conf, families); 7770 // region service is null in unit test, we need to load the coprocessor once 7771 region.setCoprocessorHost(new RegionCoprocessorHost(region, null, conf)); 7772 RegionCoprocessor regionCoprocessor = 7773 region.getCoprocessorHost().findCoprocessor(MetaTableMetrics.class.getName()); 7774 7775 // simulate when other configuration may have changed and onConfigurationChange execute once 7776 region.onConfigurationChange(conf); 7777 RegionCoprocessor regionCoprocessorAfterOnConfigurationChange = 7778 region.getCoprocessorHost().findCoprocessor(MetaTableMetrics.class.getName()); 7779 assertEquals(regionCoprocessor, regionCoprocessorAfterOnConfigurationChange); 7780 } 7781 7782 public static class NoOpRegionCoprocessor implements RegionCoprocessor, RegionObserver { 7783 // a empty region coprocessor class 7784 } 7785}