001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.regionserver; 019 020import static org.apache.hadoop.hbase.HBaseTestingUtil.COLUMNS; 021import static org.apache.hadoop.hbase.HBaseTestingUtil.fam1; 022import static org.apache.hadoop.hbase.HBaseTestingUtil.fam2; 023import static org.apache.hadoop.hbase.HBaseTestingUtil.fam3; 024import static org.junit.Assert.assertArrayEquals; 025import static org.junit.Assert.assertEquals; 026import static org.junit.Assert.assertFalse; 027import static org.junit.Assert.assertNotNull; 028import static org.junit.Assert.assertNull; 029import static org.junit.Assert.assertThrows; 030import static org.junit.Assert.assertTrue; 031import static org.junit.Assert.fail; 032import static org.mockito.ArgumentMatchers.any; 033import static org.mockito.ArgumentMatchers.anyLong; 034import static org.mockito.ArgumentMatchers.anyString; 035import static org.mockito.ArgumentMatchers.isA; 036import static org.mockito.Mockito.atLeast; 037import static org.mockito.Mockito.doAnswer; 038import static org.mockito.Mockito.doThrow; 039import static org.mockito.Mockito.mock; 040import static org.mockito.Mockito.never; 041import static org.mockito.Mockito.spy; 042import static org.mockito.Mockito.times; 043import static org.mockito.Mockito.verify; 044import static org.mockito.Mockito.when; 045 046import java.io.IOException; 047import java.io.InterruptedIOException; 048import java.math.BigDecimal; 049import java.security.PrivilegedExceptionAction; 050import java.util.ArrayList; 051import java.util.Arrays; 052import java.util.Collection; 053import java.util.List; 054import java.util.Map; 055import java.util.NavigableMap; 056import java.util.Objects; 057import java.util.Set; 058import java.util.TreeMap; 059import java.util.concurrent.Callable; 060import java.util.concurrent.CountDownLatch; 061import java.util.concurrent.ExecutorService; 062import java.util.concurrent.Executors; 063import java.util.concurrent.Future; 064import java.util.concurrent.TimeUnit; 065import java.util.concurrent.atomic.AtomicBoolean; 066import java.util.concurrent.atomic.AtomicInteger; 067import java.util.concurrent.atomic.AtomicLong; 068import java.util.concurrent.atomic.AtomicReference; 069import org.apache.commons.lang3.RandomStringUtils; 070import org.apache.hadoop.conf.Configuration; 071import org.apache.hadoop.fs.FSDataOutputStream; 072import org.apache.hadoop.fs.FileStatus; 073import org.apache.hadoop.fs.FileSystem; 074import org.apache.hadoop.fs.Path; 075import org.apache.hadoop.hbase.ArrayBackedTag; 076import org.apache.hadoop.hbase.Cell; 077import org.apache.hadoop.hbase.Cell.Type; 078import org.apache.hadoop.hbase.CellBuilderFactory; 079import org.apache.hadoop.hbase.CellBuilderType; 080import org.apache.hadoop.hbase.CellUtil; 081import org.apache.hadoop.hbase.CompareOperator; 082import org.apache.hadoop.hbase.CompatibilitySingletonFactory; 083import org.apache.hadoop.hbase.DoNotRetryIOException; 084import org.apache.hadoop.hbase.DroppedSnapshotException; 085import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; 086import org.apache.hadoop.hbase.HBaseClassTestRule; 087import org.apache.hadoop.hbase.HBaseConfiguration; 088import org.apache.hadoop.hbase.HBaseTestingUtil; 089import org.apache.hadoop.hbase.HConstants; 090import org.apache.hadoop.hbase.HConstants.OperationStatusCode; 091import org.apache.hadoop.hbase.HDFSBlocksDistribution; 092import org.apache.hadoop.hbase.KeyValue; 093import org.apache.hadoop.hbase.MultithreadedTestUtil; 094import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread; 095import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread; 096import org.apache.hadoop.hbase.NotServingRegionException; 097import org.apache.hadoop.hbase.PrivateCellUtil; 098import org.apache.hadoop.hbase.RegionTooBusyException; 099import org.apache.hadoop.hbase.ServerName; 100import org.apache.hadoop.hbase.SingleProcessHBaseCluster; 101import org.apache.hadoop.hbase.StartTestingClusterOption; 102import org.apache.hadoop.hbase.TableName; 103import org.apache.hadoop.hbase.TagType; 104import org.apache.hadoop.hbase.Waiter; 105import org.apache.hadoop.hbase.client.Append; 106import org.apache.hadoop.hbase.client.CheckAndMutate; 107import org.apache.hadoop.hbase.client.CheckAndMutateResult; 108import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; 109import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 110import org.apache.hadoop.hbase.client.Delete; 111import org.apache.hadoop.hbase.client.Durability; 112import org.apache.hadoop.hbase.client.Get; 113import org.apache.hadoop.hbase.client.Increment; 114import org.apache.hadoop.hbase.client.Mutation; 115import org.apache.hadoop.hbase.client.Put; 116import org.apache.hadoop.hbase.client.RegionInfo; 117import org.apache.hadoop.hbase.client.RegionInfoBuilder; 118import org.apache.hadoop.hbase.client.Result; 119import org.apache.hadoop.hbase.client.RowMutations; 120import org.apache.hadoop.hbase.client.Scan; 121import org.apache.hadoop.hbase.client.Table; 122import org.apache.hadoop.hbase.client.TableDescriptor; 123import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 124import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; 125import org.apache.hadoop.hbase.coprocessor.MetaTableMetrics; 126import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor; 127import org.apache.hadoop.hbase.coprocessor.RegionObserver; 128import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException; 129import org.apache.hadoop.hbase.filter.BigDecimalComparator; 130import org.apache.hadoop.hbase.filter.BinaryComparator; 131import org.apache.hadoop.hbase.filter.ColumnCountGetFilter; 132import org.apache.hadoop.hbase.filter.Filter; 133import org.apache.hadoop.hbase.filter.FilterBase; 134import org.apache.hadoop.hbase.filter.FilterList; 135import org.apache.hadoop.hbase.filter.NullComparator; 136import org.apache.hadoop.hbase.filter.PrefixFilter; 137import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter; 138import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; 139import org.apache.hadoop.hbase.filter.SubstringComparator; 140import org.apache.hadoop.hbase.filter.ValueFilter; 141import org.apache.hadoop.hbase.io.TimeRange; 142import org.apache.hadoop.hbase.io.hfile.HFile; 143import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler; 144import org.apache.hadoop.hbase.monitoring.MonitoredTask; 145import org.apache.hadoop.hbase.monitoring.TaskMonitor; 146import org.apache.hadoop.hbase.regionserver.Region.Operation; 147import org.apache.hadoop.hbase.regionserver.Region.RowLock; 148import org.apache.hadoop.hbase.regionserver.TestHStore.FaultyFileSystem; 149import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl; 150import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL; 151import org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL; 152import org.apache.hadoop.hbase.regionserver.wal.FSHLog; 153import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource; 154import org.apache.hadoop.hbase.regionserver.wal.WALUtil; 155import org.apache.hadoop.hbase.replication.regionserver.ReplicationObserver; 156import org.apache.hadoop.hbase.security.User; 157import org.apache.hadoop.hbase.test.MetricsAssertHelper; 158import org.apache.hadoop.hbase.testclassification.LargeTests; 159import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; 160import org.apache.hadoop.hbase.util.Bytes; 161import org.apache.hadoop.hbase.util.CommonFSUtils; 162import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 163import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; 164import org.apache.hadoop.hbase.util.HFileArchiveUtil; 165import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge; 166import org.apache.hadoop.hbase.util.ManualEnvironmentEdge; 167import org.apache.hadoop.hbase.util.Threads; 168import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; 169import org.apache.hadoop.hbase.wal.FaultyFSLog; 170import org.apache.hadoop.hbase.wal.NettyAsyncFSWALConfigHelper; 171import org.apache.hadoop.hbase.wal.WAL; 172import org.apache.hadoop.hbase.wal.WALEdit; 173import org.apache.hadoop.hbase.wal.WALFactory; 174import org.apache.hadoop.hbase.wal.WALKeyImpl; 175import org.apache.hadoop.hbase.wal.WALProvider; 176import org.apache.hadoop.hbase.wal.WALProvider.Writer; 177import org.apache.hadoop.hbase.wal.WALSplitUtil; 178import org.apache.hadoop.hbase.wal.WALStreamReader; 179import org.junit.After; 180import org.junit.Assert; 181import org.junit.Before; 182import org.junit.ClassRule; 183import org.junit.Ignore; 184import org.junit.Rule; 185import org.junit.Test; 186import org.junit.experimental.categories.Category; 187import org.junit.rules.ExpectedException; 188import org.junit.rules.TestName; 189import org.mockito.ArgumentCaptor; 190import org.mockito.ArgumentMatcher; 191import org.mockito.invocation.InvocationOnMock; 192import org.mockito.stubbing.Answer; 193import org.slf4j.Logger; 194import org.slf4j.LoggerFactory; 195 196import org.apache.hbase.thirdparty.com.google.common.collect.Lists; 197import org.apache.hbase.thirdparty.com.google.protobuf.ByteString; 198import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup; 199import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup; 200import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel; 201 202import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; 203import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor; 204import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor; 205import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.FlushAction; 206import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor; 207import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor; 208import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.StoreDescriptor; 209 210/** 211 * Basic stand-alone testing of HRegion. No clusters! A lot of the meta information for an HRegion 212 * now lives inside other HRegions or in the HBaseMaster, so only basic testing is possible. 213 */ 214@Category({ VerySlowRegionServerTests.class, LargeTests.class }) 215@SuppressWarnings("deprecation") 216public class TestHRegion { 217 218 @ClassRule 219 public static final HBaseClassTestRule CLASS_RULE = 220 HBaseClassTestRule.forClass(TestHRegion.class); 221 222 // Do not spin up clusters in here. If you need to spin up a cluster, do it 223 // over in TestHRegionOnCluster. 224 private static final Logger LOG = LoggerFactory.getLogger(TestHRegion.class); 225 @Rule 226 public TestName name = new TestName(); 227 @Rule 228 public final ExpectedException thrown = ExpectedException.none(); 229 230 private static final String COLUMN_FAMILY = "MyCF"; 231 private static final byte[] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY); 232 private static final EventLoopGroup GROUP = new NioEventLoopGroup(); 233 234 HRegion region = null; 235 // Do not run unit tests in parallel (? Why not? It don't work? Why not? St.Ack) 236 protected static HBaseTestingUtil TEST_UTIL; 237 public static Configuration CONF; 238 private String dir; 239 private final int MAX_VERSIONS = 2; 240 241 // Test names 242 protected TableName tableName; 243 protected String method; 244 protected final byte[] qual = Bytes.toBytes("qual"); 245 protected final byte[] qual1 = Bytes.toBytes("qual1"); 246 protected final byte[] qual2 = Bytes.toBytes("qual2"); 247 protected final byte[] qual3 = Bytes.toBytes("qual3"); 248 protected final byte[] value = Bytes.toBytes("value"); 249 protected final byte[] value1 = Bytes.toBytes("value1"); 250 protected final byte[] value2 = Bytes.toBytes("value2"); 251 protected final byte[] row = Bytes.toBytes("rowA"); 252 protected final byte[] row2 = Bytes.toBytes("rowB"); 253 254 protected final MetricsAssertHelper metricsAssertHelper = 255 CompatibilitySingletonFactory.getInstance(MetricsAssertHelper.class); 256 257 @Before 258 public void setup() throws IOException { 259 TEST_UTIL = new HBaseTestingUtil(); 260 CONF = TEST_UTIL.getConfiguration(); 261 NettyAsyncFSWALConfigHelper.setEventLoopConfig(CONF, GROUP, NioSocketChannel.class); 262 dir = TEST_UTIL.getDataTestDir("TestHRegion").toString(); 263 method = name.getMethodName(); 264 tableName = TableName.valueOf(method); 265 CONF.set(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, String.valueOf(0.09)); 266 CONF.setLong(AbstractFSWAL.WAL_SYNC_TIMEOUT_MS, 10000); 267 } 268 269 @After 270 public void tearDown() throws IOException { 271 // Region may have been closed, but it is still no harm if we close it again here using HTU. 272 HBaseTestingUtil.closeRegionAndWAL(region); 273 EnvironmentEdgeManagerTestHelper.reset(); 274 LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir()); 275 TEST_UTIL.cleanupTestDir(); 276 } 277 278 /** 279 * Test that I can use the max flushed sequence id after the close. 280 */ 281 @Test 282 public void testSequenceId() throws IOException { 283 region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES); 284 assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId()); 285 // Weird. This returns 0 if no store files or no edits. Afraid to change it. 286 assertEquals(0, (long) region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES)); 287 HBaseTestingUtil.closeRegionAndWAL(this.region); 288 assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId()); 289 assertEquals(0, (long) region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES)); 290 // Open region again. 291 region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES); 292 byte[] value = Bytes.toBytes(method); 293 // Make a random put against our cf. 294 Put put = new Put(value); 295 put.addColumn(COLUMN_FAMILY_BYTES, null, value); 296 region.put(put); 297 // No flush yet so init numbers should still be in place. 298 assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId()); 299 assertEquals(0, (long) region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES)); 300 region.flush(true); 301 long max = region.getMaxFlushedSeqId(); 302 HBaseTestingUtil.closeRegionAndWAL(this.region); 303 assertEquals(max, region.getMaxFlushedSeqId()); 304 this.region = null; 305 } 306 307 /** 308 * Test for Bug 2 of HBASE-10466. "Bug 2: Conditions for the first flush of region close 309 * (so-called pre-flush) If memstoreSize is smaller than a certain value, or when region close 310 * starts a flush is ongoing, the first flush is skipped and only the second flush takes place. 311 * However, two flushes are required in case previous flush fails and leaves some data in 312 * snapshot. The bug could cause loss of data in current memstore. The fix is removing all 313 * conditions except abort check so we ensure 2 flushes for region close." 314 */ 315 @Test 316 public void testCloseCarryingSnapshot() throws IOException { 317 region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES); 318 HStore store = region.getStore(COLUMN_FAMILY_BYTES); 319 // Get some random bytes. 320 byte[] value = Bytes.toBytes(method); 321 // Make a random put against our cf. 322 Put put = new Put(value); 323 put.addColumn(COLUMN_FAMILY_BYTES, null, value); 324 // First put something in current memstore, which will be in snapshot after flusher.prepare() 325 region.put(put); 326 StoreFlushContext storeFlushCtx = store.createFlushContext(12345, FlushLifeCycleTracker.DUMMY); 327 storeFlushCtx.prepare(); 328 // Second put something in current memstore 329 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value); 330 region.put(put); 331 // Close with something in memstore and something in the snapshot. Make sure all is cleared. 332 HBaseTestingUtil.closeRegionAndWAL(region); 333 assertEquals(0, region.getMemStoreDataSize()); 334 region = null; 335 } 336 337 /* 338 * This test is for verifying memstore snapshot size is correctly updated in case of rollback See 339 * HBASE-10845 340 */ 341 @Test 342 public void testMemstoreSnapshotSize() throws IOException { 343 class MyFaultyFSLog extends FaultyFSLog { 344 StoreFlushContext storeFlushCtx; 345 346 public MyFaultyFSLog(FileSystem fs, Path rootDir, String logName, Configuration conf) 347 throws IOException { 348 super(fs, rootDir, logName, conf); 349 } 350 351 void setStoreFlushCtx(StoreFlushContext storeFlushCtx) { 352 this.storeFlushCtx = storeFlushCtx; 353 } 354 355 @Override 356 protected void doSync(long txid, boolean forceSync) throws IOException { 357 storeFlushCtx.prepare(); 358 super.doSync(txid, forceSync); 359 } 360 } 361 362 FileSystem fs = FileSystem.get(CONF); 363 Path rootDir = new Path(dir + "testMemstoreSnapshotSize"); 364 MyFaultyFSLog faultyLog = new MyFaultyFSLog(fs, rootDir, "testMemstoreSnapshotSize", CONF); 365 faultyLog.init(); 366 region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, faultyLog, 367 COLUMN_FAMILY_BYTES); 368 369 HStore store = region.getStore(COLUMN_FAMILY_BYTES); 370 // Get some random bytes. 371 byte[] value = Bytes.toBytes(method); 372 faultyLog.setStoreFlushCtx(store.createFlushContext(12345, FlushLifeCycleTracker.DUMMY)); 373 374 Put put = new Put(value); 375 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value); 376 faultyLog.setFailureType(FaultyFSLog.FailureType.SYNC); 377 boolean threwIOE = false; 378 try { 379 region.put(put); 380 } catch (IOException ioe) { 381 threwIOE = true; 382 } finally { 383 assertTrue("The regionserver should have thrown an exception", threwIOE); 384 } 385 MemStoreSize mss = store.getFlushableSize(); 386 assertTrue("flushable size should be zero, but it is " + mss, mss.getDataSize() == 0); 387 } 388 389 /** 390 * Create a WAL outside of the usual helper in 391 * {@link HBaseTestingUtil#createWal(Configuration, Path, RegionInfo)} because that method doesn't 392 * play nicely with FaultyFileSystem. Call this method before overriding {@code fs.file.impl}. 393 * @param callingMethod a unique component for the path, probably the name of the test method. 394 */ 395 private static WAL createWALCompatibleWithFaultyFileSystem(String callingMethod, 396 Configuration conf, TableName tableName) throws IOException { 397 final Path logDir = TEST_UTIL.getDataTestDirOnTestFS(callingMethod + ".log"); 398 final Configuration walConf = new Configuration(conf); 399 CommonFSUtils.setRootDir(walConf, logDir); 400 return new WALFactory(walConf, callingMethod) 401 .getWAL(RegionInfoBuilder.newBuilder(tableName).build()); 402 } 403 404 @Test 405 public void testMemstoreSizeAccountingWithFailedPostBatchMutate() throws IOException { 406 String testName = "testMemstoreSizeAccountingWithFailedPostBatchMutate"; 407 FileSystem fs = FileSystem.get(CONF); 408 Path rootDir = new Path(dir + testName); 409 FSHLog hLog = new FSHLog(fs, rootDir, testName, CONF); 410 hLog.init(); 411 region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, hLog, 412 COLUMN_FAMILY_BYTES); 413 HStore store = region.getStore(COLUMN_FAMILY_BYTES); 414 assertEquals(0, region.getMemStoreDataSize()); 415 416 // Put one value 417 byte[] value = Bytes.toBytes(method); 418 Put put = new Put(value); 419 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value); 420 region.put(put); 421 long onePutSize = region.getMemStoreDataSize(); 422 assertTrue(onePutSize > 0); 423 424 RegionCoprocessorHost mockedCPHost = mock(RegionCoprocessorHost.class); 425 doThrow(new IOException()).when(mockedCPHost).postBatchMutate(any()); 426 region.setCoprocessorHost(mockedCPHost); 427 428 put = new Put(value); 429 put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("dfg"), value); 430 try { 431 region.put(put); 432 fail("Should have failed with IOException"); 433 } catch (IOException expected) { 434 } 435 long expectedSize = onePutSize * 2; 436 assertEquals("memstoreSize should be incremented", expectedSize, region.getMemStoreDataSize()); 437 assertEquals("flushable size should be incremented", expectedSize, 438 store.getFlushableSize().getDataSize()); 439 440 region.setCoprocessorHost(null); 441 } 442 443 /** 444 * A test case of HBASE-21041 445 * @throws Exception Exception 446 */ 447 @Test 448 public void testFlushAndMemstoreSizeCounting() throws Exception { 449 byte[] family = Bytes.toBytes("family"); 450 this.region = initHRegion(tableName, method, CONF, family); 451 final WALFactory wals = new WALFactory(CONF, method); 452 try { 453 for (byte[] row : HBaseTestingUtil.ROWS) { 454 Put put = new Put(row); 455 put.addColumn(family, family, row); 456 region.put(put); 457 } 458 region.flush(true); 459 // After flush, data size should be zero 460 assertEquals(0, region.getMemStoreDataSize()); 461 // After flush, a new active mutable segment is created, so the heap size 462 // should equal to MutableSegment.DEEP_OVERHEAD 463 assertEquals(MutableSegment.DEEP_OVERHEAD, region.getMemStoreHeapSize()); 464 // After flush, offheap should be zero 465 assertEquals(0, region.getMemStoreOffHeapSize()); 466 } finally { 467 HBaseTestingUtil.closeRegionAndWAL(this.region); 468 this.region = null; 469 wals.close(); 470 } 471 } 472 473 /** 474 * Test we do not lose data if we fail a flush and then close. Part of HBase-10466. Tests the 475 * following from the issue description: "Bug 1: Wrong calculation of HRegion.memstoreSize: When a 476 * flush fails, data to be flushed is kept in each MemStore's snapshot and wait for next flush 477 * attempt to continue on it. But when the next flush succeeds, the counter of total memstore size 478 * in HRegion is always deduced by the sum of current memstore sizes instead of snapshots left 479 * from previous failed flush. This calculation is problematic that almost every time there is 480 * failed flush, HRegion.memstoreSize gets reduced by a wrong value. If region flush could not 481 * proceed for a couple cycles, the size in current memstore could be much larger than the 482 * snapshot. It's likely to drift memstoreSize much smaller than expected. In extreme case, if the 483 * error accumulates to even bigger than HRegion's memstore size limit, any further flush is 484 * skipped because flush does not do anything if memstoreSize is not larger than 0." 485 */ 486 @Test 487 public void testFlushSizeAccounting() throws Exception { 488 final Configuration conf = HBaseConfiguration.create(CONF); 489 final WAL wal = createWALCompatibleWithFaultyFileSystem(method, conf, tableName); 490 // Only retry once. 491 conf.setInt("hbase.hstore.flush.retries.number", 1); 492 final User user = User.createUserForTesting(conf, method, new String[] { "foo" }); 493 // Inject our faulty LocalFileSystem 494 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class); 495 user.runAs(new PrivilegedExceptionAction<Object>() { 496 @Override 497 public Object run() throws Exception { 498 // Make sure it worked (above is sensitive to caching details in hadoop core) 499 FileSystem fs = FileSystem.get(conf); 500 Assert.assertEquals(FaultyFileSystem.class, fs.getClass()); 501 FaultyFileSystem ffs = (FaultyFileSystem) fs; 502 HRegion region = null; 503 try { 504 // Initialize region 505 region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, wal, 506 COLUMN_FAMILY_BYTES); 507 long size = region.getMemStoreDataSize(); 508 Assert.assertEquals(0, size); 509 // Put one item into memstore. Measure the size of one item in memstore. 510 Put p1 = new Put(row); 511 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[]) null)); 512 region.put(p1); 513 final long sizeOfOnePut = region.getMemStoreDataSize(); 514 // Fail a flush which means the current memstore will hang out as memstore 'snapshot'. 515 try { 516 LOG.info("Flushing"); 517 region.flush(true); 518 Assert.fail("Didn't bubble up IOE!"); 519 } catch (DroppedSnapshotException dse) { 520 // What we are expecting 521 region.closing.set(false); // this is needed for the rest of the test to work 522 } 523 // Make it so all writes succeed from here on out 524 ffs.fault.set(false); 525 // Check sizes. Should still be the one entry. 526 Assert.assertEquals(sizeOfOnePut, region.getMemStoreDataSize()); 527 // Now add two entries so that on this next flush that fails, we can see if we 528 // subtract the right amount, the snapshot size only. 529 Put p2 = new Put(row); 530 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[]) null)); 531 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[]) null)); 532 region.put(p2); 533 long expectedSize = sizeOfOnePut * 3; 534 Assert.assertEquals(expectedSize, region.getMemStoreDataSize()); 535 // Do a successful flush. It will clear the snapshot only. Thats how flushes work. 536 // If already a snapshot, we clear it else we move the memstore to be snapshot and flush 537 // it 538 region.flush(true); 539 // Make sure our memory accounting is right. 540 Assert.assertEquals(sizeOfOnePut * 2, region.getMemStoreDataSize()); 541 } finally { 542 HBaseTestingUtil.closeRegionAndWAL(region); 543 } 544 return null; 545 } 546 }); 547 FileSystem.closeAllForUGI(user.getUGI()); 548 } 549 550 @Test 551 public void testCloseWithFailingFlush() throws Exception { 552 final Configuration conf = HBaseConfiguration.create(CONF); 553 final WAL wal = createWALCompatibleWithFaultyFileSystem(method, conf, tableName); 554 // Only retry once. 555 conf.setInt("hbase.hstore.flush.retries.number", 1); 556 final User user = User.createUserForTesting(conf, this.method, new String[] { "foo" }); 557 // Inject our faulty LocalFileSystem 558 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class); 559 user.runAs(new PrivilegedExceptionAction<Object>() { 560 @Override 561 public Object run() throws Exception { 562 // Make sure it worked (above is sensitive to caching details in hadoop core) 563 FileSystem fs = FileSystem.get(conf); 564 Assert.assertEquals(FaultyFileSystem.class, fs.getClass()); 565 FaultyFileSystem ffs = (FaultyFileSystem) fs; 566 HRegion region = null; 567 try { 568 // Initialize region 569 region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, wal, 570 COLUMN_FAMILY_BYTES); 571 long size = region.getMemStoreDataSize(); 572 Assert.assertEquals(0, size); 573 // Put one item into memstore. Measure the size of one item in memstore. 574 Put p1 = new Put(row); 575 p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[]) null)); 576 region.put(p1); 577 // Manufacture an outstanding snapshot -- fake a failed flush by doing prepare step only. 578 HStore store = region.getStore(COLUMN_FAMILY_BYTES); 579 StoreFlushContext storeFlushCtx = 580 store.createFlushContext(12345, FlushLifeCycleTracker.DUMMY); 581 storeFlushCtx.prepare(); 582 // Now add two entries to the foreground memstore. 583 Put p2 = new Put(row); 584 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[]) null)); 585 p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[]) null)); 586 region.put(p2); 587 // Now try close on top of a failing flush. 588 HBaseTestingUtil.closeRegionAndWAL(region); 589 region = null; 590 fail(); 591 } catch (DroppedSnapshotException dse) { 592 // Expected 593 LOG.info("Expected DroppedSnapshotException"); 594 } finally { 595 // Make it so all writes succeed from here on out so can close clean 596 ffs.fault.set(false); 597 HBaseTestingUtil.closeRegionAndWAL(region); 598 } 599 return null; 600 } 601 }); 602 FileSystem.closeAllForUGI(user.getUGI()); 603 } 604 605 @Test 606 public void testCompactionAffectedByScanners() throws Exception { 607 byte[] family = Bytes.toBytes("family"); 608 this.region = initHRegion(tableName, method, CONF, family); 609 610 Put put = new Put(Bytes.toBytes("r1")); 611 put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); 612 region.put(put); 613 region.flush(true); 614 615 Scan scan = new Scan(); 616 scan.readVersions(3); 617 // open the first scanner 618 RegionScanner scanner1 = region.getScanner(scan); 619 620 Delete delete = new Delete(Bytes.toBytes("r1")); 621 region.delete(delete); 622 region.flush(true); 623 624 // open the second scanner 625 RegionScanner scanner2 = region.getScanner(scan); 626 627 List<Cell> results = new ArrayList<>(); 628 629 System.out.println("Smallest read point:" + region.getSmallestReadPoint()); 630 631 // make a major compaction 632 region.compact(true); 633 634 // open the third scanner 635 RegionScanner scanner3 = region.getScanner(scan); 636 637 // get data from scanner 1, 2, 3 after major compaction 638 scanner1.next(results); 639 System.out.println(results); 640 assertEquals(1, results.size()); 641 642 results.clear(); 643 scanner2.next(results); 644 System.out.println(results); 645 assertEquals(0, results.size()); 646 647 results.clear(); 648 scanner3.next(results); 649 System.out.println(results); 650 assertEquals(0, results.size()); 651 } 652 653 @Test 654 public void testToShowNPEOnRegionScannerReseek() throws Exception { 655 byte[] family = Bytes.toBytes("family"); 656 this.region = initHRegion(tableName, method, CONF, family); 657 658 Put put = new Put(Bytes.toBytes("r1")); 659 put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); 660 region.put(put); 661 put = new Put(Bytes.toBytes("r2")); 662 put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); 663 region.put(put); 664 region.flush(true); 665 666 Scan scan = new Scan(); 667 scan.readVersions(3); 668 // open the first scanner 669 RegionScanner scanner1 = region.getScanner(scan); 670 671 System.out.println("Smallest read point:" + region.getSmallestReadPoint()); 672 673 region.compact(true); 674 675 scanner1.reseek(Bytes.toBytes("r2")); 676 List<Cell> results = new ArrayList<>(); 677 scanner1.next(results); 678 Cell keyValue = results.get(0); 679 Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0); 680 scanner1.close(); 681 } 682 683 @Test 684 public void testArchiveRecoveredEditsReplay() throws Exception { 685 byte[] family = Bytes.toBytes("family"); 686 this.region = initHRegion(tableName, method, CONF, family); 687 final WALFactory wals = new WALFactory(CONF, method); 688 try { 689 Path regiondir = region.getRegionFileSystem().getRegionDir(); 690 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 691 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 692 693 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 694 695 long maxSeqId = 1050; 696 long minSeqId = 1000; 697 698 for (long i = minSeqId; i <= maxSeqId; i += 10) { 699 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); 700 fs.create(recoveredEdits); 701 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 702 703 long time = System.nanoTime(); 704 WALEdit edit = new WALEdit(); 705 edit.add( 706 new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i))); 707 writer.append(new WAL.Entry( 708 new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); 709 710 writer.close(); 711 } 712 MonitoredTask status = TaskMonitor.get().createStatus(method); 713 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 714 for (HStore store : region.getStores()) { 715 maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId - 1); 716 } 717 CONF.set("hbase.region.archive.recovered.edits", "true"); 718 CONF.set(CommonFSUtils.HBASE_WAL_DIR, "/custom_wal_dir"); 719 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status); 720 assertEquals(maxSeqId, seqId); 721 region.getMVCC().advanceTo(seqId); 722 String fakeFamilyName = recoveredEditsDir.getName(); 723 Path rootDir = new Path(CONF.get(HConstants.HBASE_DIR)); 724 Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePathForRootDir(rootDir, 725 region.getRegionInfo(), Bytes.toBytes(fakeFamilyName)); 726 FileStatus[] list = TEST_UTIL.getTestFileSystem().listStatus(storeArchiveDir); 727 assertEquals(6, list.length); 728 } finally { 729 CONF.set("hbase.region.archive.recovered.edits", "false"); 730 CONF.set(CommonFSUtils.HBASE_WAL_DIR, ""); 731 HBaseTestingUtil.closeRegionAndWAL(this.region); 732 this.region = null; 733 wals.close(); 734 } 735 } 736 737 @Test 738 public void testSkipRecoveredEditsReplay() throws Exception { 739 byte[] family = Bytes.toBytes("family"); 740 this.region = initHRegion(tableName, method, CONF, family); 741 final WALFactory wals = new WALFactory(CONF, method); 742 try { 743 Path regiondir = region.getRegionFileSystem().getRegionDir(); 744 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 745 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 746 747 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 748 749 long maxSeqId = 1050; 750 long minSeqId = 1000; 751 752 for (long i = minSeqId; i <= maxSeqId; i += 10) { 753 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); 754 fs.create(recoveredEdits); 755 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 756 757 long time = System.nanoTime(); 758 WALEdit edit = new WALEdit(); 759 edit.add( 760 new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i))); 761 writer.append(new WAL.Entry( 762 new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); 763 764 writer.close(); 765 } 766 MonitoredTask status = TaskMonitor.get().createStatus(method); 767 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 768 for (HStore store : region.getStores()) { 769 maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId - 1); 770 } 771 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status); 772 assertEquals(maxSeqId, seqId); 773 region.getMVCC().advanceTo(seqId); 774 Get get = new Get(row); 775 Result result = region.get(get); 776 for (long i = minSeqId; i <= maxSeqId; i += 10) { 777 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i)); 778 assertEquals(1, kvs.size()); 779 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0))); 780 } 781 } finally { 782 HBaseTestingUtil.closeRegionAndWAL(this.region); 783 this.region = null; 784 wals.close(); 785 } 786 } 787 788 @Test 789 public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception { 790 byte[] family = Bytes.toBytes("family"); 791 this.region = initHRegion(tableName, method, CONF, family); 792 final WALFactory wals = new WALFactory(CONF, method); 793 try { 794 Path regiondir = region.getRegionFileSystem().getRegionDir(); 795 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 796 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 797 798 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 799 800 long maxSeqId = 1050; 801 long minSeqId = 1000; 802 803 for (long i = minSeqId; i <= maxSeqId; i += 10) { 804 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); 805 fs.create(recoveredEdits); 806 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 807 808 long time = System.nanoTime(); 809 WALEdit edit = new WALEdit(); 810 edit.add( 811 new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i))); 812 writer.append(new WAL.Entry( 813 new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); 814 815 writer.close(); 816 } 817 long recoverSeqId = 1030; 818 MonitoredTask status = TaskMonitor.get().createStatus(method); 819 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 820 for (HStore store : region.getStores()) { 821 maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), recoverSeqId - 1); 822 } 823 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status); 824 assertEquals(maxSeqId, seqId); 825 region.getMVCC().advanceTo(seqId); 826 Get get = new Get(row); 827 Result result = region.get(get); 828 for (long i = minSeqId; i <= maxSeqId; i += 10) { 829 List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i)); 830 if (i < recoverSeqId) { 831 assertEquals(0, kvs.size()); 832 } else { 833 assertEquals(1, kvs.size()); 834 assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0))); 835 } 836 } 837 } finally { 838 HBaseTestingUtil.closeRegionAndWAL(this.region); 839 this.region = null; 840 wals.close(); 841 } 842 } 843 844 @Test 845 public void testSkipRecoveredEditsReplayAllIgnored() throws Exception { 846 byte[] family = Bytes.toBytes("family"); 847 this.region = initHRegion(tableName, method, CONF, family); 848 Path regiondir = region.getRegionFileSystem().getRegionDir(); 849 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 850 851 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 852 for (int i = 1000; i < 1050; i += 10) { 853 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); 854 FSDataOutputStream dos = fs.create(recoveredEdits); 855 dos.writeInt(i); 856 dos.close(); 857 } 858 long minSeqId = 2000; 859 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1)); 860 FSDataOutputStream dos = fs.create(recoveredEdits); 861 dos.close(); 862 863 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 864 for (HStore store : region.getStores()) { 865 maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId); 866 } 867 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, null); 868 assertEquals(minSeqId, seqId); 869 } 870 871 @Test 872 public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception { 873 byte[] family = Bytes.toBytes("family"); 874 this.region = initHRegion(tableName, method, CONF, family); 875 final WALFactory wals = new WALFactory(CONF, method); 876 try { 877 Path regiondir = region.getRegionFileSystem().getRegionDir(); 878 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 879 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 880 byte[][] columns = region.getTableDescriptor().getColumnFamilyNames().toArray(new byte[0][]); 881 882 assertEquals(0, region.getStoreFileList(columns).size()); 883 884 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 885 886 long maxSeqId = 1050; 887 long minSeqId = 1000; 888 889 for (long i = minSeqId; i <= maxSeqId; i += 10) { 890 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); 891 fs.create(recoveredEdits); 892 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 893 894 long time = System.nanoTime(); 895 WALEdit edit = null; 896 if (i == maxSeqId) { 897 edit = WALEdit.createCompaction(region.getRegionInfo(), 898 CompactionDescriptor.newBuilder().setTableName(ByteString.copyFrom(tableName.getName())) 899 .setFamilyName(ByteString.copyFrom(regionName)) 900 .setEncodedRegionName(ByteString.copyFrom(regionName)) 901 .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString()))) 902 .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName())).build()); 903 } else { 904 edit = new WALEdit(); 905 edit.add( 906 new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i))); 907 } 908 writer.append(new WAL.Entry( 909 new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit)); 910 writer.close(); 911 } 912 913 long recoverSeqId = 1030; 914 Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); 915 MonitoredTask status = TaskMonitor.get().createStatus(method); 916 for (HStore store : region.getStores()) { 917 maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), recoverSeqId - 1); 918 } 919 long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status); 920 assertEquals(maxSeqId, seqId); 921 922 // assert that the files are flushed 923 assertEquals(1, region.getStoreFileList(columns).size()); 924 925 } finally { 926 HBaseTestingUtil.closeRegionAndWAL(this.region); 927 this.region = null; 928 wals.close(); 929 } 930 } 931 932 @Test 933 public void testRecoveredEditsReplayCompaction() throws Exception { 934 testRecoveredEditsReplayCompaction(false); 935 testRecoveredEditsReplayCompaction(true); 936 } 937 938 public void testRecoveredEditsReplayCompaction(boolean mismatchedRegionName) throws Exception { 939 CONF.setClass(HConstants.REGION_IMPL, HRegionForTesting.class, Region.class); 940 byte[] family = Bytes.toBytes("family"); 941 this.region = initHRegion(tableName, method, CONF, family); 942 final WALFactory wals = new WALFactory(CONF, method); 943 try { 944 Path regiondir = region.getRegionFileSystem().getRegionDir(); 945 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 946 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 947 948 long maxSeqId = 3; 949 long minSeqId = 0; 950 951 for (long i = minSeqId; i < maxSeqId; i++) { 952 Put put = new Put(Bytes.toBytes(i)); 953 put.addColumn(family, Bytes.toBytes(i), Bytes.toBytes(i)); 954 region.put(put); 955 region.flush(true); 956 } 957 958 // this will create a region with 3 files 959 assertEquals(3, region.getStore(family).getStorefilesCount()); 960 List<Path> storeFiles = new ArrayList<>(3); 961 for (HStoreFile sf : region.getStore(family).getStorefiles()) { 962 storeFiles.add(sf.getPath()); 963 } 964 965 // disable compaction completion 966 CONF.setBoolean("hbase.hstore.compaction.complete", false); 967 region.compactStores(); 968 969 // ensure that nothing changed 970 assertEquals(3, region.getStore(family).getStorefilesCount()); 971 972 // now find the compacted file, and manually add it to the recovered edits 973 Path tmpDir = new Path(region.getRegionFileSystem().getTempDir(), Bytes.toString(family)); 974 FileStatus[] files = CommonFSUtils.listStatus(fs, tmpDir); 975 String errorMsg = "Expected to find 1 file in the region temp directory " 976 + "from the compaction, could not find any"; 977 assertNotNull(errorMsg, files); 978 assertEquals(errorMsg, 1, files.length); 979 // move the file inside region dir 980 Path newFile = 981 region.getRegionFileSystem().commitStoreFile(Bytes.toString(family), files[0].getPath()); 982 983 byte[] encodedNameAsBytes = this.region.getRegionInfo().getEncodedNameAsBytes(); 984 byte[] fakeEncodedNameAsBytes = new byte[encodedNameAsBytes.length]; 985 for (int i = 0; i < encodedNameAsBytes.length; i++) { 986 // Mix the byte array to have a new encodedName 987 fakeEncodedNameAsBytes[i] = (byte) (encodedNameAsBytes[i] + 1); 988 } 989 990 CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor( 991 this.region.getRegionInfo(), mismatchedRegionName ? fakeEncodedNameAsBytes : null, family, 992 storeFiles, Lists.newArrayList(newFile), 993 region.getRegionFileSystem().getStoreDir(Bytes.toString(family))); 994 995 WALUtil.writeCompactionMarker(region.getWAL(), this.region.getReplicationScope(), 996 this.region.getRegionInfo(), compactionDescriptor, region.getMVCC(), null); 997 998 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 999 1000 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000)); 1001 fs.create(recoveredEdits); 1002 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 1003 1004 long time = System.nanoTime(); 1005 1006 writer.append(new WAL.Entry( 1007 new WALKeyImpl(regionName, tableName, 10, time, HConstants.DEFAULT_CLUSTER_ID), 1008 WALEdit.createCompaction(region.getRegionInfo(), compactionDescriptor))); 1009 writer.close(); 1010 1011 // close the region now, and reopen again 1012 region.getTableDescriptor(); 1013 region.getRegionInfo(); 1014 HBaseTestingUtil.closeRegionAndWAL(this.region); 1015 try { 1016 region = HRegion.openHRegion(region, null); 1017 } catch (WrongRegionException wre) { 1018 fail("Matching encoded region name should not have produced WrongRegionException"); 1019 } 1020 1021 // now check whether we have only one store file, the compacted one 1022 Collection<HStoreFile> sfs = region.getStore(family).getStorefiles(); 1023 for (HStoreFile sf : sfs) { 1024 LOG.info(Objects.toString(sf.getPath())); 1025 } 1026 if (!mismatchedRegionName) { 1027 assertEquals(1, region.getStore(family).getStorefilesCount()); 1028 } 1029 files = CommonFSUtils.listStatus(fs, tmpDir); 1030 assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0); 1031 1032 for (long i = minSeqId; i < maxSeqId; i++) { 1033 Get get = new Get(Bytes.toBytes(i)); 1034 Result result = region.get(get); 1035 byte[] value = result.getValue(family, Bytes.toBytes(i)); 1036 assertArrayEquals(Bytes.toBytes(i), value); 1037 } 1038 } finally { 1039 HBaseTestingUtil.closeRegionAndWAL(this.region); 1040 this.region = null; 1041 wals.close(); 1042 CONF.setClass(HConstants.REGION_IMPL, HRegion.class, Region.class); 1043 } 1044 } 1045 1046 @Test 1047 public void testFlushMarkers() throws Exception { 1048 // tests that flush markers are written to WAL and handled at recovered edits 1049 byte[] family = Bytes.toBytes("family"); 1050 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + ".log"); 1051 final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration()); 1052 CommonFSUtils.setRootDir(walConf, logDir); 1053 final WALFactory wals = new WALFactory(walConf, method); 1054 final WAL wal = wals.getWAL(RegionInfoBuilder.newBuilder(tableName).build()); 1055 1056 this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF, 1057 false, Durability.USE_DEFAULT, wal, family); 1058 try { 1059 Path regiondir = region.getRegionFileSystem().getRegionDir(); 1060 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 1061 byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); 1062 1063 long maxSeqId = 3; 1064 long minSeqId = 0; 1065 1066 for (long i = minSeqId; i < maxSeqId; i++) { 1067 Put put = new Put(Bytes.toBytes(i)); 1068 put.addColumn(family, Bytes.toBytes(i), Bytes.toBytes(i)); 1069 region.put(put); 1070 region.flush(true); 1071 } 1072 1073 // this will create a region with 3 files from flush 1074 assertEquals(3, region.getStore(family).getStorefilesCount()); 1075 List<String> storeFiles = new ArrayList<>(3); 1076 for (HStoreFile sf : region.getStore(family).getStorefiles()) { 1077 storeFiles.add(sf.getPath().getName()); 1078 } 1079 1080 // now verify that the flush markers are written 1081 wal.shutdown(); 1082 WALStreamReader reader = WALFactory.createStreamReader(fs, 1083 AbstractFSWALProvider.getCurrentFileName(wal), TEST_UTIL.getConfiguration()); 1084 try { 1085 List<WAL.Entry> flushDescriptors = new ArrayList<>(); 1086 long lastFlushSeqId = -1; 1087 while (true) { 1088 WAL.Entry entry = reader.next(); 1089 if (entry == null) { 1090 break; 1091 } 1092 Cell cell = entry.getEdit().getCells().get(0); 1093 if (WALEdit.isMetaEditFamily(cell)) { 1094 FlushDescriptor flushDesc = WALEdit.getFlushDescriptor(cell); 1095 assertNotNull(flushDesc); 1096 assertArrayEquals(tableName.getName(), flushDesc.getTableName().toByteArray()); 1097 if (flushDesc.getAction() == FlushAction.START_FLUSH) { 1098 assertTrue(flushDesc.getFlushSequenceNumber() > lastFlushSeqId); 1099 } else if (flushDesc.getAction() == FlushAction.COMMIT_FLUSH) { 1100 assertTrue(flushDesc.getFlushSequenceNumber() == lastFlushSeqId); 1101 } 1102 lastFlushSeqId = flushDesc.getFlushSequenceNumber(); 1103 assertArrayEquals(regionName, flushDesc.getEncodedRegionName().toByteArray()); 1104 assertEquals(1, flushDesc.getStoreFlushesCount()); // only one store 1105 StoreFlushDescriptor storeFlushDesc = flushDesc.getStoreFlushes(0); 1106 assertArrayEquals(family, storeFlushDesc.getFamilyName().toByteArray()); 1107 assertEquals("family", storeFlushDesc.getStoreHomeDir()); 1108 if (flushDesc.getAction() == FlushAction.START_FLUSH) { 1109 assertEquals(0, storeFlushDesc.getFlushOutputCount()); 1110 } else { 1111 assertEquals(1, storeFlushDesc.getFlushOutputCount()); // only one file from flush 1112 assertTrue(storeFiles.contains(storeFlushDesc.getFlushOutput(0))); 1113 } 1114 1115 flushDescriptors.add(entry); 1116 } 1117 } 1118 1119 assertEquals(3 * 2, flushDescriptors.size()); // START_FLUSH and COMMIT_FLUSH per flush 1120 1121 // now write those markers to the recovered edits again. 1122 1123 Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir); 1124 1125 Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000)); 1126 fs.create(recoveredEdits); 1127 WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits); 1128 1129 for (WAL.Entry entry : flushDescriptors) { 1130 writer.append(entry); 1131 } 1132 writer.close(); 1133 } finally { 1134 reader.close(); 1135 } 1136 1137 // close the region now, and reopen again 1138 HBaseTestingUtil.closeRegionAndWAL(this.region); 1139 region = HRegion.openHRegion(region, null); 1140 1141 // now check whether we have can read back the data from region 1142 for (long i = minSeqId; i < maxSeqId; i++) { 1143 Get get = new Get(Bytes.toBytes(i)); 1144 Result result = region.get(get); 1145 byte[] value = result.getValue(family, Bytes.toBytes(i)); 1146 assertArrayEquals(Bytes.toBytes(i), value); 1147 } 1148 } finally { 1149 HBaseTestingUtil.closeRegionAndWAL(this.region); 1150 this.region = null; 1151 wals.close(); 1152 } 1153 } 1154 1155 static class IsFlushWALMarker implements ArgumentMatcher<WALEdit> { 1156 volatile FlushAction[] actions; 1157 1158 public IsFlushWALMarker(FlushAction... actions) { 1159 this.actions = actions; 1160 } 1161 1162 @Override 1163 public boolean matches(WALEdit edit) { 1164 List<Cell> cells = edit.getCells(); 1165 if (cells.isEmpty()) { 1166 return false; 1167 } 1168 if (WALEdit.isMetaEditFamily(cells.get(0))) { 1169 FlushDescriptor desc; 1170 try { 1171 desc = WALEdit.getFlushDescriptor(cells.get(0)); 1172 } catch (IOException e) { 1173 LOG.warn(e.toString(), e); 1174 return false; 1175 } 1176 if (desc != null) { 1177 for (FlushAction action : actions) { 1178 if (desc.getAction() == action) { 1179 return true; 1180 } 1181 } 1182 } 1183 } 1184 return false; 1185 } 1186 1187 public IsFlushWALMarker set(FlushAction... actions) { 1188 this.actions = actions; 1189 return this; 1190 } 1191 } 1192 1193 @Test 1194 public void testFlushMarkersWALFail() throws Exception { 1195 // test the cases where the WAL append for flush markers fail. 1196 byte[] family = Bytes.toBytes("family"); 1197 1198 // spy an actual WAL implementation to throw exception (was not able to mock) 1199 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + "log"); 1200 1201 final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration()); 1202 CommonFSUtils.setRootDir(walConf, logDir); 1203 // Make up a WAL that we can manipulate at append time. 1204 class FailAppendFlushMarkerWAL extends FSHLog { 1205 volatile FlushAction[] flushActions = null; 1206 1207 public FailAppendFlushMarkerWAL(FileSystem fs, Path root, String logDir, Configuration conf) 1208 throws IOException { 1209 super(fs, root, logDir, conf); 1210 } 1211 1212 @Override 1213 protected Writer createWriterInstance(Path path) throws IOException { 1214 final Writer w = super.createWriterInstance(path); 1215 return new Writer() { 1216 @Override 1217 public void close() throws IOException { 1218 w.close(); 1219 } 1220 1221 @Override 1222 public void sync(boolean forceSync) throws IOException { 1223 w.sync(forceSync); 1224 } 1225 1226 @Override 1227 public void append(Entry entry) throws IOException { 1228 List<Cell> cells = entry.getEdit().getCells(); 1229 if (WALEdit.isMetaEditFamily(cells.get(0))) { 1230 FlushDescriptor desc = WALEdit.getFlushDescriptor(cells.get(0)); 1231 if (desc != null) { 1232 for (FlushAction flushAction : flushActions) { 1233 if (desc.getAction().equals(flushAction)) { 1234 throw new IOException("Failed to append flush marker! " + flushAction); 1235 } 1236 } 1237 } 1238 } 1239 w.append(entry); 1240 } 1241 1242 @Override 1243 public long getLength() { 1244 return w.getLength(); 1245 } 1246 1247 @Override 1248 public long getSyncedLength() { 1249 return w.getSyncedLength(); 1250 } 1251 }; 1252 } 1253 } 1254 FailAppendFlushMarkerWAL wal = new FailAppendFlushMarkerWAL(FileSystem.get(walConf), 1255 CommonFSUtils.getRootDir(walConf), method, walConf); 1256 wal.init(); 1257 this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF, 1258 false, Durability.USE_DEFAULT, wal, family); 1259 int i = 0; 1260 Put put = new Put(Bytes.toBytes(i)); 1261 put.setDurability(Durability.SKIP_WAL); // have to skip mocked wal 1262 put.addColumn(family, Bytes.toBytes(i), Bytes.toBytes(i)); 1263 region.put(put); 1264 1265 // 1. Test case where START_FLUSH throws exception 1266 wal.flushActions = new FlushAction[] { FlushAction.START_FLUSH }; 1267 1268 // start cache flush will throw exception 1269 try { 1270 region.flush(true); 1271 fail("This should have thrown exception"); 1272 } catch (DroppedSnapshotException unexpected) { 1273 // this should not be a dropped snapshot exception. Meaning that RS will not abort 1274 throw unexpected; 1275 } catch (IOException expected) { 1276 // expected 1277 } 1278 // The WAL is hosed now. It has two edits appended. We cannot roll the log without it 1279 // throwing a DroppedSnapshotException to force an abort. Just clean up the mess. 1280 region.close(true); 1281 wal.close(); 1282 1283 // 2. Test case where START_FLUSH succeeds but COMMIT_FLUSH will throw exception 1284 wal.flushActions = new FlushAction[] { FlushAction.COMMIT_FLUSH }; 1285 wal = new FailAppendFlushMarkerWAL(FileSystem.get(walConf), CommonFSUtils.getRootDir(walConf), 1286 method, walConf); 1287 wal.init(); 1288 this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF, 1289 false, Durability.USE_DEFAULT, wal, family); 1290 region.put(put); 1291 // 3. Test case where ABORT_FLUSH will throw exception. 1292 // Even if ABORT_FLUSH throws exception, we should not fail with IOE, but continue with 1293 // DroppedSnapshotException. Below COMMIT_FLUSH will cause flush to abort 1294 wal.flushActions = new FlushAction[] { FlushAction.COMMIT_FLUSH, FlushAction.ABORT_FLUSH }; 1295 1296 try { 1297 region.flush(true); 1298 fail("This should have thrown exception"); 1299 } catch (DroppedSnapshotException expected) { 1300 // we expect this exception, since we were able to write the snapshot, but failed to 1301 // write the flush marker to WAL 1302 } catch (IOException unexpected) { 1303 throw unexpected; 1304 } 1305 } 1306 1307 @Test 1308 public void testGetWhileRegionClose() throws IOException { 1309 Configuration hc = initSplit(); 1310 int numRows = 100; 1311 byte[][] families = { fam1, fam2, fam3 }; 1312 1313 // Setting up region 1314 this.region = initHRegion(tableName, method, hc, families); 1315 // Put data in region 1316 final int startRow = 100; 1317 putData(startRow, numRows, qual1, families); 1318 putData(startRow, numRows, qual2, families); 1319 putData(startRow, numRows, qual3, families); 1320 final AtomicBoolean done = new AtomicBoolean(false); 1321 final AtomicInteger gets = new AtomicInteger(0); 1322 GetTillDoneOrException[] threads = new GetTillDoneOrException[10]; 1323 try { 1324 // Set ten threads running concurrently getting from the region. 1325 for (int i = 0; i < threads.length / 2; i++) { 1326 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets); 1327 threads[i].setDaemon(true); 1328 threads[i].start(); 1329 } 1330 // Artificially make the condition by setting closing flag explicitly. 1331 // I can't make the issue happen with a call to region.close(). 1332 this.region.closing.set(true); 1333 for (int i = threads.length / 2; i < threads.length; i++) { 1334 threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets); 1335 threads[i].setDaemon(true); 1336 threads[i].start(); 1337 } 1338 } finally { 1339 if (this.region != null) { 1340 HBaseTestingUtil.closeRegionAndWAL(this.region); 1341 this.region = null; 1342 } 1343 } 1344 done.set(true); 1345 for (GetTillDoneOrException t : threads) { 1346 try { 1347 t.join(); 1348 } catch (InterruptedException e) { 1349 e.printStackTrace(); 1350 } 1351 if (t.e != null) { 1352 LOG.info("Exception=" + t.e); 1353 assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException); 1354 } 1355 } 1356 } 1357 1358 /* 1359 * Thread that does get on single row until 'done' flag is flipped. If an exception causes us to 1360 * fail, it records it. 1361 */ 1362 class GetTillDoneOrException extends Thread { 1363 private final Get g; 1364 private final AtomicBoolean done; 1365 private final AtomicInteger count; 1366 private Exception e; 1367 1368 GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d, 1369 final AtomicInteger c) { 1370 super("getter." + i); 1371 this.g = new Get(r); 1372 this.done = d; 1373 this.count = c; 1374 } 1375 1376 @Override 1377 public void run() { 1378 while (!this.done.get()) { 1379 try { 1380 assertTrue(region.get(g).size() > 0); 1381 this.count.incrementAndGet(); 1382 } catch (Exception e) { 1383 this.e = e; 1384 break; 1385 } 1386 } 1387 } 1388 } 1389 1390 /* 1391 * An involved filter test. Has multiple column families and deletes in mix. 1392 */ 1393 @Test 1394 public void testWeirdCacheBehaviour() throws Exception { 1395 final TableName tableName = TableName.valueOf(name.getMethodName()); 1396 byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"), 1397 Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") }; 1398 this.region = initHRegion(tableName, method, CONF, FAMILIES); 1399 String value = "this is the value"; 1400 String value2 = "this is some other value"; 1401 String keyPrefix1 = "prefix1"; 1402 String keyPrefix2 = "prefix2"; 1403 String keyPrefix3 = "prefix3"; 1404 putRows(this.region, 3, value, keyPrefix1); 1405 putRows(this.region, 3, value, keyPrefix2); 1406 putRows(this.region, 3, value, keyPrefix3); 1407 putRows(this.region, 3, value2, keyPrefix1); 1408 putRows(this.region, 3, value2, keyPrefix2); 1409 putRows(this.region, 3, value2, keyPrefix3); 1410 System.out.println("Checking values for key: " + keyPrefix1); 1411 assertEquals("Got back incorrect number of rows from scan", 3, 1412 getNumberOfRows(keyPrefix1, value2, this.region)); 1413 System.out.println("Checking values for key: " + keyPrefix2); 1414 assertEquals("Got back incorrect number of rows from scan", 3, 1415 getNumberOfRows(keyPrefix2, value2, this.region)); 1416 System.out.println("Checking values for key: " + keyPrefix3); 1417 assertEquals("Got back incorrect number of rows from scan", 3, 1418 getNumberOfRows(keyPrefix3, value2, this.region)); 1419 deleteColumns(this.region, value2, keyPrefix1); 1420 deleteColumns(this.region, value2, keyPrefix2); 1421 deleteColumns(this.region, value2, keyPrefix3); 1422 System.out.println("Starting important checks....."); 1423 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0, 1424 getNumberOfRows(keyPrefix1, value2, this.region)); 1425 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0, 1426 getNumberOfRows(keyPrefix2, value2, this.region)); 1427 assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0, 1428 getNumberOfRows(keyPrefix3, value2, this.region)); 1429 } 1430 1431 @Test 1432 public void testAppendWithReadOnlyTable() throws Exception { 1433 final TableName tableName = TableName.valueOf(name.getMethodName()); 1434 this.region = initHRegion(tableName, method, CONF, true, Bytes.toBytes("somefamily")); 1435 boolean exceptionCaught = false; 1436 Append append = new Append(Bytes.toBytes("somerow")); 1437 append.setDurability(Durability.SKIP_WAL); 1438 append.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1439 Bytes.toBytes("somevalue")); 1440 try { 1441 region.append(append); 1442 } catch (IOException e) { 1443 exceptionCaught = true; 1444 } 1445 assertTrue(exceptionCaught == true); 1446 } 1447 1448 @Test 1449 public void testIncrWithReadOnlyTable() throws Exception { 1450 final TableName tableName = TableName.valueOf(name.getMethodName()); 1451 this.region = initHRegion(tableName, method, CONF, true, Bytes.toBytes("somefamily")); 1452 boolean exceptionCaught = false; 1453 Increment inc = new Increment(Bytes.toBytes("somerow")); 1454 inc.setDurability(Durability.SKIP_WAL); 1455 inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L); 1456 try { 1457 region.increment(inc); 1458 } catch (IOException e) { 1459 exceptionCaught = true; 1460 } 1461 assertTrue(exceptionCaught == true); 1462 } 1463 1464 private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException { 1465 InternalScanner scanner = buildScanner(keyPrefix, value, r); 1466 int count = 0; 1467 boolean more = false; 1468 List<Cell> results = new ArrayList<>(); 1469 do { 1470 more = scanner.next(results); 1471 if (results != null && !results.isEmpty()) count++; 1472 else break; 1473 Delete delete = new Delete(CellUtil.cloneRow(results.get(0))); 1474 delete.addColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2")); 1475 r.delete(delete); 1476 results.clear(); 1477 } while (more); 1478 assertEquals("Did not perform correct number of deletes", 3, count); 1479 } 1480 1481 private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception { 1482 InternalScanner resultScanner = buildScanner(keyPrefix, value, r); 1483 int numberOfResults = 0; 1484 List<Cell> results = new ArrayList<>(); 1485 boolean more = false; 1486 do { 1487 more = resultScanner.next(results); 1488 if (results != null && !results.isEmpty()) numberOfResults++; 1489 else break; 1490 for (Cell kv : results) { 1491 System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv))); 1492 } 1493 results.clear(); 1494 } while (more); 1495 return numberOfResults; 1496 } 1497 1498 private InternalScanner buildScanner(String keyPrefix, String value, HRegion r) 1499 throws IOException { 1500 // Defaults FilterList.Operator.MUST_PASS_ALL. 1501 FilterList allFilters = new FilterList(); 1502 allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix))); 1503 // Only return rows where this column value exists in the row. 1504 SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"), 1505 Bytes.toBytes("qual2"), CompareOperator.EQUAL, Bytes.toBytes(value)); 1506 filter.setFilterIfMissing(true); 1507 allFilters.addFilter(filter); 1508 Scan scan = new Scan(); 1509 scan.addFamily(Bytes.toBytes("trans-blob")); 1510 scan.addFamily(Bytes.toBytes("trans-type")); 1511 scan.addFamily(Bytes.toBytes("trans-date")); 1512 scan.addFamily(Bytes.toBytes("trans-tags")); 1513 scan.addFamily(Bytes.toBytes("trans-group")); 1514 scan.setFilter(allFilters); 1515 return r.getScanner(scan); 1516 } 1517 1518 private void putRows(HRegion r, int numRows, String value, String key) throws IOException { 1519 for (int i = 0; i < numRows; i++) { 1520 String row = key + "_" + i/* UUID.randomUUID().toString() */; 1521 System.out.println(String.format("Saving row: %s, with value %s", row, value)); 1522 Put put = new Put(Bytes.toBytes(row)); 1523 put.setDurability(Durability.SKIP_WAL); 1524 put.addColumn(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob")); 1525 put.addColumn(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement")); 1526 put.addColumn(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999")); 1527 put.addColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value)); 1528 put.addColumn(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId")); 1529 r.put(put); 1530 } 1531 } 1532 1533 @Test 1534 public void testFamilyWithAndWithoutColon() throws Exception { 1535 byte[] cf = Bytes.toBytes(COLUMN_FAMILY); 1536 this.region = initHRegion(tableName, method, CONF, cf); 1537 Put p = new Put(tableName.toBytes()); 1538 byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":"); 1539 p.addColumn(cfwithcolon, cfwithcolon, cfwithcolon); 1540 boolean exception = false; 1541 try { 1542 this.region.put(p); 1543 } catch (NoSuchColumnFamilyException e) { 1544 exception = true; 1545 } 1546 assertTrue(exception); 1547 } 1548 1549 @Test 1550 public void testBatchPut_whileNoRowLocksHeld() throws IOException { 1551 final Put[] puts = new Put[10]; 1552 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); 1553 long syncs = prepareRegionForBachPut(puts, source, false); 1554 1555 OperationStatus[] codes = this.region.batchMutate(puts); 1556 assertEquals(10, codes.length); 1557 for (int i = 0; i < 10; i++) { 1558 assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode()); 1559 } 1560 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source); 1561 1562 LOG.info("Next a batch put with one invalid family"); 1563 puts[5].addColumn(Bytes.toBytes("BAD_CF"), qual, value); 1564 codes = this.region.batchMutate(puts); 1565 assertEquals(10, codes.length); 1566 for (int i = 0; i < 10; i++) { 1567 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS, 1568 codes[i].getOperationStatusCode()); 1569 } 1570 1571 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source); 1572 } 1573 1574 @Test 1575 public void testBatchPut_whileMultipleRowLocksHeld() throws Exception { 1576 final Put[] puts = new Put[10]; 1577 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); 1578 long syncs = prepareRegionForBachPut(puts, source, false); 1579 1580 puts[5].addColumn(Bytes.toBytes("BAD_CF"), qual, value); 1581 1582 LOG.info("batchPut will have to break into four batches to avoid row locks"); 1583 RowLock rowLock1 = region.getRowLock(Bytes.toBytes("row_2")); 1584 RowLock rowLock2 = region.getRowLock(Bytes.toBytes("row_1")); 1585 RowLock rowLock3 = region.getRowLock(Bytes.toBytes("row_3")); 1586 RowLock rowLock4 = region.getRowLock(Bytes.toBytes("row_3"), true); 1587 1588 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF); 1589 final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<>(); 1590 final CountDownLatch startingPuts = new CountDownLatch(1); 1591 final CountDownLatch startingClose = new CountDownLatch(1); 1592 TestThread putter = new TestThread(ctx) { 1593 @Override 1594 public void doWork() throws IOException { 1595 startingPuts.countDown(); 1596 retFromThread.set(region.batchMutate(puts)); 1597 } 1598 }; 1599 LOG.info("...starting put thread while holding locks"); 1600 ctx.addThread(putter); 1601 ctx.startThreads(); 1602 1603 // Now attempt to close the region from another thread. Prior to HBASE-12565 1604 // this would cause the in-progress batchMutate operation to to fail with 1605 // exception because it use to release and re-acquire the close-guard lock 1606 // between batches. Caller then didn't get status indicating which writes succeeded. 1607 // We now expect this thread to block until the batchMutate call finishes. 1608 Thread regionCloseThread = new TestThread(ctx) { 1609 @Override 1610 public void doWork() { 1611 try { 1612 startingPuts.await(); 1613 // Give some time for the batch mutate to get in. 1614 // We don't want to race with the mutate 1615 Thread.sleep(10); 1616 startingClose.countDown(); 1617 HBaseTestingUtil.closeRegionAndWAL(region); 1618 region = null; 1619 } catch (IOException e) { 1620 throw new RuntimeException(e); 1621 } catch (InterruptedException e) { 1622 throw new RuntimeException(e); 1623 } 1624 } 1625 }; 1626 regionCloseThread.start(); 1627 1628 startingClose.await(); 1629 startingPuts.await(); 1630 Thread.sleep(100); 1631 LOG.info("...releasing row lock 1, which should let put thread continue"); 1632 rowLock1.release(); 1633 rowLock2.release(); 1634 rowLock3.release(); 1635 waitForCounter(source, "syncTimeNumOps", syncs + 1); 1636 1637 LOG.info("...joining on put thread"); 1638 ctx.stop(); 1639 regionCloseThread.join(); 1640 1641 OperationStatus[] codes = retFromThread.get(); 1642 for (int i = 0; i < codes.length; i++) { 1643 assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS, 1644 codes[i].getOperationStatusCode()); 1645 } 1646 rowLock4.release(); 1647 } 1648 1649 private void waitForCounter(MetricsWALSource source, String metricName, long expectedCount) 1650 throws InterruptedException { 1651 long startWait = EnvironmentEdgeManager.currentTime(); 1652 long currentCount; 1653 while ((currentCount = metricsAssertHelper.getCounter(metricName, source)) < expectedCount) { 1654 Thread.sleep(100); 1655 if (EnvironmentEdgeManager.currentTime() - startWait > 10000) { 1656 fail(String.format("Timed out waiting for '%s' >= '%s', currentCount=%s", metricName, 1657 expectedCount, currentCount)); 1658 } 1659 } 1660 } 1661 1662 @Test 1663 public void testAtomicBatchPut() throws IOException { 1664 final Put[] puts = new Put[10]; 1665 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); 1666 long syncs = prepareRegionForBachPut(puts, source, false); 1667 1668 // 1. Straight forward case, should succeed 1669 OperationStatus[] codes = this.region.batchMutate(puts, true); 1670 assertEquals(10, codes.length); 1671 for (int i = 0; i < 10; i++) { 1672 assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode()); 1673 } 1674 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source); 1675 1676 // 2. Failed to get lock 1677 RowLock lock = region.getRowLock(Bytes.toBytes("row_" + 3)); 1678 // Method {@link HRegion#getRowLock(byte[])} is reentrant. As 'row_3' is locked in this 1679 // thread, need to run {@link HRegion#batchMutate(HRegion.BatchOperation)} in different thread 1680 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF); 1681 final AtomicReference<IOException> retFromThread = new AtomicReference<>(); 1682 final CountDownLatch finishedPuts = new CountDownLatch(1); 1683 TestThread putter = new TestThread(ctx) { 1684 @Override 1685 public void doWork() throws IOException { 1686 try { 1687 region.batchMutate(puts, true); 1688 } catch (IOException ioe) { 1689 LOG.error("test failed!", ioe); 1690 retFromThread.set(ioe); 1691 } 1692 finishedPuts.countDown(); 1693 } 1694 }; 1695 LOG.info("...starting put thread while holding locks"); 1696 ctx.addThread(putter); 1697 ctx.startThreads(); 1698 LOG.info("...waiting for batch puts while holding locks"); 1699 try { 1700 finishedPuts.await(); 1701 } catch (InterruptedException e) { 1702 LOG.error("Interrupted!", e); 1703 } finally { 1704 if (lock != null) { 1705 lock.release(); 1706 } 1707 } 1708 assertNotNull(retFromThread.get()); 1709 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source); 1710 1711 // 3. Exception thrown in validation 1712 LOG.info("Next a batch put with one invalid family"); 1713 puts[5].addColumn(Bytes.toBytes("BAD_CF"), qual, value); 1714 thrown.expect(NoSuchColumnFamilyException.class); 1715 this.region.batchMutate(puts, true); 1716 } 1717 1718 @Test 1719 public void testBatchPutWithTsSlop() throws Exception { 1720 // add data with a timestamp that is too recent for range. Ensure assert 1721 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000); 1722 final Put[] puts = new Put[10]; 1723 MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); 1724 1725 long syncs = prepareRegionForBachPut(puts, source, true); 1726 1727 OperationStatus[] codes = this.region.batchMutate(puts); 1728 assertEquals(10, codes.length); 1729 for (int i = 0; i < 10; i++) { 1730 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode()); 1731 } 1732 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source); 1733 } 1734 1735 /** Returns syncs initial syncTimeNumOps */ 1736 private long prepareRegionForBachPut(final Put[] puts, final MetricsWALSource source, 1737 boolean slop) throws IOException { 1738 this.region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES); 1739 1740 LOG.info("First a batch put with all valid puts"); 1741 for (int i = 0; i < puts.length; i++) { 1742 puts[i] = slop 1743 ? new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100) 1744 : new Put(Bytes.toBytes("row_" + i)); 1745 puts[i].addColumn(COLUMN_FAMILY_BYTES, qual, value); 1746 } 1747 1748 long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source); 1749 metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source); 1750 return syncs; 1751 } 1752 1753 // //////////////////////////////////////////////////////////////////////////// 1754 // checkAndMutate tests 1755 // //////////////////////////////////////////////////////////////////////////// 1756 @Test 1757 @Deprecated 1758 public void testCheckAndMutate_WithEmptyRowValue() throws IOException { 1759 byte[] row1 = Bytes.toBytes("row1"); 1760 byte[] fam1 = Bytes.toBytes("fam1"); 1761 byte[] qf1 = Bytes.toBytes("qualifier"); 1762 byte[] emptyVal = new byte[] {}; 1763 byte[] val1 = Bytes.toBytes("value1"); 1764 byte[] val2 = Bytes.toBytes("value2"); 1765 1766 // Setting up region 1767 this.region = initHRegion(tableName, method, CONF, fam1); 1768 // Putting empty data in key 1769 Put put = new Put(row1); 1770 put.addColumn(fam1, qf1, emptyVal); 1771 1772 // checkAndPut with empty value 1773 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1774 new BinaryComparator(emptyVal), put); 1775 assertTrue(res); 1776 1777 // Putting data in key 1778 put = new Put(row1); 1779 put.addColumn(fam1, qf1, val1); 1780 1781 // checkAndPut with correct value 1782 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1783 new BinaryComparator(emptyVal), put); 1784 assertTrue(res); 1785 1786 // not empty anymore 1787 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1788 new BinaryComparator(emptyVal), put); 1789 assertFalse(res); 1790 1791 Delete delete = new Delete(row1); 1792 delete.addColumn(fam1, qf1); 1793 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1794 new BinaryComparator(emptyVal), delete); 1795 assertFalse(res); 1796 1797 put = new Put(row1); 1798 put.addColumn(fam1, qf1, val2); 1799 // checkAndPut with correct value 1800 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val1), 1801 put); 1802 assertTrue(res); 1803 1804 // checkAndDelete with correct value 1805 delete = new Delete(row1); 1806 delete.addColumn(fam1, qf1); 1807 delete.addColumn(fam1, qf1); 1808 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val2), 1809 delete); 1810 assertTrue(res); 1811 1812 delete = new Delete(row1); 1813 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1814 new BinaryComparator(emptyVal), delete); 1815 assertTrue(res); 1816 1817 // checkAndPut looking for a null value 1818 put = new Put(row1); 1819 put.addColumn(fam1, qf1, val1); 1820 1821 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new NullComparator(), put); 1822 assertTrue(res); 1823 } 1824 1825 @Test 1826 @Deprecated 1827 public void testCheckAndMutate_WithWrongValue() throws IOException { 1828 byte[] row1 = Bytes.toBytes("row1"); 1829 byte[] fam1 = Bytes.toBytes("fam1"); 1830 byte[] qf1 = Bytes.toBytes("qualifier"); 1831 byte[] val1 = Bytes.toBytes("value1"); 1832 byte[] val2 = Bytes.toBytes("value2"); 1833 BigDecimal bd1 = new BigDecimal(Double.MAX_VALUE); 1834 BigDecimal bd2 = new BigDecimal(Double.MIN_VALUE); 1835 1836 // Setting up region 1837 this.region = initHRegion(tableName, method, CONF, fam1); 1838 // Putting data in key 1839 Put put = new Put(row1); 1840 put.addColumn(fam1, qf1, val1); 1841 region.put(put); 1842 1843 // checkAndPut with wrong value 1844 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1845 new BinaryComparator(val2), put); 1846 assertEquals(false, res); 1847 1848 // checkAndDelete with wrong value 1849 Delete delete = new Delete(row1); 1850 delete.addFamily(fam1); 1851 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val2), 1852 put); 1853 assertEquals(false, res); 1854 1855 // Putting data in key 1856 put = new Put(row1); 1857 put.addColumn(fam1, qf1, Bytes.toBytes(bd1)); 1858 region.put(put); 1859 1860 // checkAndPut with wrong value 1861 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1862 new BigDecimalComparator(bd2), put); 1863 assertEquals(false, res); 1864 1865 // checkAndDelete with wrong value 1866 delete = new Delete(row1); 1867 delete.addFamily(fam1); 1868 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1869 new BigDecimalComparator(bd2), put); 1870 assertEquals(false, res); 1871 } 1872 1873 @Test 1874 @Deprecated 1875 public void testCheckAndMutate_WithCorrectValue() throws IOException { 1876 byte[] row1 = Bytes.toBytes("row1"); 1877 byte[] fam1 = Bytes.toBytes("fam1"); 1878 byte[] qf1 = Bytes.toBytes("qualifier"); 1879 byte[] val1 = Bytes.toBytes("value1"); 1880 BigDecimal bd1 = new BigDecimal(Double.MIN_VALUE); 1881 1882 // Setting up region 1883 this.region = initHRegion(tableName, method, CONF, fam1); 1884 // Putting data in key 1885 long now = EnvironmentEdgeManager.currentTime(); 1886 Put put = new Put(row1); 1887 put.addColumn(fam1, qf1, now, val1); 1888 region.put(put); 1889 1890 // checkAndPut with correct value 1891 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1892 new BinaryComparator(val1), put); 1893 assertEquals("First", true, res); 1894 1895 // checkAndDelete with correct value 1896 Delete delete = new Delete(row1, now + 1); 1897 delete.addColumn(fam1, qf1); 1898 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val1), 1899 delete); 1900 assertEquals("Delete", true, res); 1901 1902 // Putting data in key 1903 put = new Put(row1); 1904 put.addColumn(fam1, qf1, now + 2, Bytes.toBytes(bd1)); 1905 region.put(put); 1906 1907 // checkAndPut with correct value 1908 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1909 new BigDecimalComparator(bd1), put); 1910 assertEquals("Second put", true, res); 1911 1912 // checkAndDelete with correct value 1913 delete = new Delete(row1, now + 3); 1914 delete.addColumn(fam1, qf1); 1915 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 1916 new BigDecimalComparator(bd1), delete); 1917 assertEquals("Second delete", true, res); 1918 } 1919 1920 @Test 1921 @Deprecated 1922 public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException { 1923 byte[] row1 = Bytes.toBytes("row1"); 1924 byte[] fam1 = Bytes.toBytes("fam1"); 1925 byte[] qf1 = Bytes.toBytes("qualifier"); 1926 byte[] val1 = Bytes.toBytes("value1"); 1927 byte[] val2 = Bytes.toBytes("value2"); 1928 byte[] val3 = Bytes.toBytes("value3"); 1929 byte[] val4 = Bytes.toBytes("value4"); 1930 1931 // Setting up region 1932 this.region = initHRegion(tableName, method, CONF, fam1); 1933 // Putting val3 in key 1934 Put put = new Put(row1); 1935 put.addColumn(fam1, qf1, val3); 1936 region.put(put); 1937 1938 // Test CompareOp.LESS: original = val3, compare with val3, fail 1939 boolean res = 1940 region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS, new BinaryComparator(val3), put); 1941 assertEquals(false, res); 1942 1943 // Test CompareOp.LESS: original = val3, compare with val4, fail 1944 res = 1945 region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS, new BinaryComparator(val4), put); 1946 assertEquals(false, res); 1947 1948 // Test CompareOp.LESS: original = val3, compare with val2, 1949 // succeed (now value = val2) 1950 put = new Put(row1); 1951 put.addColumn(fam1, qf1, val2); 1952 res = 1953 region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS, new BinaryComparator(val2), put); 1954 assertEquals(true, res); 1955 1956 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val3, fail 1957 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS_OR_EQUAL, 1958 new BinaryComparator(val3), put); 1959 assertEquals(false, res); 1960 1961 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val2, 1962 // succeed (value still = val2) 1963 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS_OR_EQUAL, 1964 new BinaryComparator(val2), put); 1965 assertEquals(true, res); 1966 1967 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val1, 1968 // succeed (now value = val3) 1969 put = new Put(row1); 1970 put.addColumn(fam1, qf1, val3); 1971 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS_OR_EQUAL, 1972 new BinaryComparator(val1), put); 1973 assertEquals(true, res); 1974 1975 // Test CompareOp.GREATER: original = val3, compare with val3, fail 1976 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER, 1977 new BinaryComparator(val3), put); 1978 assertEquals(false, res); 1979 1980 // Test CompareOp.GREATER: original = val3, compare with val2, fail 1981 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER, 1982 new BinaryComparator(val2), put); 1983 assertEquals(false, res); 1984 1985 // Test CompareOp.GREATER: original = val3, compare with val4, 1986 // succeed (now value = val2) 1987 put = new Put(row1); 1988 put.addColumn(fam1, qf1, val2); 1989 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER, 1990 new BinaryComparator(val4), put); 1991 assertEquals(true, res); 1992 1993 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val1, fail 1994 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER_OR_EQUAL, 1995 new BinaryComparator(val1), put); 1996 assertEquals(false, res); 1997 1998 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val2, 1999 // succeed (value still = val2) 2000 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER_OR_EQUAL, 2001 new BinaryComparator(val2), put); 2002 assertEquals(true, res); 2003 2004 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val3, succeed 2005 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER_OR_EQUAL, 2006 new BinaryComparator(val3), put); 2007 assertEquals(true, res); 2008 } 2009 2010 @Test 2011 @Deprecated 2012 public void testCheckAndPut_ThatPutWasWritten() throws IOException { 2013 byte[] row1 = Bytes.toBytes("row1"); 2014 byte[] fam1 = Bytes.toBytes("fam1"); 2015 byte[] fam2 = Bytes.toBytes("fam2"); 2016 byte[] qf1 = Bytes.toBytes("qualifier"); 2017 byte[] val1 = Bytes.toBytes("value1"); 2018 byte[] val2 = Bytes.toBytes("value2"); 2019 2020 byte[][] families = { fam1, fam2 }; 2021 2022 // Setting up region 2023 this.region = initHRegion(tableName, method, CONF, families); 2024 // Putting data in the key to check 2025 Put put = new Put(row1); 2026 put.addColumn(fam1, qf1, val1); 2027 region.put(put); 2028 2029 // Creating put to add 2030 long ts = EnvironmentEdgeManager.currentTime(); 2031 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2); 2032 put = new Put(row1); 2033 put.add(kv); 2034 2035 // checkAndPut with wrong value 2036 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 2037 new BinaryComparator(val1), put); 2038 assertEquals(true, res); 2039 2040 Get get = new Get(row1); 2041 get.addColumn(fam2, qf1); 2042 Cell[] actual = region.get(get).rawCells(); 2043 2044 Cell[] expected = { kv }; 2045 2046 assertEquals(expected.length, actual.length); 2047 for (int i = 0; i < actual.length; i++) { 2048 assertEquals(expected[i], actual[i]); 2049 } 2050 } 2051 2052 @Test 2053 @Deprecated 2054 public void testCheckAndPut_wrongRowInPut() throws IOException { 2055 this.region = initHRegion(tableName, method, CONF, COLUMNS); 2056 Put put = new Put(row2); 2057 put.addColumn(fam1, qual1, value1); 2058 try { 2059 region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(value2), 2060 put); 2061 fail(); 2062 } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) { 2063 // expected exception. 2064 } 2065 } 2066 2067 @Test 2068 @Deprecated 2069 public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException { 2070 byte[] row1 = Bytes.toBytes("row1"); 2071 byte[] fam1 = Bytes.toBytes("fam1"); 2072 byte[] fam2 = Bytes.toBytes("fam2"); 2073 byte[] qf1 = Bytes.toBytes("qualifier1"); 2074 byte[] qf2 = Bytes.toBytes("qualifier2"); 2075 byte[] qf3 = Bytes.toBytes("qualifier3"); 2076 byte[] val1 = Bytes.toBytes("value1"); 2077 byte[] val2 = Bytes.toBytes("value2"); 2078 byte[] val3 = Bytes.toBytes("value3"); 2079 byte[] emptyVal = new byte[] {}; 2080 2081 byte[][] families = { fam1, fam2 }; 2082 2083 // Setting up region 2084 this.region = initHRegion(tableName, method, CONF, families); 2085 // Put content 2086 Put put = new Put(row1); 2087 put.addColumn(fam1, qf1, val1); 2088 region.put(put); 2089 Threads.sleep(2); 2090 2091 put = new Put(row1); 2092 put.addColumn(fam1, qf1, val2); 2093 put.addColumn(fam2, qf1, val3); 2094 put.addColumn(fam2, qf2, val2); 2095 put.addColumn(fam2, qf3, val1); 2096 put.addColumn(fam1, qf3, val1); 2097 region.put(put); 2098 2099 LOG.info("get={}", region.get(new Get(row1).addColumn(fam1, qf1)).toString()); 2100 2101 // Multi-column delete 2102 Delete delete = new Delete(row1); 2103 delete.addColumn(fam1, qf1); 2104 delete.addColumn(fam2, qf1); 2105 delete.addColumn(fam1, qf3); 2106 boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, 2107 new BinaryComparator(val2), delete); 2108 assertEquals(true, res); 2109 2110 Get get = new Get(row1); 2111 get.addColumn(fam1, qf1); 2112 get.addColumn(fam1, qf3); 2113 get.addColumn(fam2, qf2); 2114 Result r = region.get(get); 2115 assertEquals(2, r.size()); 2116 assertArrayEquals(val1, r.getValue(fam1, qf1)); 2117 assertArrayEquals(val2, r.getValue(fam2, qf2)); 2118 2119 // Family delete 2120 delete = new Delete(row1); 2121 delete.addFamily(fam2); 2122 res = region.checkAndMutate(row1, fam2, qf1, CompareOperator.EQUAL, 2123 new BinaryComparator(emptyVal), delete); 2124 assertEquals(true, res); 2125 2126 get = new Get(row1); 2127 r = region.get(get); 2128 assertEquals(1, r.size()); 2129 assertArrayEquals(val1, r.getValue(fam1, qf1)); 2130 2131 // Row delete 2132 delete = new Delete(row1); 2133 res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val1), 2134 delete); 2135 assertEquals(true, res); 2136 get = new Get(row1); 2137 r = region.get(get); 2138 assertEquals(0, r.size()); 2139 } 2140 2141 @Test 2142 @Deprecated 2143 public void testCheckAndMutate_WithFilters() throws Throwable { 2144 final byte[] FAMILY = Bytes.toBytes("fam"); 2145 2146 // Setting up region 2147 this.region = initHRegion(tableName, method, CONF, FAMILY); 2148 2149 // Put one row 2150 Put put = new Put(row); 2151 put.addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")); 2152 put.addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")); 2153 put.addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")); 2154 region.put(put); 2155 2156 // Put with success 2157 boolean ok = region.checkAndMutate(row, 2158 new FilterList( 2159 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2160 Bytes.toBytes("a")), 2161 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2162 Bytes.toBytes("b"))), 2163 new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d"))); 2164 assertTrue(ok); 2165 2166 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))); 2167 assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D")))); 2168 2169 // Put with failure 2170 ok = region.checkAndMutate(row, 2171 new FilterList( 2172 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2173 Bytes.toBytes("a")), 2174 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2175 Bytes.toBytes("c"))), 2176 new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e"))); 2177 assertFalse(ok); 2178 2179 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E"))).isEmpty()); 2180 2181 // Delete with success 2182 ok = region.checkAndMutate(row, 2183 new FilterList( 2184 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2185 Bytes.toBytes("a")), 2186 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2187 Bytes.toBytes("b"))), 2188 new Delete(row).addColumns(FAMILY, Bytes.toBytes("D"))); 2189 assertTrue(ok); 2190 2191 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))).isEmpty()); 2192 2193 // Mutate with success 2194 ok = region.checkAndRowMutate(row, 2195 new FilterList( 2196 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2197 Bytes.toBytes("a")), 2198 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2199 Bytes.toBytes("b"))), 2200 new RowMutations(row) 2201 .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e"))) 2202 .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A")))); 2203 assertTrue(ok); 2204 2205 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E"))); 2206 assertEquals("e", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("E")))); 2207 2208 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty()); 2209 } 2210 2211 @Test 2212 @Deprecated 2213 public void testCheckAndMutate_WithFiltersAndTimeRange() throws Throwable { 2214 final byte[] FAMILY = Bytes.toBytes("fam"); 2215 2216 // Setting up region 2217 this.region = initHRegion(tableName, method, CONF, FAMILY); 2218 2219 // Put with specifying the timestamp 2220 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), 100, Bytes.toBytes("a"))); 2221 2222 // Put with success 2223 boolean ok = region.checkAndMutate(row, 2224 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2225 Bytes.toBytes("a")), 2226 TimeRange.between(0, 101), 2227 new Put(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b"))); 2228 assertTrue(ok); 2229 2230 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2231 assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2232 2233 // Put with failure 2234 ok = region.checkAndMutate(row, 2235 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2236 Bytes.toBytes("a")), 2237 TimeRange.between(0, 100), 2238 new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c"))); 2239 assertFalse(ok); 2240 2241 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("C"))).isEmpty()); 2242 2243 // Mutate with success 2244 ok = region.checkAndRowMutate(row, 2245 new SingleColumnValueFilter( 2246 FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, Bytes.toBytes("a")), 2247 TimeRange.between(0, 101), 2248 new RowMutations(row) 2249 .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d"))) 2250 .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A")))); 2251 assertTrue(ok); 2252 2253 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))); 2254 assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D")))); 2255 2256 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty()); 2257 } 2258 2259 @Test 2260 @Deprecated 2261 public void testCheckAndMutate_wrongMutationType() throws Throwable { 2262 // Setting up region 2263 this.region = initHRegion(tableName, method, CONF, fam1); 2264 2265 try { 2266 region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(value1), 2267 new Increment(row).addColumn(fam1, qual1, 1)); 2268 fail("should throw DoNotRetryIOException"); 2269 } catch (DoNotRetryIOException e) { 2270 assertEquals("Unsupported mutate type: INCREMENT", e.getMessage()); 2271 } 2272 2273 try { 2274 region.checkAndMutate(row, 2275 new SingleColumnValueFilter(fam1, qual1, CompareOperator.EQUAL, value1), 2276 new Increment(row).addColumn(fam1, qual1, 1)); 2277 fail("should throw DoNotRetryIOException"); 2278 } catch (DoNotRetryIOException e) { 2279 assertEquals("Unsupported mutate type: INCREMENT", e.getMessage()); 2280 } 2281 } 2282 2283 @Test 2284 @Deprecated 2285 public void testCheckAndMutate_wrongRow() throws Throwable { 2286 final byte[] wrongRow = Bytes.toBytes("wrongRow"); 2287 2288 // Setting up region 2289 this.region = initHRegion(tableName, method, CONF, fam1); 2290 2291 try { 2292 region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(value1), 2293 new Put(wrongRow).addColumn(fam1, qual1, value1)); 2294 fail("should throw DoNotRetryIOException"); 2295 } catch (DoNotRetryIOException e) { 2296 assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>", 2297 e.getMessage()); 2298 } 2299 2300 try { 2301 region.checkAndMutate(row, 2302 new SingleColumnValueFilter(fam1, qual1, CompareOperator.EQUAL, value1), 2303 new Put(wrongRow).addColumn(fam1, qual1, value1)); 2304 fail("should throw DoNotRetryIOException"); 2305 } catch (DoNotRetryIOException e) { 2306 assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>", 2307 e.getMessage()); 2308 } 2309 2310 try { 2311 region.checkAndRowMutate(row, fam1, qual1, CompareOperator.EQUAL, 2312 new BinaryComparator(value1), 2313 new RowMutations(wrongRow).add((Mutation) new Put(wrongRow).addColumn(fam1, qual1, value1)) 2314 .add((Mutation) new Delete(wrongRow).addColumns(fam1, qual2))); 2315 fail("should throw DoNotRetryIOException"); 2316 } catch (DoNotRetryIOException e) { 2317 assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>", 2318 e.getMessage()); 2319 } 2320 2321 try { 2322 region.checkAndRowMutate(row, 2323 new SingleColumnValueFilter(fam1, qual1, CompareOperator.EQUAL, value1), 2324 new RowMutations(wrongRow).add((Mutation) new Put(wrongRow).addColumn(fam1, qual1, value1)) 2325 .add((Mutation) new Delete(wrongRow).addColumns(fam1, qual2))); 2326 fail("should throw DoNotRetryIOException"); 2327 } catch (DoNotRetryIOException e) { 2328 assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>", 2329 e.getMessage()); 2330 } 2331 } 2332 2333 @Test 2334 public void testCheckAndMutateWithEmptyRowValue() throws IOException { 2335 byte[] row1 = Bytes.toBytes("row1"); 2336 byte[] fam1 = Bytes.toBytes("fam1"); 2337 byte[] qf1 = Bytes.toBytes("qualifier"); 2338 byte[] emptyVal = new byte[] {}; 2339 byte[] val1 = Bytes.toBytes("value1"); 2340 byte[] val2 = Bytes.toBytes("value2"); 2341 2342 // Setting up region 2343 this.region = initHRegion(tableName, method, CONF, fam1); 2344 // Putting empty data in key 2345 Put put = new Put(row1); 2346 put.addColumn(fam1, qf1, emptyVal); 2347 2348 // checkAndPut with empty value 2349 CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2350 .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(put)); 2351 assertTrue(res.isSuccess()); 2352 assertNull(res.getResult()); 2353 2354 // Putting data in key 2355 put = new Put(row1); 2356 put.addColumn(fam1, qf1, val1); 2357 2358 // checkAndPut with correct value 2359 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2360 .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(put)); 2361 assertTrue(res.isSuccess()); 2362 assertNull(res.getResult()); 2363 2364 // not empty anymore 2365 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2366 .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(put)); 2367 assertFalse(res.isSuccess()); 2368 assertNull(res.getResult()); 2369 2370 Delete delete = new Delete(row1); 2371 delete.addColumn(fam1, qf1); 2372 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2373 .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(delete)); 2374 assertFalse(res.isSuccess()); 2375 assertNull(res.getResult()); 2376 2377 put = new Put(row1); 2378 put.addColumn(fam1, qf1, val2); 2379 // checkAndPut with correct value 2380 res = region.checkAndMutate( 2381 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(put)); 2382 assertTrue(res.isSuccess()); 2383 assertNull(res.getResult()); 2384 2385 // checkAndDelete with correct value 2386 delete = new Delete(row1); 2387 delete.addColumn(fam1, qf1); 2388 delete.addColumn(fam1, qf1); 2389 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2390 .ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(delete)); 2391 assertTrue(res.isSuccess()); 2392 assertNull(res.getResult()); 2393 2394 delete = new Delete(row1); 2395 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2396 .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(delete)); 2397 assertTrue(res.isSuccess()); 2398 assertNull(res.getResult()); 2399 2400 // checkAndPut looking for a null value 2401 put = new Put(row1); 2402 put.addColumn(fam1, qf1, val1); 2403 2404 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1).ifNotExists(fam1, qf1).build(put)); 2405 assertTrue(res.isSuccess()); 2406 assertNull(res.getResult()); 2407 } 2408 2409 @Test 2410 public void testCheckAndMutateWithWrongValue() throws IOException { 2411 byte[] row1 = Bytes.toBytes("row1"); 2412 byte[] fam1 = Bytes.toBytes("fam1"); 2413 byte[] qf1 = Bytes.toBytes("qualifier"); 2414 byte[] val1 = Bytes.toBytes("value1"); 2415 byte[] val2 = Bytes.toBytes("value2"); 2416 BigDecimal bd1 = new BigDecimal(Double.MAX_VALUE); 2417 BigDecimal bd2 = new BigDecimal(Double.MIN_VALUE); 2418 2419 // Setting up region 2420 this.region = initHRegion(tableName, method, CONF, fam1); 2421 // Putting data in key 2422 Put put = new Put(row1); 2423 put.addColumn(fam1, qf1, val1); 2424 region.put(put); 2425 2426 // checkAndPut with wrong value 2427 CheckAndMutateResult res = region.checkAndMutate( 2428 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(put)); 2429 assertFalse(res.isSuccess()); 2430 assertNull(res.getResult()); 2431 2432 // checkAndDelete with wrong value 2433 Delete delete = new Delete(row1); 2434 delete.addFamily(fam1); 2435 res = region.checkAndMutate( 2436 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(put)); 2437 assertFalse(res.isSuccess()); 2438 assertNull(res.getResult()); 2439 2440 // Putting data in key 2441 put = new Put(row1); 2442 put.addColumn(fam1, qf1, Bytes.toBytes(bd1)); 2443 region.put(put); 2444 2445 // checkAndPut with wrong value 2446 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2447 .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd2)).build(put)); 2448 assertFalse(res.isSuccess()); 2449 assertNull(res.getResult()); 2450 2451 // checkAndDelete with wrong value 2452 delete = new Delete(row1); 2453 delete.addFamily(fam1); 2454 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2455 .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd2)).build(delete)); 2456 assertFalse(res.isSuccess()); 2457 assertNull(res.getResult()); 2458 } 2459 2460 @Test 2461 public void testCheckAndMutateWithCorrectValue() throws IOException { 2462 byte[] row1 = Bytes.toBytes("row1"); 2463 byte[] fam1 = Bytes.toBytes("fam1"); 2464 byte[] qf1 = Bytes.toBytes("qualifier"); 2465 byte[] val1 = Bytes.toBytes("value1"); 2466 BigDecimal bd1 = new BigDecimal(Double.MIN_VALUE); 2467 2468 // Setting up region 2469 this.region = initHRegion(tableName, method, CONF, fam1); 2470 // Putting data in key 2471 long now = EnvironmentEdgeManager.currentTime(); 2472 Put put = new Put(row1); 2473 put.addColumn(fam1, qf1, now, val1); 2474 region.put(put); 2475 2476 // checkAndPut with correct value 2477 CheckAndMutateResult res = region.checkAndMutate( 2478 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(put)); 2479 assertTrue("First", res.isSuccess()); 2480 2481 // checkAndDelete with correct value 2482 Delete delete = new Delete(row1, now + 1); 2483 delete.addColumn(fam1, qf1); 2484 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2485 .ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(delete)); 2486 assertTrue("Delete", res.isSuccess()); 2487 assertNull(res.getResult()); 2488 2489 // Putting data in key 2490 put = new Put(row1); 2491 put.addColumn(fam1, qf1, now + 2, Bytes.toBytes(bd1)); 2492 region.put(put); 2493 2494 // checkAndPut with correct value 2495 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2496 .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd1)).build(put)); 2497 assertTrue("Second put", res.isSuccess()); 2498 assertNull(res.getResult()); 2499 2500 // checkAndDelete with correct value 2501 delete = new Delete(row1, now + 3); 2502 delete.addColumn(fam1, qf1); 2503 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2504 .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd1)).build(delete)); 2505 assertTrue("Second delete", res.isSuccess()); 2506 assertNull(res.getResult()); 2507 } 2508 2509 @Test 2510 public void testCheckAndMutateWithNonEqualCompareOp() throws IOException { 2511 byte[] row1 = Bytes.toBytes("row1"); 2512 byte[] fam1 = Bytes.toBytes("fam1"); 2513 byte[] qf1 = Bytes.toBytes("qualifier"); 2514 byte[] val1 = Bytes.toBytes("value1"); 2515 byte[] val2 = Bytes.toBytes("value2"); 2516 byte[] val3 = Bytes.toBytes("value3"); 2517 byte[] val4 = Bytes.toBytes("value4"); 2518 2519 // Setting up region 2520 this.region = initHRegion(tableName, method, CONF, fam1); 2521 // Putting val3 in key 2522 Put put = new Put(row1); 2523 put.addColumn(fam1, qf1, val3); 2524 region.put(put); 2525 2526 // Test CompareOp.LESS: original = val3, compare with val3, fail 2527 CheckAndMutateResult res = region.checkAndMutate( 2528 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.LESS, val3).build(put)); 2529 assertFalse(res.isSuccess()); 2530 assertNull(res.getResult()); 2531 2532 // Test CompareOp.LESS: original = val3, compare with val4, fail 2533 res = region.checkAndMutate( 2534 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.LESS, val4).build(put)); 2535 assertFalse(res.isSuccess()); 2536 assertNull(res.getResult()); 2537 2538 // Test CompareOp.LESS: original = val3, compare with val2, 2539 // succeed (now value = val2) 2540 put = new Put(row1); 2541 put.addColumn(fam1, qf1, val2); 2542 res = region.checkAndMutate( 2543 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.LESS, val2).build(put)); 2544 assertTrue(res.isSuccess()); 2545 assertNull(res.getResult()); 2546 2547 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val3, fail 2548 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2549 .ifMatches(fam1, qf1, CompareOperator.LESS_OR_EQUAL, val3).build(put)); 2550 assertFalse(res.isSuccess()); 2551 assertNull(res.getResult()); 2552 2553 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val2, 2554 // succeed (value still = val2) 2555 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2556 .ifMatches(fam1, qf1, CompareOperator.LESS_OR_EQUAL, val2).build(put)); 2557 assertTrue(res.isSuccess()); 2558 assertNull(res.getResult()); 2559 2560 // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val1, 2561 // succeed (now value = val3) 2562 put = new Put(row1); 2563 put.addColumn(fam1, qf1, val3); 2564 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2565 .ifMatches(fam1, qf1, CompareOperator.LESS_OR_EQUAL, val1).build(put)); 2566 assertTrue(res.isSuccess()); 2567 assertNull(res.getResult()); 2568 2569 // Test CompareOp.GREATER: original = val3, compare with val3, fail 2570 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2571 .ifMatches(fam1, qf1, CompareOperator.GREATER, val3).build(put)); 2572 assertFalse(res.isSuccess()); 2573 assertNull(res.getResult()); 2574 2575 // Test CompareOp.GREATER: original = val3, compare with val2, fail 2576 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2577 .ifMatches(fam1, qf1, CompareOperator.GREATER, val2).build(put)); 2578 assertFalse(res.isSuccess()); 2579 assertNull(res.getResult()); 2580 2581 // Test CompareOp.GREATER: original = val3, compare with val4, 2582 // succeed (now value = val2) 2583 put = new Put(row1); 2584 put.addColumn(fam1, qf1, val2); 2585 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2586 .ifMatches(fam1, qf1, CompareOperator.GREATER, val4).build(put)); 2587 assertTrue(res.isSuccess()); 2588 assertNull(res.getResult()); 2589 2590 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val1, fail 2591 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2592 .ifMatches(fam1, qf1, CompareOperator.GREATER_OR_EQUAL, val1).build(put)); 2593 assertFalse(res.isSuccess()); 2594 assertNull(res.getResult()); 2595 2596 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val2, 2597 // succeed (value still = val2) 2598 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2599 .ifMatches(fam1, qf1, CompareOperator.GREATER_OR_EQUAL, val2).build(put)); 2600 assertTrue(res.isSuccess()); 2601 assertNull(res.getResult()); 2602 2603 // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val3, succeed 2604 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2605 .ifMatches(fam1, qf1, CompareOperator.GREATER_OR_EQUAL, val3).build(put)); 2606 assertTrue(res.isSuccess()); 2607 assertNull(res.getResult()); 2608 } 2609 2610 @Test 2611 public void testCheckAndPutThatPutWasWritten() throws IOException { 2612 byte[] row1 = Bytes.toBytes("row1"); 2613 byte[] fam1 = Bytes.toBytes("fam1"); 2614 byte[] fam2 = Bytes.toBytes("fam2"); 2615 byte[] qf1 = Bytes.toBytes("qualifier"); 2616 byte[] val1 = Bytes.toBytes("value1"); 2617 byte[] val2 = Bytes.toBytes("value2"); 2618 2619 byte[][] families = { fam1, fam2 }; 2620 2621 // Setting up region 2622 this.region = initHRegion(tableName, method, CONF, families); 2623 // Putting data in the key to check 2624 Put put = new Put(row1); 2625 put.addColumn(fam1, qf1, val1); 2626 region.put(put); 2627 2628 // Creating put to add 2629 long ts = EnvironmentEdgeManager.currentTime(); 2630 KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2); 2631 put = new Put(row1); 2632 put.add(kv); 2633 2634 // checkAndPut with wrong value 2635 CheckAndMutateResult res = region.checkAndMutate( 2636 CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(put)); 2637 assertTrue(res.isSuccess()); 2638 assertNull(res.getResult()); 2639 2640 Get get = new Get(row1); 2641 get.addColumn(fam2, qf1); 2642 Cell[] actual = region.get(get).rawCells(); 2643 2644 Cell[] expected = { kv }; 2645 2646 assertEquals(expected.length, actual.length); 2647 for (int i = 0; i < actual.length; i++) { 2648 assertEquals(expected[i], actual[i]); 2649 } 2650 } 2651 2652 @Test 2653 public void testCheckAndDeleteThatDeleteWasWritten() throws IOException { 2654 byte[] row1 = Bytes.toBytes("row1"); 2655 byte[] fam1 = Bytes.toBytes("fam1"); 2656 byte[] fam2 = Bytes.toBytes("fam2"); 2657 byte[] qf1 = Bytes.toBytes("qualifier1"); 2658 byte[] qf2 = Bytes.toBytes("qualifier2"); 2659 byte[] qf3 = Bytes.toBytes("qualifier3"); 2660 byte[] val1 = Bytes.toBytes("value1"); 2661 byte[] val2 = Bytes.toBytes("value2"); 2662 byte[] val3 = Bytes.toBytes("value3"); 2663 byte[] emptyVal = new byte[] {}; 2664 2665 byte[][] families = { fam1, fam2 }; 2666 2667 // Setting up region 2668 this.region = initHRegion(tableName, method, CONF, families); 2669 // Put content 2670 Put put = new Put(row1); 2671 put.addColumn(fam1, qf1, val1); 2672 region.put(put); 2673 Threads.sleep(2); 2674 2675 put = new Put(row1); 2676 put.addColumn(fam1, qf1, val2); 2677 put.addColumn(fam2, qf1, val3); 2678 put.addColumn(fam2, qf2, val2); 2679 put.addColumn(fam2, qf3, val1); 2680 put.addColumn(fam1, qf3, val1); 2681 region.put(put); 2682 2683 LOG.info("get={}", region.get(new Get(row1).addColumn(fam1, qf1)).toString()); 2684 2685 // Multi-column delete 2686 Delete delete = new Delete(row1); 2687 delete.addColumn(fam1, qf1); 2688 delete.addColumn(fam2, qf1); 2689 delete.addColumn(fam1, qf3); 2690 CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2691 .ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(delete)); 2692 assertTrue(res.isSuccess()); 2693 assertNull(res.getResult()); 2694 2695 Get get = new Get(row1); 2696 get.addColumn(fam1, qf1); 2697 get.addColumn(fam1, qf3); 2698 get.addColumn(fam2, qf2); 2699 Result r = region.get(get); 2700 assertEquals(2, r.size()); 2701 assertArrayEquals(val1, r.getValue(fam1, qf1)); 2702 assertArrayEquals(val2, r.getValue(fam2, qf2)); 2703 2704 // Family delete 2705 delete = new Delete(row1); 2706 delete.addFamily(fam2); 2707 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2708 .ifMatches(fam2, qf1, CompareOperator.EQUAL, emptyVal).build(delete)); 2709 assertTrue(res.isSuccess()); 2710 assertNull(res.getResult()); 2711 2712 get = new Get(row1); 2713 r = region.get(get); 2714 assertEquals(1, r.size()); 2715 assertArrayEquals(val1, r.getValue(fam1, qf1)); 2716 2717 // Row delete 2718 delete = new Delete(row1); 2719 res = region.checkAndMutate(CheckAndMutate.newBuilder(row1) 2720 .ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(delete)); 2721 assertTrue(res.isSuccess()); 2722 assertNull(res.getResult()); 2723 2724 get = new Get(row1); 2725 r = region.get(get); 2726 assertEquals(0, r.size()); 2727 } 2728 2729 @Test 2730 public void testCheckAndMutateWithFilters() throws Throwable { 2731 final byte[] FAMILY = Bytes.toBytes("fam"); 2732 2733 // Setting up region 2734 this.region = initHRegion(tableName, method, CONF, FAMILY); 2735 2736 // Put one row 2737 Put put = new Put(row); 2738 put.addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")); 2739 put.addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")); 2740 put.addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")); 2741 region.put(put); 2742 2743 // Put with success 2744 CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2745 .ifMatches(new FilterList( 2746 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2747 Bytes.toBytes("a")), 2748 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2749 Bytes.toBytes("b")))) 2750 .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d")))); 2751 assertTrue(res.isSuccess()); 2752 assertNull(res.getResult()); 2753 2754 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))); 2755 assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D")))); 2756 2757 // Put with failure 2758 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2759 .ifMatches(new FilterList( 2760 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2761 Bytes.toBytes("a")), 2762 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2763 Bytes.toBytes("c")))) 2764 .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e")))); 2765 assertFalse(res.isSuccess()); 2766 assertNull(res.getResult()); 2767 2768 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E"))).isEmpty()); 2769 2770 // Delete with success 2771 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2772 .ifMatches(new FilterList( 2773 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2774 Bytes.toBytes("a")), 2775 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2776 Bytes.toBytes("b")))) 2777 .build(new Delete(row).addColumns(FAMILY, Bytes.toBytes("D")))); 2778 assertTrue(res.isSuccess()); 2779 assertNull(res.getResult()); 2780 2781 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))).isEmpty()); 2782 2783 // Mutate with success 2784 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2785 .ifMatches(new FilterList( 2786 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2787 Bytes.toBytes("a")), 2788 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL, 2789 Bytes.toBytes("b")))) 2790 .build(new RowMutations(row) 2791 .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e"))) 2792 .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A"))))); 2793 assertTrue(res.isSuccess()); 2794 assertNull(res.getResult()); 2795 2796 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E"))); 2797 assertEquals("e", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("E")))); 2798 2799 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty()); 2800 } 2801 2802 @Test 2803 public void testCheckAndMutateWithFiltersAndTimeRange() throws Throwable { 2804 final byte[] FAMILY = Bytes.toBytes("fam"); 2805 2806 // Setting up region 2807 this.region = initHRegion(tableName, method, CONF, FAMILY); 2808 2809 // Put with specifying the timestamp 2810 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), 100, Bytes.toBytes("a"))); 2811 2812 // Put with success 2813 CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2814 .ifMatches(new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2815 Bytes.toBytes("a"))) 2816 .timeRange(TimeRange.between(0, 101)) 2817 .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")))); 2818 assertTrue(res.isSuccess()); 2819 assertNull(res.getResult()); 2820 2821 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2822 assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2823 2824 // Put with failure 2825 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2826 .ifMatches(new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2827 Bytes.toBytes("a"))) 2828 .timeRange(TimeRange.between(0, 100)) 2829 .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")))); 2830 assertFalse(res.isSuccess()); 2831 assertNull(res.getResult()); 2832 2833 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("C"))).isEmpty()); 2834 2835 // RowMutations with success 2836 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2837 .ifMatches(new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2838 Bytes.toBytes("a"))) 2839 .timeRange(TimeRange.between(0, 101)) 2840 .build(new RowMutations(row) 2841 .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d"))) 2842 .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A"))))); 2843 assertTrue(res.isSuccess()); 2844 assertNull(res.getResult()); 2845 2846 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))); 2847 assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D")))); 2848 2849 assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty()); 2850 } 2851 2852 @Test 2853 public void testCheckAndIncrement() throws Throwable { 2854 final byte[] FAMILY = Bytes.toBytes("fam"); 2855 2856 // Setting up region 2857 this.region = initHRegion(tableName, method, CONF, FAMILY); 2858 2859 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a"))); 2860 2861 // CheckAndIncrement with correct value 2862 CheckAndMutateResult res = region.checkAndMutate( 2863 CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")) 2864 .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 1))); 2865 assertTrue(res.isSuccess()); 2866 assertEquals(1, Bytes.toLong(res.getResult().getValue(FAMILY, Bytes.toBytes("B")))); 2867 2868 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2869 assertEquals(1, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B")))); 2870 2871 // CheckAndIncrement with wrong value 2872 res = region.checkAndMutate( 2873 CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("b")) 2874 .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 1))); 2875 assertFalse(res.isSuccess()); 2876 assertNull(res.getResult()); 2877 2878 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2879 assertEquals(1, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B")))); 2880 2881 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c"))); 2882 2883 // CheckAndIncrement with a filter and correct value 2884 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2885 .ifMatches(new FilterList( 2886 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2887 Bytes.toBytes("a")), 2888 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL, 2889 Bytes.toBytes("c")))) 2890 .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 2))); 2891 assertTrue(res.isSuccess()); 2892 assertEquals(3, Bytes.toLong(res.getResult().getValue(FAMILY, Bytes.toBytes("B")))); 2893 2894 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2895 assertEquals(3, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B")))); 2896 2897 // CheckAndIncrement with a filter and correct value 2898 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2899 .ifMatches(new FilterList( 2900 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2901 Bytes.toBytes("b")), 2902 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL, 2903 Bytes.toBytes("d")))) 2904 .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 2))); 2905 assertFalse(res.isSuccess()); 2906 assertNull(res.getResult()); 2907 2908 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2909 assertEquals(3, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B")))); 2910 } 2911 2912 @Test 2913 public void testCheckAndAppend() throws Throwable { 2914 final byte[] FAMILY = Bytes.toBytes("fam"); 2915 2916 // Setting up region 2917 this.region = initHRegion(tableName, method, CONF, FAMILY); 2918 2919 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a"))); 2920 2921 // CheckAndAppend with correct value 2922 CheckAndMutateResult res = region.checkAndMutate( 2923 CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")) 2924 .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")))); 2925 assertTrue(res.isSuccess()); 2926 assertEquals("b", Bytes.toString(res.getResult().getValue(FAMILY, Bytes.toBytes("B")))); 2927 2928 Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2929 assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2930 2931 // CheckAndAppend with wrong value 2932 res = region.checkAndMutate( 2933 CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("b")) 2934 .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")))); 2935 assertFalse(res.isSuccess()); 2936 assertNull(res.getResult()); 2937 2938 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2939 assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2940 2941 region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c"))); 2942 2943 // CheckAndAppend with a filter and correct value 2944 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2945 .ifMatches(new FilterList( 2946 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2947 Bytes.toBytes("a")), 2948 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL, 2949 Bytes.toBytes("c")))) 2950 .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("bb")))); 2951 assertTrue(res.isSuccess()); 2952 assertEquals("bbb", Bytes.toString(res.getResult().getValue(FAMILY, Bytes.toBytes("B")))); 2953 2954 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2955 assertEquals("bbb", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2956 2957 // CheckAndAppend with a filter and wrong value 2958 res = region.checkAndMutate(CheckAndMutate.newBuilder(row) 2959 .ifMatches(new FilterList( 2960 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, 2961 Bytes.toBytes("b")), 2962 new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL, 2963 Bytes.toBytes("d")))) 2964 .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("bb")))); 2965 assertFalse(res.isSuccess()); 2966 assertNull(res.getResult()); 2967 2968 result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B"))); 2969 assertEquals("bbb", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B")))); 2970 } 2971 2972 @Test 2973 public void testCheckAndIncrementAndAppend() throws Throwable { 2974 // Setting up region 2975 this.region = initHRegion(tableName, method, CONF, fam1); 2976 2977 // CheckAndMutate with Increment and Append 2978 CheckAndMutate checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, qual) 2979 .build(new RowMutations(row).add((Mutation) new Increment(row).addColumn(fam1, qual1, 1L)) 2980 .add((Mutation) new Append(row).addColumn(fam1, qual2, Bytes.toBytes("a")))); 2981 2982 CheckAndMutateResult result = region.checkAndMutate(checkAndMutate); 2983 assertTrue(result.isSuccess()); 2984 assertEquals(1L, Bytes.toLong(result.getResult().getValue(fam1, qual1))); 2985 assertEquals("a", Bytes.toString(result.getResult().getValue(fam1, qual2))); 2986 2987 Result r = region.get(new Get(row)); 2988 assertEquals(1L, Bytes.toLong(r.getValue(fam1, qual1))); 2989 assertEquals("a", Bytes.toString(r.getValue(fam1, qual2))); 2990 2991 // Set return results to false 2992 checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, qual) 2993 .build(new RowMutations(row) 2994 .add((Mutation) new Increment(row).addColumn(fam1, qual1, 1L).setReturnResults(false)) 2995 .add((Mutation) new Append(row).addColumn(fam1, qual2, Bytes.toBytes("a")) 2996 .setReturnResults(false))); 2997 2998 result = region.checkAndMutate(checkAndMutate); 2999 assertTrue(result.isSuccess()); 3000 assertNull(result.getResult().getValue(fam1, qual1)); 3001 assertNull(result.getResult().getValue(fam1, qual2)); 3002 3003 r = region.get(new Get(row)); 3004 assertEquals(2L, Bytes.toLong(r.getValue(fam1, qual1))); 3005 assertEquals("aa", Bytes.toString(r.getValue(fam1, qual2))); 3006 3007 checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, qual) 3008 .build(new RowMutations(row).add((Mutation) new Increment(row).addColumn(fam1, qual1, 1L)) 3009 .add((Mutation) new Append(row).addColumn(fam1, qual2, Bytes.toBytes("a")) 3010 .setReturnResults(false))); 3011 3012 result = region.checkAndMutate(checkAndMutate); 3013 assertTrue(result.isSuccess()); 3014 assertEquals(3L, Bytes.toLong(result.getResult().getValue(fam1, qual1))); 3015 assertNull(result.getResult().getValue(fam1, qual2)); 3016 3017 r = region.get(new Get(row)); 3018 assertEquals(3L, Bytes.toLong(r.getValue(fam1, qual1))); 3019 assertEquals("aaa", Bytes.toString(r.getValue(fam1, qual2))); 3020 } 3021 3022 @Test 3023 public void testCheckAndRowMutations() throws Throwable { 3024 final byte[] row = Bytes.toBytes("row"); 3025 final byte[] q1 = Bytes.toBytes("q1"); 3026 final byte[] q2 = Bytes.toBytes("q2"); 3027 final byte[] q3 = Bytes.toBytes("q3"); 3028 final byte[] q4 = Bytes.toBytes("q4"); 3029 final String v1 = "v1"; 3030 3031 region = initHRegion(tableName, method, CONF, fam1); 3032 3033 // Initial values 3034 region 3035 .batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q2, Bytes.toBytes("toBeDeleted")), 3036 new Put(row).addColumn(fam1, q3, Bytes.toBytes(5L)), 3037 new Put(row).addColumn(fam1, q4, Bytes.toBytes("a")), }); 3038 3039 // Do CheckAndRowMutations 3040 CheckAndMutate checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, q1).build( 3041 new RowMutations(row).add(Arrays.asList(new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1)), 3042 new Delete(row).addColumns(fam1, q2), new Increment(row).addColumn(fam1, q3, 1), 3043 new Append(row).addColumn(fam1, q4, Bytes.toBytes("b"))))); 3044 3045 CheckAndMutateResult result = region.checkAndMutate(checkAndMutate); 3046 assertTrue(result.isSuccess()); 3047 assertEquals(6L, Bytes.toLong(result.getResult().getValue(fam1, q3))); 3048 assertEquals("ab", Bytes.toString(result.getResult().getValue(fam1, q4))); 3049 3050 // Verify the value 3051 Result r = region.get(new Get(row)); 3052 assertEquals(v1, Bytes.toString(r.getValue(fam1, q1))); 3053 assertNull(r.getValue(fam1, q2)); 3054 assertEquals(6L, Bytes.toLong(r.getValue(fam1, q3))); 3055 assertEquals("ab", Bytes.toString(r.getValue(fam1, q4))); 3056 3057 // Do CheckAndRowMutations again 3058 checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, q1) 3059 .build(new RowMutations(row).add(Arrays.asList(new Delete(row).addColumns(fam1, q1), 3060 new Put(row).addColumn(fam1, q2, Bytes.toBytes(v1)), 3061 new Increment(row).addColumn(fam1, q3, 1), 3062 new Append(row).addColumn(fam1, q4, Bytes.toBytes("b"))))); 3063 3064 result = region.checkAndMutate(checkAndMutate); 3065 assertFalse(result.isSuccess()); 3066 assertNull(result.getResult()); 3067 3068 // Verify the value 3069 r = region.get(new Get(row)); 3070 assertEquals(v1, Bytes.toString(r.getValue(fam1, q1))); 3071 assertNull(r.getValue(fam1, q2)); 3072 assertEquals(6L, Bytes.toLong(r.getValue(fam1, q3))); 3073 assertEquals("ab", Bytes.toString(r.getValue(fam1, q4))); 3074 } 3075 3076 // //////////////////////////////////////////////////////////////////////////// 3077 // Delete tests 3078 // //////////////////////////////////////////////////////////////////////////// 3079 @Test 3080 public void testDelete_multiDeleteColumn() throws IOException { 3081 byte[] row1 = Bytes.toBytes("row1"); 3082 byte[] fam1 = Bytes.toBytes("fam1"); 3083 byte[] qual = Bytes.toBytes("qualifier"); 3084 byte[] value = Bytes.toBytes("value"); 3085 3086 Put put = new Put(row1); 3087 put.addColumn(fam1, qual, 1, value); 3088 put.addColumn(fam1, qual, 2, value); 3089 3090 this.region = initHRegion(tableName, method, CONF, fam1); 3091 region.put(put); 3092 3093 // We do support deleting more than 1 'latest' version 3094 Delete delete = new Delete(row1); 3095 delete.addColumn(fam1, qual); 3096 delete.addColumn(fam1, qual); 3097 region.delete(delete); 3098 3099 Get get = new Get(row1); 3100 get.addFamily(fam1); 3101 Result r = region.get(get); 3102 assertEquals(0, r.size()); 3103 } 3104 3105 @Test 3106 public void testDelete_CheckFamily() throws IOException { 3107 byte[] row1 = Bytes.toBytes("row1"); 3108 byte[] fam1 = Bytes.toBytes("fam1"); 3109 byte[] fam2 = Bytes.toBytes("fam2"); 3110 byte[] fam3 = Bytes.toBytes("fam3"); 3111 byte[] fam4 = Bytes.toBytes("fam4"); 3112 3113 // Setting up region 3114 this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3); 3115 List<Cell> kvs = new ArrayList<>(); 3116 kvs.add(new KeyValue(row1, fam4, null, null)); 3117 3118 byte[] forUnitTestsOnly = Bytes.toBytes("ForUnitTestsOnly"); 3119 3120 // testing existing family 3121 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<>(Bytes.BYTES_COMPARATOR); 3122 deleteMap.put(fam2, kvs); 3123 region.delete(new Delete(forUnitTestsOnly, HConstants.LATEST_TIMESTAMP, deleteMap)); 3124 3125 // testing non existing family 3126 NavigableMap<byte[], List<Cell>> deleteMap2 = new TreeMap<>(Bytes.BYTES_COMPARATOR); 3127 deleteMap2.put(fam4, kvs); 3128 assertThrows("Family " + Bytes.toString(fam4) + " does exist", 3129 NoSuchColumnFamilyException.class, 3130 () -> region.delete(new Delete(forUnitTestsOnly, HConstants.LATEST_TIMESTAMP, deleteMap2))); 3131 } 3132 3133 @Test 3134 public void testDelete_mixed() throws IOException, InterruptedException { 3135 byte[] fam = Bytes.toBytes("info"); 3136 byte[][] families = { fam }; 3137 this.region = initHRegion(tableName, method, CONF, families); 3138 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge()); 3139 3140 byte[] row = Bytes.toBytes("table_name"); 3141 // column names 3142 byte[] serverinfo = Bytes.toBytes("serverinfo"); 3143 byte[] splitA = Bytes.toBytes("splitA"); 3144 byte[] splitB = Bytes.toBytes("splitB"); 3145 3146 // add some data: 3147 Put put = new Put(row); 3148 put.addColumn(fam, splitA, Bytes.toBytes("reference_A")); 3149 region.put(put); 3150 3151 put = new Put(row); 3152 put.addColumn(fam, splitB, Bytes.toBytes("reference_B")); 3153 region.put(put); 3154 3155 put = new Put(row); 3156 put.addColumn(fam, serverinfo, Bytes.toBytes("ip_address")); 3157 region.put(put); 3158 3159 // ok now delete a split: 3160 Delete delete = new Delete(row); 3161 delete.addColumns(fam, splitA); 3162 region.delete(delete); 3163 3164 // assert some things: 3165 Get get = new Get(row).addColumn(fam, serverinfo); 3166 Result result = region.get(get); 3167 assertEquals(1, result.size()); 3168 3169 get = new Get(row).addColumn(fam, splitA); 3170 result = region.get(get); 3171 assertEquals(0, result.size()); 3172 3173 get = new Get(row).addColumn(fam, splitB); 3174 result = region.get(get); 3175 assertEquals(1, result.size()); 3176 3177 // Assert that after a delete, I can put. 3178 put = new Put(row); 3179 put.addColumn(fam, splitA, Bytes.toBytes("reference_A")); 3180 region.put(put); 3181 get = new Get(row); 3182 result = region.get(get); 3183 assertEquals(3, result.size()); 3184 3185 // Now delete all... then test I can add stuff back 3186 delete = new Delete(row); 3187 region.delete(delete); 3188 assertEquals(0, region.get(get).size()); 3189 3190 region.put(new Put(row).addColumn(fam, splitA, Bytes.toBytes("reference_A"))); 3191 result = region.get(get); 3192 assertEquals(1, result.size()); 3193 } 3194 3195 @Test 3196 public void testDeleteRowWithFutureTs() throws IOException { 3197 byte[] fam = Bytes.toBytes("info"); 3198 byte[][] families = { fam }; 3199 this.region = initHRegion(tableName, method, CONF, families); 3200 byte[] row = Bytes.toBytes("table_name"); 3201 // column names 3202 byte[] serverinfo = Bytes.toBytes("serverinfo"); 3203 3204 // add data in the far future 3205 Put put = new Put(row); 3206 put.addColumn(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value")); 3207 region.put(put); 3208 3209 // now delete something in the present 3210 Delete delete = new Delete(row); 3211 region.delete(delete); 3212 3213 // make sure we still see our data 3214 Get get = new Get(row).addColumn(fam, serverinfo); 3215 Result result = region.get(get); 3216 assertEquals(1, result.size()); 3217 3218 // delete the future row 3219 delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3); 3220 region.delete(delete); 3221 3222 // make sure it is gone 3223 get = new Get(row).addColumn(fam, serverinfo); 3224 result = region.get(get); 3225 assertEquals(0, result.size()); 3226 } 3227 3228 /** 3229 * Tests that the special LATEST_TIMESTAMP option for puts gets replaced by the actual timestamp 3230 */ 3231 @Test 3232 public void testPutWithLatestTS() throws IOException { 3233 byte[] fam = Bytes.toBytes("info"); 3234 byte[][] families = { fam }; 3235 this.region = initHRegion(tableName, method, CONF, families); 3236 byte[] row = Bytes.toBytes("row1"); 3237 // column names 3238 byte[] qual = Bytes.toBytes("qual"); 3239 3240 // add data with LATEST_TIMESTAMP, put without WAL 3241 Put put = new Put(row); 3242 put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value")); 3243 region.put(put); 3244 3245 // Make sure it shows up with an actual timestamp 3246 Get get = new Get(row).addColumn(fam, qual); 3247 Result result = region.get(get); 3248 assertEquals(1, result.size()); 3249 Cell kv = result.rawCells()[0]; 3250 LOG.info("Got: " + kv); 3251 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp", 3252 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP); 3253 3254 // Check same with WAL enabled (historically these took different 3255 // code paths, so check both) 3256 row = Bytes.toBytes("row2"); 3257 put = new Put(row); 3258 put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value")); 3259 region.put(put); 3260 3261 // Make sure it shows up with an actual timestamp 3262 get = new Get(row).addColumn(fam, qual); 3263 result = region.get(get); 3264 assertEquals(1, result.size()); 3265 kv = result.rawCells()[0]; 3266 LOG.info("Got: " + kv); 3267 assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp", 3268 kv.getTimestamp() != HConstants.LATEST_TIMESTAMP); 3269 } 3270 3271 /** 3272 * Tests that there is server-side filtering for invalid timestamp upper bound. Note that the 3273 * timestamp lower bound is automatically handled for us by the TTL field. 3274 */ 3275 @Test 3276 public void testPutWithTsSlop() throws IOException { 3277 byte[] fam = Bytes.toBytes("info"); 3278 byte[][] families = { fam }; 3279 3280 // add data with a timestamp that is too recent for range. Ensure assert 3281 CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000); 3282 this.region = initHRegion(tableName, method, CONF, families); 3283 boolean caughtExcep = false; 3284 try { 3285 // no TS specified == use latest. should not error 3286 region.put(new Put(row).addColumn(fam, Bytes.toBytes("qual"), Bytes.toBytes("value"))); 3287 // TS out of range. should error 3288 region.put(new Put(row).addColumn(fam, Bytes.toBytes("qual"), 3289 EnvironmentEdgeManager.currentTime() + 2000, Bytes.toBytes("value"))); 3290 fail("Expected IOE for TS out of configured timerange"); 3291 } catch (FailedSanityCheckException ioe) { 3292 LOG.debug("Received expected exception", ioe); 3293 caughtExcep = true; 3294 } 3295 assertTrue("Should catch FailedSanityCheckException", caughtExcep); 3296 } 3297 3298 @Test 3299 public void testScanner_DeleteOneFamilyNotAnother() throws IOException { 3300 byte[] fam1 = Bytes.toBytes("columnA"); 3301 byte[] fam2 = Bytes.toBytes("columnB"); 3302 this.region = initHRegion(tableName, method, CONF, fam1, fam2); 3303 byte[] rowA = Bytes.toBytes("rowA"); 3304 byte[] rowB = Bytes.toBytes("rowB"); 3305 3306 byte[] value = Bytes.toBytes("value"); 3307 3308 Delete delete = new Delete(rowA); 3309 delete.addFamily(fam1); 3310 3311 region.delete(delete); 3312 3313 // now create data. 3314 Put put = new Put(rowA); 3315 put.addColumn(fam2, null, value); 3316 region.put(put); 3317 3318 put = new Put(rowB); 3319 put.addColumn(fam1, null, value); 3320 put.addColumn(fam2, null, value); 3321 region.put(put); 3322 3323 Scan scan = new Scan(); 3324 scan.addFamily(fam1).addFamily(fam2); 3325 InternalScanner s = region.getScanner(scan); 3326 List<Cell> results = new ArrayList<>(); 3327 s.next(results); 3328 assertTrue(CellUtil.matchingRows(results.get(0), rowA)); 3329 3330 results.clear(); 3331 s.next(results); 3332 assertTrue(CellUtil.matchingRows(results.get(0), rowB)); 3333 } 3334 3335 @Test 3336 public void testDataInMemoryWithoutWAL() throws IOException { 3337 FileSystem fs = FileSystem.get(CONF); 3338 Path rootDir = new Path(dir + "testDataInMemoryWithoutWAL"); 3339 FSHLog hLog = new FSHLog(fs, rootDir, "testDataInMemoryWithoutWAL", CONF); 3340 hLog.init(); 3341 // This chunk creation is done throughout the code base. Do we want to move it into core? 3342 // It is missing from this test. W/o it we NPE. 3343 region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, hLog, 3344 COLUMN_FAMILY_BYTES); 3345 3346 Cell originalCell = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row) 3347 .setFamily(COLUMN_FAMILY_BYTES).setQualifier(qual1) 3348 .setTimestamp(EnvironmentEdgeManager.currentTime()).setType(KeyValue.Type.Put.getCode()) 3349 .setValue(value1).build(); 3350 final long originalSize = originalCell.getSerializedSize(); 3351 3352 Cell addCell = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row) 3353 .setFamily(COLUMN_FAMILY_BYTES).setQualifier(qual1) 3354 .setTimestamp(EnvironmentEdgeManager.currentTime()).setType(KeyValue.Type.Put.getCode()) 3355 .setValue(Bytes.toBytes("xxxxxxxxxx")).build(); 3356 final long addSize = addCell.getSerializedSize(); 3357 3358 LOG.info("originalSize:" + originalSize + ", addSize:" + addSize); 3359 // start test. We expect that the addPut's durability will be replaced 3360 // by originalPut's durability. 3361 3362 // case 1: 3363 testDataInMemoryWithoutWAL(region, 3364 new Put(row).add(originalCell).setDurability(Durability.SKIP_WAL), 3365 new Put(row).add(addCell).setDurability(Durability.SKIP_WAL), originalSize + addSize); 3366 3367 // case 2: 3368 testDataInMemoryWithoutWAL(region, 3369 new Put(row).add(originalCell).setDurability(Durability.SKIP_WAL), 3370 new Put(row).add(addCell).setDurability(Durability.SYNC_WAL), originalSize + addSize); 3371 3372 // case 3: 3373 testDataInMemoryWithoutWAL(region, 3374 new Put(row).add(originalCell).setDurability(Durability.SYNC_WAL), 3375 new Put(row).add(addCell).setDurability(Durability.SKIP_WAL), 0); 3376 3377 // case 4: 3378 testDataInMemoryWithoutWAL(region, 3379 new Put(row).add(originalCell).setDurability(Durability.SYNC_WAL), 3380 new Put(row).add(addCell).setDurability(Durability.SYNC_WAL), 0); 3381 } 3382 3383 private static void testDataInMemoryWithoutWAL(HRegion region, Put originalPut, final Put addPut, 3384 long delta) throws IOException { 3385 final long initSize = region.getDataInMemoryWithoutWAL(); 3386 // save normalCPHost and replaced by mockedCPHost 3387 RegionCoprocessorHost normalCPHost = region.getCoprocessorHost(); 3388 RegionCoprocessorHost mockedCPHost = mock(RegionCoprocessorHost.class); 3389 // Because the preBatchMutate returns void, we can't do usual Mockito when...then form. Must 3390 // do below format (from Mockito doc). 3391 doAnswer(new Answer<Void>() { 3392 @Override 3393 public Void answer(InvocationOnMock invocation) throws Throwable { 3394 MiniBatchOperationInProgress<Mutation> mb = invocation.getArgument(0); 3395 mb.addOperationsFromCP(0, new Mutation[] { addPut }); 3396 return null; 3397 } 3398 }).when(mockedCPHost).preBatchMutate(isA(MiniBatchOperationInProgress.class)); 3399 ColumnFamilyDescriptorBuilder builder = 3400 ColumnFamilyDescriptorBuilder.newBuilder(COLUMN_FAMILY_BYTES); 3401 ScanInfo info = new ScanInfo(CONF, builder.build(), Long.MAX_VALUE, Long.MAX_VALUE, 3402 region.getCellComparator()); 3403 when(mockedCPHost.preFlushScannerOpen(any(HStore.class), any())).thenReturn(info); 3404 3405 when(mockedCPHost.preFlush(any(), any(StoreScanner.class), any())) 3406 .thenAnswer(i -> i.getArgument(1)); 3407 region.setCoprocessorHost(mockedCPHost); 3408 3409 region.put(originalPut); 3410 region.setCoprocessorHost(normalCPHost); 3411 final long finalSize = region.getDataInMemoryWithoutWAL(); 3412 assertEquals("finalSize:" + finalSize + ", initSize:" + initSize + ", delta:" + delta, 3413 finalSize, initSize + delta); 3414 } 3415 3416 @Test 3417 public void testDeleteColumns_PostInsert() throws IOException, InterruptedException { 3418 Delete delete = new Delete(row); 3419 delete.addColumns(fam1, qual1); 3420 doTestDelete_AndPostInsert(delete); 3421 } 3422 3423 @Test 3424 public void testaddFamily_PostInsert() throws IOException, InterruptedException { 3425 Delete delete = new Delete(row); 3426 delete.addFamily(fam1); 3427 doTestDelete_AndPostInsert(delete); 3428 } 3429 3430 public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException { 3431 this.region = initHRegion(tableName, method, CONF, fam1); 3432 EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge()); 3433 Put put = new Put(row); 3434 put.addColumn(fam1, qual1, value1); 3435 region.put(put); 3436 3437 // now delete the value: 3438 region.delete(delete); 3439 3440 // ok put data: 3441 put = new Put(row); 3442 put.addColumn(fam1, qual1, value2); 3443 region.put(put); 3444 3445 // ok get: 3446 Get get = new Get(row); 3447 get.addColumn(fam1, qual1); 3448 3449 Result r = region.get(get); 3450 assertEquals(1, r.size()); 3451 assertArrayEquals(value2, r.getValue(fam1, qual1)); 3452 3453 // next: 3454 Scan scan = new Scan().withStartRow(row); 3455 scan.addColumn(fam1, qual1); 3456 InternalScanner s = region.getScanner(scan); 3457 3458 List<Cell> results = new ArrayList<>(); 3459 assertEquals(false, s.next(results)); 3460 assertEquals(1, results.size()); 3461 Cell kv = results.get(0); 3462 3463 assertArrayEquals(value2, CellUtil.cloneValue(kv)); 3464 assertArrayEquals(fam1, CellUtil.cloneFamily(kv)); 3465 assertArrayEquals(qual1, CellUtil.cloneQualifier(kv)); 3466 assertArrayEquals(row, CellUtil.cloneRow(kv)); 3467 } 3468 3469 @Test 3470 public void testDelete_CheckTimestampUpdated() throws IOException { 3471 byte[] row1 = Bytes.toBytes("row1"); 3472 byte[] col1 = Bytes.toBytes("col1"); 3473 byte[] col2 = Bytes.toBytes("col2"); 3474 byte[] col3 = Bytes.toBytes("col3"); 3475 3476 byte[] forUnitTestsOnly = Bytes.toBytes("ForUnitTestsOnly"); 3477 3478 // Setting up region 3479 this.region = initHRegion(tableName, method, CONF, fam1); 3480 // Building checkerList 3481 List<Cell> kvs = new ArrayList<>(); 3482 kvs.add(new KeyValue(row1, fam1, col1, null)); 3483 kvs.add(new KeyValue(row1, fam1, col2, null)); 3484 kvs.add(new KeyValue(row1, fam1, col3, null)); 3485 3486 NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<>(Bytes.BYTES_COMPARATOR); 3487 deleteMap.put(fam1, kvs); 3488 region.delete(new Delete(forUnitTestsOnly, HConstants.LATEST_TIMESTAMP, deleteMap)); 3489 3490 // extract the key values out the memstore: 3491 // This is kinda hacky, but better than nothing... 3492 long now = EnvironmentEdgeManager.currentTime(); 3493 AbstractMemStore memstore = (AbstractMemStore) region.getStore(fam1).memstore; 3494 Cell firstCell = memstore.getActive().first(); 3495 assertTrue(firstCell.getTimestamp() <= now); 3496 now = firstCell.getTimestamp(); 3497 for (Cell cell : memstore.getActive().getCellSet()) { 3498 assertTrue(cell.getTimestamp() <= now); 3499 now = cell.getTimestamp(); 3500 } 3501 } 3502 3503 // //////////////////////////////////////////////////////////////////////////// 3504 // Get tests 3505 // //////////////////////////////////////////////////////////////////////////// 3506 @Test 3507 public void testGet_FamilyChecker() throws IOException { 3508 byte[] row1 = Bytes.toBytes("row1"); 3509 byte[] fam1 = Bytes.toBytes("fam1"); 3510 byte[] fam2 = Bytes.toBytes("False"); 3511 byte[] col1 = Bytes.toBytes("col1"); 3512 3513 // Setting up region 3514 this.region = initHRegion(tableName, method, CONF, fam1); 3515 Get get = new Get(row1); 3516 get.addColumn(fam2, col1); 3517 3518 // Test 3519 try { 3520 region.get(get); 3521 fail("Expecting DoNotRetryIOException in get but did not get any"); 3522 } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) { 3523 LOG.info("Got expected DoNotRetryIOException successfully"); 3524 } 3525 } 3526 3527 @Test 3528 public void testGet_Basic() throws IOException { 3529 byte[] row1 = Bytes.toBytes("row1"); 3530 byte[] fam1 = Bytes.toBytes("fam1"); 3531 byte[] col1 = Bytes.toBytes("col1"); 3532 byte[] col2 = Bytes.toBytes("col2"); 3533 byte[] col3 = Bytes.toBytes("col3"); 3534 byte[] col4 = Bytes.toBytes("col4"); 3535 byte[] col5 = Bytes.toBytes("col5"); 3536 3537 // Setting up region 3538 this.region = initHRegion(tableName, method, CONF, fam1); 3539 // Add to memstore 3540 Put put = new Put(row1); 3541 put.addColumn(fam1, col1, null); 3542 put.addColumn(fam1, col2, null); 3543 put.addColumn(fam1, col3, null); 3544 put.addColumn(fam1, col4, null); 3545 put.addColumn(fam1, col5, null); 3546 region.put(put); 3547 3548 Get get = new Get(row1); 3549 get.addColumn(fam1, col2); 3550 get.addColumn(fam1, col4); 3551 // Expected result 3552 KeyValue kv1 = new KeyValue(row1, fam1, col2); 3553 KeyValue kv2 = new KeyValue(row1, fam1, col4); 3554 KeyValue[] expected = { kv1, kv2 }; 3555 3556 // Test 3557 Result res = region.get(get); 3558 assertEquals(expected.length, res.size()); 3559 for (int i = 0; i < res.size(); i++) { 3560 assertTrue(CellUtil.matchingRows(expected[i], res.rawCells()[i])); 3561 assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i])); 3562 assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i])); 3563 } 3564 3565 // Test using a filter on a Get 3566 Get g = new Get(row1); 3567 final int count = 2; 3568 g.setFilter(new ColumnCountGetFilter(count)); 3569 res = region.get(g); 3570 assertEquals(count, res.size()); 3571 } 3572 3573 @Test 3574 public void testGet_Empty() throws IOException { 3575 byte[] row = Bytes.toBytes("row"); 3576 byte[] fam = Bytes.toBytes("fam"); 3577 3578 this.region = initHRegion(tableName, method, CONF, fam); 3579 Get get = new Get(row); 3580 get.addFamily(fam); 3581 Result r = region.get(get); 3582 3583 assertTrue(r.isEmpty()); 3584 } 3585 3586 @Test 3587 public void testGetWithFilter() throws IOException, InterruptedException { 3588 byte[] row1 = Bytes.toBytes("row1"); 3589 byte[] fam1 = Bytes.toBytes("fam1"); 3590 byte[] col1 = Bytes.toBytes("col1"); 3591 byte[] value1 = Bytes.toBytes("value1"); 3592 byte[] value2 = Bytes.toBytes("value2"); 3593 3594 final int maxVersions = 3; 3595 TableDescriptor tableDescriptor = 3596 TableDescriptorBuilder.newBuilder(TableName.valueOf("testFilterAndColumnTracker")) 3597 .setColumnFamily( 3598 ColumnFamilyDescriptorBuilder.newBuilder(fam1).setMaxVersions(maxVersions).build()) 3599 .build(); 3600 ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null, 3601 MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT); 3602 RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 3603 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + ".log"); 3604 final WAL wal = HBaseTestingUtil.createWal(TEST_UTIL.getConfiguration(), logDir, info); 3605 this.region = TEST_UTIL.createLocalHRegion(info, CONF, tableDescriptor, wal); 3606 3607 // Put 4 version to memstore 3608 long ts = 0; 3609 Put put = new Put(row1, ts); 3610 put.addColumn(fam1, col1, value1); 3611 region.put(put); 3612 put = new Put(row1, ts + 1); 3613 put.addColumn(fam1, col1, Bytes.toBytes("filter1")); 3614 region.put(put); 3615 put = new Put(row1, ts + 2); 3616 put.addColumn(fam1, col1, Bytes.toBytes("filter2")); 3617 region.put(put); 3618 put = new Put(row1, ts + 3); 3619 put.addColumn(fam1, col1, value2); 3620 region.put(put); 3621 3622 Get get = new Get(row1); 3623 get.readAllVersions(); 3624 Result res = region.get(get); 3625 // Get 3 versions, the oldest version has gone from user view 3626 assertEquals(maxVersions, res.size()); 3627 3628 get.setFilter(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("value"))); 3629 res = region.get(get); 3630 // When use value filter, the oldest version should still gone from user view and it 3631 // should only return one key vaule 3632 assertEquals(1, res.size()); 3633 assertTrue(CellUtil.matchingValue(new KeyValue(row1, fam1, col1, value2), res.rawCells()[0])); 3634 assertEquals(ts + 3, res.rawCells()[0].getTimestamp()); 3635 3636 region.flush(true); 3637 region.compact(true); 3638 Thread.sleep(1000); 3639 res = region.get(get); 3640 // After flush and compact, the result should be consistent with previous result 3641 assertEquals(1, res.size()); 3642 assertTrue(CellUtil.matchingValue(new KeyValue(row1, fam1, col1, value2), res.rawCells()[0])); 3643 } 3644 3645 // //////////////////////////////////////////////////////////////////////////// 3646 // Scanner tests 3647 // //////////////////////////////////////////////////////////////////////////// 3648 @Test 3649 public void testGetScanner_WithOkFamilies() throws IOException { 3650 byte[] fam1 = Bytes.toBytes("fam1"); 3651 byte[] fam2 = Bytes.toBytes("fam2"); 3652 3653 byte[][] families = { fam1, fam2 }; 3654 3655 // Setting up region 3656 this.region = initHRegion(tableName, method, CONF, families); 3657 Scan scan = new Scan(); 3658 scan.addFamily(fam1); 3659 scan.addFamily(fam2); 3660 try { 3661 region.getScanner(scan); 3662 } catch (Exception e) { 3663 assertTrue("Families could not be found in Region", false); 3664 } 3665 } 3666 3667 @Test 3668 public void testGetScanner_WithNotOkFamilies() throws IOException { 3669 byte[] fam1 = Bytes.toBytes("fam1"); 3670 byte[] fam2 = Bytes.toBytes("fam2"); 3671 3672 byte[][] families = { fam1 }; 3673 3674 // Setting up region 3675 this.region = initHRegion(tableName, method, CONF, families); 3676 Scan scan = new Scan(); 3677 scan.addFamily(fam2); 3678 boolean ok = false; 3679 try { 3680 region.getScanner(scan); 3681 } catch (Exception e) { 3682 ok = true; 3683 } 3684 assertTrue("Families could not be found in Region", ok); 3685 } 3686 3687 @Test 3688 public void testGetScanner_WithNoFamilies() throws IOException { 3689 byte[] row1 = Bytes.toBytes("row1"); 3690 byte[] fam1 = Bytes.toBytes("fam1"); 3691 byte[] fam2 = Bytes.toBytes("fam2"); 3692 byte[] fam3 = Bytes.toBytes("fam3"); 3693 byte[] fam4 = Bytes.toBytes("fam4"); 3694 3695 byte[][] families = { fam1, fam2, fam3, fam4 }; 3696 3697 // Setting up region 3698 this.region = initHRegion(tableName, method, CONF, families); 3699 // Putting data in Region 3700 Put put = new Put(row1); 3701 put.addColumn(fam1, null, null); 3702 put.addColumn(fam2, null, null); 3703 put.addColumn(fam3, null, null); 3704 put.addColumn(fam4, null, null); 3705 region.put(put); 3706 3707 Scan scan = null; 3708 RegionScannerImpl is = null; 3709 3710 // Testing to see how many scanners that is produced by getScanner, 3711 // starting 3712 // with known number, 2 - current = 1 3713 scan = new Scan(); 3714 scan.addFamily(fam2); 3715 scan.addFamily(fam4); 3716 is = region.getScanner(scan); 3717 assertEquals(1, is.storeHeap.getHeap().size()); 3718 3719 scan = new Scan(); 3720 is = region.getScanner(scan); 3721 assertEquals(families.length - 1, is.storeHeap.getHeap().size()); 3722 } 3723 3724 /** 3725 * This method tests https://issues.apache.org/jira/browse/HBASE-2516. 3726 */ 3727 @Test 3728 public void testGetScanner_WithRegionClosed() throws IOException { 3729 byte[] fam1 = Bytes.toBytes("fam1"); 3730 byte[] fam2 = Bytes.toBytes("fam2"); 3731 3732 byte[][] families = { fam1, fam2 }; 3733 3734 // Setting up region 3735 try { 3736 this.region = initHRegion(tableName, method, CONF, families); 3737 } catch (IOException e) { 3738 e.printStackTrace(); 3739 fail("Got IOException during initHRegion, " + e.getMessage()); 3740 } 3741 region.closed.set(true); 3742 try { 3743 region.getScanner(null); 3744 fail("Expected to get an exception during getScanner on a region that is closed"); 3745 } catch (NotServingRegionException e) { 3746 // this is the correct exception that is expected 3747 } catch (IOException e) { 3748 fail("Got wrong type of exception - should be a NotServingRegionException, " 3749 + "but was an IOException: " + e.getMessage()); 3750 } 3751 } 3752 3753 @Test 3754 public void testRegionScanner_Next() throws IOException { 3755 byte[] row1 = Bytes.toBytes("row1"); 3756 byte[] row2 = Bytes.toBytes("row2"); 3757 byte[] fam1 = Bytes.toBytes("fam1"); 3758 byte[] fam2 = Bytes.toBytes("fam2"); 3759 byte[] fam3 = Bytes.toBytes("fam3"); 3760 byte[] fam4 = Bytes.toBytes("fam4"); 3761 3762 byte[][] families = { fam1, fam2, fam3, fam4 }; 3763 long ts = EnvironmentEdgeManager.currentTime(); 3764 3765 // Setting up region 3766 this.region = initHRegion(tableName, method, CONF, families); 3767 // Putting data in Region 3768 Put put = null; 3769 put = new Put(row1); 3770 put.addColumn(fam1, (byte[]) null, ts, null); 3771 put.addColumn(fam2, (byte[]) null, ts, null); 3772 put.addColumn(fam3, (byte[]) null, ts, null); 3773 put.addColumn(fam4, (byte[]) null, ts, null); 3774 region.put(put); 3775 3776 put = new Put(row2); 3777 put.addColumn(fam1, (byte[]) null, ts, null); 3778 put.addColumn(fam2, (byte[]) null, ts, null); 3779 put.addColumn(fam3, (byte[]) null, ts, null); 3780 put.addColumn(fam4, (byte[]) null, ts, null); 3781 region.put(put); 3782 3783 Scan scan = new Scan(); 3784 scan.addFamily(fam2); 3785 scan.addFamily(fam4); 3786 InternalScanner is = region.getScanner(scan); 3787 3788 List<Cell> res = null; 3789 3790 // Result 1 3791 List<Cell> expected1 = new ArrayList<>(); 3792 expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null)); 3793 expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null)); 3794 3795 res = new ArrayList<>(); 3796 is.next(res); 3797 for (int i = 0; i < res.size(); i++) { 3798 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected1.get(i), res.get(i))); 3799 } 3800 3801 // Result 2 3802 List<Cell> expected2 = new ArrayList<>(); 3803 expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null)); 3804 expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null)); 3805 3806 res = new ArrayList<>(); 3807 is.next(res); 3808 for (int i = 0; i < res.size(); i++) { 3809 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected2.get(i), res.get(i))); 3810 } 3811 } 3812 3813 @Test 3814 public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException { 3815 byte[] row1 = Bytes.toBytes("row1"); 3816 byte[] qf1 = Bytes.toBytes("qualifier1"); 3817 byte[] qf2 = Bytes.toBytes("qualifier2"); 3818 byte[] fam1 = Bytes.toBytes("fam1"); 3819 byte[][] families = { fam1 }; 3820 3821 long ts1 = EnvironmentEdgeManager.currentTime(); 3822 long ts2 = ts1 + 1; 3823 long ts3 = ts1 + 2; 3824 3825 // Setting up region 3826 this.region = initHRegion(tableName, method, CONF, families); 3827 // Putting data in Region 3828 Put put = null; 3829 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 3830 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 3831 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 3832 3833 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 3834 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 3835 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 3836 3837 put = new Put(row1); 3838 put.add(kv13); 3839 put.add(kv12); 3840 put.add(kv11); 3841 put.add(kv23); 3842 put.add(kv22); 3843 put.add(kv21); 3844 region.put(put); 3845 3846 // Expected 3847 List<Cell> expected = new ArrayList<>(); 3848 expected.add(kv13); 3849 expected.add(kv12); 3850 3851 Scan scan = new Scan().withStartRow(row1); 3852 scan.addColumn(fam1, qf1); 3853 scan.readVersions(MAX_VERSIONS); 3854 List<Cell> actual = new ArrayList<>(); 3855 InternalScanner scanner = region.getScanner(scan); 3856 3857 boolean hasNext = scanner.next(actual); 3858 assertEquals(false, hasNext); 3859 3860 // Verify result 3861 for (int i = 0; i < expected.size(); i++) { 3862 assertEquals(expected.get(i), actual.get(i)); 3863 } 3864 } 3865 3866 @Test 3867 public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException { 3868 byte[] row1 = Bytes.toBytes("row1"); 3869 byte[] qf1 = Bytes.toBytes("qualifier1"); 3870 byte[] qf2 = Bytes.toBytes("qualifier2"); 3871 byte[] fam1 = Bytes.toBytes("fam1"); 3872 byte[][] families = { fam1 }; 3873 3874 long ts1 = 1; 3875 long ts2 = ts1 + 1; 3876 long ts3 = ts1 + 2; 3877 3878 // Setting up region 3879 this.region = initHRegion(tableName, method, CONF, families); 3880 // Putting data in Region 3881 Put put = null; 3882 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 3883 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 3884 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 3885 3886 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 3887 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 3888 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 3889 3890 put = new Put(row1); 3891 put.add(kv13); 3892 put.add(kv12); 3893 put.add(kv11); 3894 put.add(kv23); 3895 put.add(kv22); 3896 put.add(kv21); 3897 region.put(put); 3898 region.flush(true); 3899 3900 // Expected 3901 List<Cell> expected = new ArrayList<>(); 3902 expected.add(kv13); 3903 expected.add(kv12); 3904 expected.add(kv23); 3905 expected.add(kv22); 3906 3907 Scan scan = new Scan().withStartRow(row1); 3908 scan.addColumn(fam1, qf1); 3909 scan.addColumn(fam1, qf2); 3910 scan.readVersions(MAX_VERSIONS); 3911 List<Cell> actual = new ArrayList<>(); 3912 InternalScanner scanner = region.getScanner(scan); 3913 3914 boolean hasNext = scanner.next(actual); 3915 assertEquals(false, hasNext); 3916 3917 // Verify result 3918 for (int i = 0; i < expected.size(); i++) { 3919 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); 3920 } 3921 } 3922 3923 @Test 3924 public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions() 3925 throws IOException { 3926 byte[] row1 = Bytes.toBytes("row1"); 3927 byte[] fam1 = Bytes.toBytes("fam1"); 3928 byte[][] families = { fam1 }; 3929 byte[] qf1 = Bytes.toBytes("qualifier1"); 3930 byte[] qf2 = Bytes.toBytes("qualifier2"); 3931 3932 long ts1 = 1; 3933 long ts2 = ts1 + 1; 3934 long ts3 = ts1 + 2; 3935 long ts4 = ts1 + 3; 3936 3937 // Setting up region 3938 this.region = initHRegion(tableName, method, CONF, families); 3939 // Putting data in Region 3940 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null); 3941 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 3942 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 3943 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 3944 3945 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null); 3946 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 3947 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 3948 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 3949 3950 Put put = null; 3951 put = new Put(row1); 3952 put.add(kv14); 3953 put.add(kv24); 3954 region.put(put); 3955 region.flush(true); 3956 3957 put = new Put(row1); 3958 put.add(kv23); 3959 put.add(kv13); 3960 region.put(put); 3961 region.flush(true); 3962 3963 put = new Put(row1); 3964 put.add(kv22); 3965 put.add(kv12); 3966 region.put(put); 3967 region.flush(true); 3968 3969 put = new Put(row1); 3970 put.add(kv21); 3971 put.add(kv11); 3972 region.put(put); 3973 3974 // Expected 3975 List<Cell> expected = new ArrayList<>(); 3976 expected.add(kv14); 3977 expected.add(kv13); 3978 expected.add(kv12); 3979 expected.add(kv24); 3980 expected.add(kv23); 3981 expected.add(kv22); 3982 3983 Scan scan = new Scan().withStartRow(row1); 3984 scan.addColumn(fam1, qf1); 3985 scan.addColumn(fam1, qf2); 3986 int versions = 3; 3987 scan.readVersions(versions); 3988 List<Cell> actual = new ArrayList<>(); 3989 InternalScanner scanner = region.getScanner(scan); 3990 3991 boolean hasNext = scanner.next(actual); 3992 assertEquals(false, hasNext); 3993 3994 // Verify result 3995 for (int i = 0; i < expected.size(); i++) { 3996 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); 3997 } 3998 } 3999 4000 @Test 4001 public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException { 4002 byte[] row1 = Bytes.toBytes("row1"); 4003 byte[] qf1 = Bytes.toBytes("qualifier1"); 4004 byte[] qf2 = Bytes.toBytes("qualifier2"); 4005 byte[] fam1 = Bytes.toBytes("fam1"); 4006 byte[][] families = { fam1 }; 4007 4008 long ts1 = EnvironmentEdgeManager.currentTime(); 4009 long ts2 = ts1 + 1; 4010 long ts3 = ts1 + 2; 4011 4012 // Setting up region 4013 this.region = initHRegion(tableName, method, CONF, families); 4014 // Putting data in Region 4015 Put put = null; 4016 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 4017 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 4018 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 4019 4020 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 4021 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 4022 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 4023 4024 put = new Put(row1); 4025 put.add(kv13); 4026 put.add(kv12); 4027 put.add(kv11); 4028 put.add(kv23); 4029 put.add(kv22); 4030 put.add(kv21); 4031 region.put(put); 4032 4033 // Expected 4034 List<Cell> expected = new ArrayList<>(); 4035 expected.add(kv13); 4036 expected.add(kv12); 4037 expected.add(kv23); 4038 expected.add(kv22); 4039 4040 Scan scan = new Scan().withStartRow(row1); 4041 scan.addFamily(fam1); 4042 scan.readVersions(MAX_VERSIONS); 4043 List<Cell> actual = new ArrayList<>(); 4044 InternalScanner scanner = region.getScanner(scan); 4045 4046 boolean hasNext = scanner.next(actual); 4047 assertEquals(false, hasNext); 4048 4049 // Verify result 4050 for (int i = 0; i < expected.size(); i++) { 4051 assertEquals(expected.get(i), actual.get(i)); 4052 } 4053 } 4054 4055 @Test 4056 public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException { 4057 byte[] row1 = Bytes.toBytes("row1"); 4058 byte[] qf1 = Bytes.toBytes("qualifier1"); 4059 byte[] qf2 = Bytes.toBytes("qualifier2"); 4060 byte[] fam1 = Bytes.toBytes("fam1"); 4061 4062 long ts1 = 1; 4063 long ts2 = ts1 + 1; 4064 long ts3 = ts1 + 2; 4065 4066 // Setting up region 4067 this.region = initHRegion(tableName, method, CONF, fam1); 4068 // Putting data in Region 4069 Put put = null; 4070 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 4071 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 4072 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 4073 4074 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 4075 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 4076 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 4077 4078 put = new Put(row1); 4079 put.add(kv13); 4080 put.add(kv12); 4081 put.add(kv11); 4082 put.add(kv23); 4083 put.add(kv22); 4084 put.add(kv21); 4085 region.put(put); 4086 region.flush(true); 4087 4088 // Expected 4089 List<Cell> expected = new ArrayList<>(); 4090 expected.add(kv13); 4091 expected.add(kv12); 4092 expected.add(kv23); 4093 expected.add(kv22); 4094 4095 Scan scan = new Scan().withStartRow(row1); 4096 scan.addFamily(fam1); 4097 scan.readVersions(MAX_VERSIONS); 4098 List<Cell> actual = new ArrayList<>(); 4099 InternalScanner scanner = region.getScanner(scan); 4100 4101 boolean hasNext = scanner.next(actual); 4102 assertEquals(false, hasNext); 4103 4104 // Verify result 4105 for (int i = 0; i < expected.size(); i++) { 4106 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); 4107 } 4108 } 4109 4110 @Test 4111 public void testScanner_StopRow1542() throws IOException { 4112 byte[] family = Bytes.toBytes("testFamily"); 4113 this.region = initHRegion(tableName, method, CONF, family); 4114 byte[] row1 = Bytes.toBytes("row111"); 4115 byte[] row2 = Bytes.toBytes("row222"); 4116 byte[] row3 = Bytes.toBytes("row333"); 4117 byte[] row4 = Bytes.toBytes("row444"); 4118 byte[] row5 = Bytes.toBytes("row555"); 4119 4120 byte[] col1 = Bytes.toBytes("Pub111"); 4121 byte[] col2 = Bytes.toBytes("Pub222"); 4122 4123 Put put = new Put(row1); 4124 put.addColumn(family, col1, Bytes.toBytes(10L)); 4125 region.put(put); 4126 4127 put = new Put(row2); 4128 put.addColumn(family, col1, Bytes.toBytes(15L)); 4129 region.put(put); 4130 4131 put = new Put(row3); 4132 put.addColumn(family, col2, Bytes.toBytes(20L)); 4133 region.put(put); 4134 4135 put = new Put(row4); 4136 put.addColumn(family, col2, Bytes.toBytes(30L)); 4137 region.put(put); 4138 4139 put = new Put(row5); 4140 put.addColumn(family, col1, Bytes.toBytes(40L)); 4141 region.put(put); 4142 4143 Scan scan = new Scan().withStartRow(row3).withStopRow(row4); 4144 scan.readAllVersions(); 4145 scan.addColumn(family, col1); 4146 InternalScanner s = region.getScanner(scan); 4147 4148 List<Cell> results = new ArrayList<>(); 4149 assertEquals(false, s.next(results)); 4150 assertEquals(0, results.size()); 4151 } 4152 4153 @Test 4154 public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException { 4155 byte[] row1 = Bytes.toBytes("row1"); 4156 byte[] fam1 = Bytes.toBytes("fam1"); 4157 byte[] qf1 = Bytes.toBytes("qualifier1"); 4158 byte[] qf2 = Bytes.toBytes("quateslifier2"); 4159 4160 long ts1 = 1; 4161 long ts2 = ts1 + 1; 4162 long ts3 = ts1 + 2; 4163 long ts4 = ts1 + 3; 4164 4165 // Setting up region 4166 this.region = initHRegion(tableName, method, CONF, fam1); 4167 // Putting data in Region 4168 KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null); 4169 KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null); 4170 KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null); 4171 KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null); 4172 4173 KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null); 4174 KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null); 4175 KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null); 4176 KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null); 4177 4178 Put put = null; 4179 put = new Put(row1); 4180 put.add(kv14); 4181 put.add(kv24); 4182 region.put(put); 4183 region.flush(true); 4184 4185 put = new Put(row1); 4186 put.add(kv23); 4187 put.add(kv13); 4188 region.put(put); 4189 region.flush(true); 4190 4191 put = new Put(row1); 4192 put.add(kv22); 4193 put.add(kv12); 4194 region.put(put); 4195 region.flush(true); 4196 4197 put = new Put(row1); 4198 put.add(kv21); 4199 put.add(kv11); 4200 region.put(put); 4201 4202 // Expected 4203 List<KeyValue> expected = new ArrayList<>(); 4204 expected.add(kv14); 4205 expected.add(kv13); 4206 expected.add(kv12); 4207 expected.add(kv24); 4208 expected.add(kv23); 4209 expected.add(kv22); 4210 4211 Scan scan = new Scan().withStartRow(row1); 4212 int versions = 3; 4213 scan.readVersions(versions); 4214 List<Cell> actual = new ArrayList<>(); 4215 InternalScanner scanner = region.getScanner(scan); 4216 4217 boolean hasNext = scanner.next(actual); 4218 assertEquals(false, hasNext); 4219 4220 // Verify result 4221 for (int i = 0; i < expected.size(); i++) { 4222 assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); 4223 } 4224 } 4225 4226 /** 4227 * Added for HBASE-5416 Here we test scan optimization when only subset of CFs are used in filter 4228 * conditions. 4229 */ 4230 @Test 4231 public void testScanner_JoinedScanners() throws IOException { 4232 byte[] cf_essential = Bytes.toBytes("essential"); 4233 byte[] cf_joined = Bytes.toBytes("joined"); 4234 byte[] cf_alpha = Bytes.toBytes("alpha"); 4235 this.region = initHRegion(tableName, method, CONF, cf_essential, cf_joined, cf_alpha); 4236 byte[] row1 = Bytes.toBytes("row1"); 4237 byte[] row2 = Bytes.toBytes("row2"); 4238 byte[] row3 = Bytes.toBytes("row3"); 4239 4240 byte[] col_normal = Bytes.toBytes("d"); 4241 byte[] col_alpha = Bytes.toBytes("a"); 4242 4243 byte[] filtered_val = Bytes.toBytes(3); 4244 4245 Put put = new Put(row1); 4246 put.addColumn(cf_essential, col_normal, Bytes.toBytes(1)); 4247 put.addColumn(cf_joined, col_alpha, Bytes.toBytes(1)); 4248 region.put(put); 4249 4250 put = new Put(row2); 4251 put.addColumn(cf_essential, col_alpha, Bytes.toBytes(2)); 4252 put.addColumn(cf_joined, col_normal, Bytes.toBytes(2)); 4253 put.addColumn(cf_alpha, col_alpha, Bytes.toBytes(2)); 4254 region.put(put); 4255 4256 put = new Put(row3); 4257 put.addColumn(cf_essential, col_normal, filtered_val); 4258 put.addColumn(cf_joined, col_normal, filtered_val); 4259 region.put(put); 4260 4261 // Check two things: 4262 // 1. result list contains expected values 4263 // 2. result list is sorted properly 4264 4265 Scan scan = new Scan(); 4266 Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal, 4267 CompareOperator.NOT_EQUAL, filtered_val); 4268 scan.setFilter(filter); 4269 scan.setLoadColumnFamiliesOnDemand(true); 4270 InternalScanner s = region.getScanner(scan); 4271 4272 List<Cell> results = new ArrayList<>(); 4273 assertTrue(s.next(results)); 4274 assertEquals(1, results.size()); 4275 results.clear(); 4276 4277 assertTrue(s.next(results)); 4278 assertEquals(3, results.size()); 4279 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha)); 4280 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential)); 4281 assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined)); 4282 results.clear(); 4283 4284 assertFalse(s.next(results)); 4285 assertEquals(0, results.size()); 4286 } 4287 4288 /** 4289 * HBASE-5416 Test case when scan limits amount of KVs returned on each next() call. 4290 */ 4291 @Test 4292 public void testScanner_JoinedScannersWithLimits() throws IOException { 4293 final byte[] cf_first = Bytes.toBytes("first"); 4294 final byte[] cf_second = Bytes.toBytes("second"); 4295 4296 this.region = initHRegion(tableName, method, CONF, cf_first, cf_second); 4297 final byte[] col_a = Bytes.toBytes("a"); 4298 final byte[] col_b = Bytes.toBytes("b"); 4299 4300 Put put; 4301 4302 for (int i = 0; i < 10; i++) { 4303 put = new Put(Bytes.toBytes("r" + Integer.toString(i))); 4304 put.addColumn(cf_first, col_a, Bytes.toBytes(i)); 4305 if (i < 5) { 4306 put.addColumn(cf_first, col_b, Bytes.toBytes(i)); 4307 put.addColumn(cf_second, col_a, Bytes.toBytes(i)); 4308 put.addColumn(cf_second, col_b, Bytes.toBytes(i)); 4309 } 4310 region.put(put); 4311 } 4312 4313 Scan scan = new Scan(); 4314 scan.setLoadColumnFamiliesOnDemand(true); 4315 Filter bogusFilter = new FilterBase() { 4316 @Override 4317 public ReturnCode filterCell(final Cell ignored) throws IOException { 4318 return ReturnCode.INCLUDE; 4319 } 4320 4321 @Override 4322 public boolean isFamilyEssential(byte[] name) { 4323 return Bytes.equals(name, cf_first); 4324 } 4325 }; 4326 4327 scan.setFilter(bogusFilter); 4328 InternalScanner s = region.getScanner(scan); 4329 4330 // Our data looks like this: 4331 // r0: first:a, first:b, second:a, second:b 4332 // r1: first:a, first:b, second:a, second:b 4333 // r2: first:a, first:b, second:a, second:b 4334 // r3: first:a, first:b, second:a, second:b 4335 // r4: first:a, first:b, second:a, second:b 4336 // r5: first:a 4337 // r6: first:a 4338 // r7: first:a 4339 // r8: first:a 4340 // r9: first:a 4341 4342 // But due to next's limit set to 3, we should get this: 4343 // r0: first:a, first:b, second:a 4344 // r0: second:b 4345 // r1: first:a, first:b, second:a 4346 // r1: second:b 4347 // r2: first:a, first:b, second:a 4348 // r2: second:b 4349 // r3: first:a, first:b, second:a 4350 // r3: second:b 4351 // r4: first:a, first:b, second:a 4352 // r4: second:b 4353 // r5: first:a 4354 // r6: first:a 4355 // r7: first:a 4356 // r8: first:a 4357 // r9: first:a 4358 4359 List<Cell> results = new ArrayList<>(); 4360 int index = 0; 4361 ScannerContext scannerContext = ScannerContext.newBuilder().setBatchLimit(3).build(); 4362 while (true) { 4363 boolean more = s.next(results, scannerContext); 4364 if ((index >> 1) < 5) { 4365 if (index % 2 == 0) { 4366 assertEquals(3, results.size()); 4367 } else { 4368 assertEquals(1, results.size()); 4369 } 4370 } else { 4371 assertEquals(1, results.size()); 4372 } 4373 results.clear(); 4374 index++; 4375 if (!more) { 4376 break; 4377 } 4378 } 4379 } 4380 4381 @Test 4382 public void testScannerOperationId() throws IOException { 4383 region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES); 4384 Scan scan = new Scan(); 4385 RegionScanner scanner = region.getScanner(scan); 4386 assertNull(scanner.getOperationId()); 4387 scanner.close(); 4388 4389 String operationId = "test_operation_id_0101"; 4390 scan = new Scan().setId(operationId); 4391 scanner = region.getScanner(scan); 4392 assertEquals(operationId, scanner.getOperationId()); 4393 scanner.close(); 4394 4395 HBaseTestingUtil.closeRegionAndWAL(this.region); 4396 } 4397 4398 /** 4399 * Write an HFile block full with Cells whose qualifier that are identical between 0 and 4400 * Short.MAX_VALUE. See HBASE-13329. 4401 */ 4402 @Test 4403 public void testLongQualifier() throws Exception { 4404 byte[] family = Bytes.toBytes("family"); 4405 this.region = initHRegion(tableName, method, CONF, family); 4406 byte[] q = new byte[Short.MAX_VALUE + 2]; 4407 Arrays.fill(q, 0, q.length - 1, (byte) 42); 4408 for (byte i = 0; i < 10; i++) { 4409 Put p = new Put(Bytes.toBytes("row")); 4410 // qualifiers that differ past Short.MAX_VALUE 4411 q[q.length - 1] = i; 4412 p.addColumn(family, q, q); 4413 region.put(p); 4414 } 4415 region.flush(false); 4416 } 4417 4418 /** 4419 * Flushes the cache in a thread while scanning. The tests verify that the scan is coherent - e.g. 4420 * the returned results are always of the same or later update as the previous results. scan / 4421 * compact thread join 4422 */ 4423 @Test 4424 public void testFlushCacheWhileScanning() throws IOException, InterruptedException { 4425 byte[] family = Bytes.toBytes("family"); 4426 int numRows = 1000; 4427 int flushAndScanInterval = 10; 4428 int compactInterval = 10 * flushAndScanInterval; 4429 4430 this.region = initHRegion(tableName, method, CONF, family); 4431 FlushThread flushThread = new FlushThread(); 4432 try { 4433 flushThread.start(); 4434 4435 Scan scan = new Scan(); 4436 scan.addFamily(family); 4437 scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOperator.EQUAL, 4438 new BinaryComparator(Bytes.toBytes(5L)))); 4439 4440 int expectedCount = 0; 4441 List<Cell> res = new ArrayList<>(); 4442 4443 boolean toggle = true; 4444 for (long i = 0; i < numRows; i++) { 4445 Put put = new Put(Bytes.toBytes(i)); 4446 put.setDurability(Durability.SKIP_WAL); 4447 put.addColumn(family, qual1, Bytes.toBytes(i % 10)); 4448 region.put(put); 4449 4450 if (i != 0 && i % compactInterval == 0) { 4451 LOG.debug("iteration = " + i + " ts=" + EnvironmentEdgeManager.currentTime()); 4452 region.compact(true); 4453 } 4454 4455 if (i % 10 == 5L) { 4456 expectedCount++; 4457 } 4458 4459 if (i != 0 && i % flushAndScanInterval == 0) { 4460 res.clear(); 4461 InternalScanner scanner = region.getScanner(scan); 4462 if (toggle) { 4463 flushThread.flush(); 4464 } 4465 while (scanner.next(res)) 4466 ; 4467 if (!toggle) { 4468 flushThread.flush(); 4469 } 4470 assertEquals( 4471 "toggle=" + toggle + "i=" + i + " ts=" + EnvironmentEdgeManager.currentTime(), 4472 expectedCount, res.size()); 4473 toggle = !toggle; 4474 } 4475 } 4476 4477 } finally { 4478 try { 4479 flushThread.done(); 4480 flushThread.join(); 4481 flushThread.checkNoError(); 4482 } catch (InterruptedException ie) { 4483 LOG.warn("Caught exception when joining with flushThread", ie); 4484 } 4485 HBaseTestingUtil.closeRegionAndWAL(this.region); 4486 this.region = null; 4487 } 4488 } 4489 4490 protected class FlushThread extends Thread { 4491 private volatile boolean done; 4492 private Throwable error = null; 4493 4494 FlushThread() { 4495 super("FlushThread"); 4496 } 4497 4498 public void done() { 4499 done = true; 4500 synchronized (this) { 4501 interrupt(); 4502 } 4503 } 4504 4505 public void checkNoError() { 4506 if (error != null) { 4507 assertNull(error); 4508 } 4509 } 4510 4511 @Override 4512 public void run() { 4513 done = false; 4514 while (!done) { 4515 synchronized (this) { 4516 try { 4517 wait(); 4518 } catch (InterruptedException ignored) { 4519 if (done) { 4520 break; 4521 } 4522 } 4523 } 4524 try { 4525 region.flush(true); 4526 } catch (IOException e) { 4527 if (!done) { 4528 LOG.error("Error while flushing cache", e); 4529 error = e; 4530 } 4531 break; 4532 } catch (Throwable t) { 4533 LOG.error("Uncaught exception", t); 4534 throw t; 4535 } 4536 } 4537 } 4538 4539 public void flush() { 4540 synchronized (this) { 4541 notify(); 4542 } 4543 } 4544 } 4545 4546 /** 4547 * So can be overridden in subclasses. 4548 */ 4549 int getNumQualifiersForTestWritesWhileScanning() { 4550 return 100; 4551 } 4552 4553 /** 4554 * So can be overridden in subclasses. 4555 */ 4556 int getTestCountForTestWritesWhileScanning() { 4557 return 100; 4558 } 4559 4560 /** 4561 * Writes very wide records and scans for the latest every time.. Flushes and compacts the region 4562 * every now and then to keep things realistic. by flush / scan / compaction when joining threads 4563 */ 4564 @Test 4565 public void testWritesWhileScanning() throws IOException, InterruptedException { 4566 int testCount = getTestCountForTestWritesWhileScanning(); 4567 int numRows = 1; 4568 int numFamilies = 10; 4569 int numQualifiers = getNumQualifiersForTestWritesWhileScanning(); 4570 int flushInterval = 7; 4571 int compactInterval = 5 * flushInterval; 4572 byte[][] families = new byte[numFamilies][]; 4573 for (int i = 0; i < numFamilies; i++) { 4574 families[i] = Bytes.toBytes("family" + i); 4575 } 4576 byte[][] qualifiers = new byte[numQualifiers][]; 4577 for (int i = 0; i < numQualifiers; i++) { 4578 qualifiers[i] = Bytes.toBytes("qual" + i); 4579 } 4580 4581 this.region = initHRegion(tableName, method, CONF, families); 4582 FlushThread flushThread = new FlushThread(); 4583 PutThread putThread = new PutThread(numRows, families, qualifiers); 4584 try { 4585 putThread.start(); 4586 putThread.waitForFirstPut(); 4587 4588 flushThread.start(); 4589 4590 Scan scan = new Scan().withStartRow(Bytes.toBytes("row0")).withStopRow(Bytes.toBytes("row1")); 4591 4592 int expectedCount = numFamilies * numQualifiers; 4593 List<Cell> res = new ArrayList<>(); 4594 4595 long prevTimestamp = 0L; 4596 for (int i = 0; i < testCount; i++) { 4597 4598 if (i != 0 && i % compactInterval == 0) { 4599 region.compact(true); 4600 for (HStore store : region.getStores()) { 4601 store.closeAndArchiveCompactedFiles(); 4602 } 4603 } 4604 4605 if (i != 0 && i % flushInterval == 0) { 4606 flushThread.flush(); 4607 } 4608 4609 boolean previousEmpty = res.isEmpty(); 4610 res.clear(); 4611 try (InternalScanner scanner = region.getScanner(scan)) { 4612 boolean moreRows; 4613 do { 4614 moreRows = scanner.next(res); 4615 } while (moreRows); 4616 } 4617 if (!res.isEmpty() || !previousEmpty || i > compactInterval) { 4618 assertEquals("i=" + i, expectedCount, res.size()); 4619 long timestamp = res.get(0).getTimestamp(); 4620 assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp, 4621 timestamp >= prevTimestamp); 4622 prevTimestamp = timestamp; 4623 } 4624 } 4625 4626 putThread.done(); 4627 4628 region.flush(true); 4629 4630 } finally { 4631 try { 4632 flushThread.done(); 4633 flushThread.join(); 4634 flushThread.checkNoError(); 4635 4636 putThread.join(); 4637 putThread.checkNoError(); 4638 } catch (InterruptedException ie) { 4639 LOG.warn("Caught exception when joining with flushThread", ie); 4640 } 4641 4642 try { 4643 HBaseTestingUtil.closeRegionAndWAL(this.region); 4644 } catch (DroppedSnapshotException dse) { 4645 // We could get this on way out because we interrupt the background flusher and it could 4646 // fail anywhere causing a DSE over in the background flusher... only it is not properly 4647 // dealt with so could still be memory hanging out when we get to here -- memory we can't 4648 // flush because the accounting is 'off' since original DSE. 4649 } 4650 this.region = null; 4651 } 4652 } 4653 4654 protected class PutThread extends Thread { 4655 private volatile boolean done; 4656 private volatile int numPutsFinished = 0; 4657 4658 private Throwable error = null; 4659 private int numRows; 4660 private byte[][] families; 4661 private byte[][] qualifiers; 4662 4663 private PutThread(int numRows, byte[][] families, byte[][] qualifiers) { 4664 super("PutThread"); 4665 this.numRows = numRows; 4666 this.families = families; 4667 this.qualifiers = qualifiers; 4668 } 4669 4670 /** 4671 * Block calling thread until this instance of PutThread has put at least one row. 4672 */ 4673 public void waitForFirstPut() throws InterruptedException { 4674 // wait until put thread actually puts some data 4675 while (isAlive() && numPutsFinished == 0) { 4676 checkNoError(); 4677 Thread.sleep(50); 4678 } 4679 } 4680 4681 public void done() { 4682 done = true; 4683 synchronized (this) { 4684 interrupt(); 4685 } 4686 } 4687 4688 public void checkNoError() { 4689 if (error != null) { 4690 assertNull(error); 4691 } 4692 } 4693 4694 @Override 4695 public void run() { 4696 done = false; 4697 while (!done) { 4698 try { 4699 for (int r = 0; r < numRows; r++) { 4700 byte[] row = Bytes.toBytes("row" + r); 4701 Put put = new Put(row); 4702 put.setDurability(Durability.SKIP_WAL); 4703 byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished)); 4704 for (byte[] family : families) { 4705 for (byte[] qualifier : qualifiers) { 4706 put.addColumn(family, qualifier, numPutsFinished, value); 4707 } 4708 } 4709 region.put(put); 4710 numPutsFinished++; 4711 if (numPutsFinished > 0 && numPutsFinished % 47 == 0) { 4712 LOG.debug("put iteration = {}", numPutsFinished); 4713 Delete delete = new Delete(row, (long) numPutsFinished - 30); 4714 region.delete(delete); 4715 } 4716 numPutsFinished++; 4717 } 4718 } catch (InterruptedIOException e) { 4719 // This is fine. It means we are done, or didn't get the lock on time 4720 LOG.info("Interrupted", e); 4721 } catch (IOException e) { 4722 LOG.error("Error while putting records", e); 4723 error = e; 4724 break; 4725 } 4726 } 4727 4728 } 4729 4730 } 4731 4732 /** 4733 * Writes very wide records and gets the latest row every time.. Flushes and compacts the region 4734 * aggressivly to catch issues. by flush / scan / compaction when joining threads 4735 */ 4736 @Test 4737 public void testWritesWhileGetting() throws Exception { 4738 int testCount = 50; 4739 int numRows = 1; 4740 int numFamilies = 10; 4741 int numQualifiers = 100; 4742 int compactInterval = 100; 4743 byte[][] families = new byte[numFamilies][]; 4744 for (int i = 0; i < numFamilies; i++) { 4745 families[i] = Bytes.toBytes("family" + i); 4746 } 4747 byte[][] qualifiers = new byte[numQualifiers][]; 4748 for (int i = 0; i < numQualifiers; i++) { 4749 qualifiers[i] = Bytes.toBytes("qual" + i); 4750 } 4751 4752 // This test flushes constantly and can cause many files to be created, 4753 // possibly 4754 // extending over the ulimit. Make sure compactions are aggressive in 4755 // reducing 4756 // the number of HFiles created. 4757 Configuration conf = HBaseConfiguration.create(CONF); 4758 conf.setInt("hbase.hstore.compaction.min", 1); 4759 conf.setInt("hbase.hstore.compaction.max", 1000); 4760 this.region = initHRegion(tableName, method, conf, families); 4761 PutThread putThread = null; 4762 MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf); 4763 try { 4764 putThread = new PutThread(numRows, families, qualifiers); 4765 putThread.start(); 4766 putThread.waitForFirstPut(); 4767 4768 // Add a thread that flushes as fast as possible 4769 ctx.addThread(new RepeatingTestThread(ctx) { 4770 4771 @Override 4772 public void doAnAction() throws Exception { 4773 region.flush(true); 4774 // Compact regularly to avoid creating too many files and exceeding 4775 // the ulimit. 4776 region.compact(false); 4777 for (HStore store : region.getStores()) { 4778 store.closeAndArchiveCompactedFiles(); 4779 } 4780 } 4781 }); 4782 ctx.startThreads(); 4783 4784 Get get = new Get(Bytes.toBytes("row0")); 4785 Result result = null; 4786 4787 int expectedCount = numFamilies * numQualifiers; 4788 4789 long prevTimestamp = 0L; 4790 for (int i = 0; i < testCount; i++) { 4791 LOG.info("testWritesWhileGetting verify turn " + i); 4792 boolean previousEmpty = result == null || result.isEmpty(); 4793 result = region.get(get); 4794 if (!result.isEmpty() || !previousEmpty || i > compactInterval) { 4795 assertEquals("i=" + i, expectedCount, result.size()); 4796 // TODO this was removed, now what dangit?! 4797 // search looking for the qualifier in question? 4798 long timestamp = 0; 4799 for (Cell kv : result.rawCells()) { 4800 if ( 4801 CellUtil.matchingFamily(kv, families[0]) 4802 && CellUtil.matchingQualifier(kv, qualifiers[0]) 4803 ) { 4804 timestamp = kv.getTimestamp(); 4805 } 4806 } 4807 assertTrue(timestamp >= prevTimestamp); 4808 prevTimestamp = timestamp; 4809 Cell previousKV = null; 4810 4811 for (Cell kv : result.rawCells()) { 4812 byte[] thisValue = CellUtil.cloneValue(kv); 4813 if (previousKV != null) { 4814 if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) { 4815 LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV 4816 + "(memStoreTS:" + previousKV.getSequenceId() + ")" + ", New KV: " + kv 4817 + "(memStoreTS:" + kv.getSequenceId() + ")"); 4818 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue)); 4819 } 4820 } 4821 previousKV = kv; 4822 } 4823 } 4824 } 4825 } finally { 4826 if (putThread != null) putThread.done(); 4827 4828 region.flush(true); 4829 4830 if (putThread != null) { 4831 putThread.join(); 4832 putThread.checkNoError(); 4833 } 4834 4835 ctx.stop(); 4836 HBaseTestingUtil.closeRegionAndWAL(this.region); 4837 this.region = null; 4838 } 4839 } 4840 4841 @Test 4842 public void testHolesInMeta() throws Exception { 4843 byte[] family = Bytes.toBytes("family"); 4844 this.region = 4845 initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF, false, family); 4846 byte[] rowNotServed = Bytes.toBytes("a"); 4847 Get g = new Get(rowNotServed); 4848 try { 4849 region.get(g); 4850 fail(); 4851 } catch (WrongRegionException x) { 4852 // OK 4853 } 4854 byte[] row = Bytes.toBytes("y"); 4855 g = new Get(row); 4856 region.get(g); 4857 } 4858 4859 @Test 4860 public void testIndexesScanWithOneDeletedRow() throws IOException { 4861 byte[] family = Bytes.toBytes("family"); 4862 4863 // Setting up region 4864 this.region = initHRegion(tableName, method, CONF, family); 4865 Put put = new Put(Bytes.toBytes(1L)); 4866 put.addColumn(family, qual1, 1L, Bytes.toBytes(1L)); 4867 region.put(put); 4868 4869 region.flush(true); 4870 4871 Delete delete = new Delete(Bytes.toBytes(1L), 1L); 4872 region.delete(delete); 4873 4874 put = new Put(Bytes.toBytes(2L)); 4875 put.addColumn(family, qual1, 2L, Bytes.toBytes(2L)); 4876 region.put(put); 4877 4878 Scan idxScan = new Scan(); 4879 idxScan.addFamily(family); 4880 idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, 4881 Arrays.<Filter> asList( 4882 new SingleColumnValueFilter(family, qual1, CompareOperator.GREATER_OR_EQUAL, 4883 new BinaryComparator(Bytes.toBytes(0L))), 4884 new SingleColumnValueFilter(family, qual1, CompareOperator.LESS_OR_EQUAL, 4885 new BinaryComparator(Bytes.toBytes(3L)))))); 4886 InternalScanner scanner = region.getScanner(idxScan); 4887 List<Cell> res = new ArrayList<>(); 4888 4889 while (scanner.next(res)) { 4890 // Ignore res value. 4891 } 4892 assertEquals(1L, res.size()); 4893 } 4894 4895 // //////////////////////////////////////////////////////////////////////////// 4896 // Bloom filter test 4897 // //////////////////////////////////////////////////////////////////////////// 4898 @Test 4899 public void testBloomFilterSize() throws IOException { 4900 byte[] fam1 = Bytes.toBytes("fam1"); 4901 byte[] qf1 = Bytes.toBytes("col"); 4902 byte[] val1 = Bytes.toBytes("value1"); 4903 // Create Table 4904 TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName) 4905 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam1) 4906 .setMaxVersions(Integer.MAX_VALUE).setBloomFilterType(BloomType.ROWCOL).build()) 4907 .build(); 4908 RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 4909 this.region = TEST_UTIL.createLocalHRegion(info, tableDescriptor); 4910 int num_unique_rows = 10; 4911 int duplicate_multiplier = 2; 4912 int num_storefiles = 4; 4913 4914 int version = 0; 4915 for (int f = 0; f < num_storefiles; f++) { 4916 for (int i = 0; i < duplicate_multiplier; i++) { 4917 for (int j = 0; j < num_unique_rows; j++) { 4918 Put put = new Put(Bytes.toBytes("row" + j)); 4919 put.setDurability(Durability.SKIP_WAL); 4920 long ts = version++; 4921 put.addColumn(fam1, qf1, ts, val1); 4922 region.put(put); 4923 } 4924 } 4925 region.flush(true); 4926 } 4927 // before compaction 4928 HStore store = region.getStore(fam1); 4929 Collection<HStoreFile> storeFiles = store.getStorefiles(); 4930 for (HStoreFile storefile : storeFiles) { 4931 StoreFileReader reader = storefile.getReader(); 4932 reader.loadFileInfo(); 4933 reader.loadBloomfilter(); 4934 assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries()); 4935 assertEquals(num_unique_rows, reader.getFilterEntries()); 4936 } 4937 4938 region.compact(true); 4939 4940 // after compaction 4941 storeFiles = store.getStorefiles(); 4942 for (HStoreFile storefile : storeFiles) { 4943 StoreFileReader reader = storefile.getReader(); 4944 reader.loadFileInfo(); 4945 reader.loadBloomfilter(); 4946 assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries()); 4947 assertEquals(num_unique_rows, reader.getFilterEntries()); 4948 } 4949 } 4950 4951 @Test 4952 public void testAllColumnsWithBloomFilter() throws IOException { 4953 byte[] TABLE = Bytes.toBytes(name.getMethodName()); 4954 byte[] FAMILY = Bytes.toBytes("family"); 4955 4956 // Create table 4957 TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(TableName.valueOf(TABLE)) 4958 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY) 4959 .setMaxVersions(Integer.MAX_VALUE).setBloomFilterType(BloomType.ROWCOL).build()) 4960 .build(); 4961 RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 4962 this.region = TEST_UTIL.createLocalHRegion(info, tableDescriptor); 4963 // For row:0, col:0: insert versions 1 through 5. 4964 byte[] row = Bytes.toBytes("row:" + 0); 4965 byte[] column = Bytes.toBytes("column:" + 0); 4966 Put put = new Put(row); 4967 put.setDurability(Durability.SKIP_WAL); 4968 for (long idx = 1; idx <= 4; idx++) { 4969 put.addColumn(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx)); 4970 } 4971 region.put(put); 4972 4973 // Flush 4974 region.flush(true); 4975 4976 // Get rows 4977 Get get = new Get(row); 4978 get.readAllVersions(); 4979 Cell[] kvs = region.get(get).rawCells(); 4980 4981 // Check if rows are correct 4982 assertEquals(4, kvs.length); 4983 checkOneCell(kvs[0], FAMILY, 0, 0, 4); 4984 checkOneCell(kvs[1], FAMILY, 0, 0, 3); 4985 checkOneCell(kvs[2], FAMILY, 0, 0, 2); 4986 checkOneCell(kvs[3], FAMILY, 0, 0, 1); 4987 } 4988 4989 /** 4990 * Testcase to cover bug-fix for HBASE-2823 Ensures correct delete when issuing delete row on 4991 * columns with bloom filter set to row+col (BloomType.ROWCOL) 4992 */ 4993 @Test 4994 public void testDeleteRowWithBloomFilter() throws IOException { 4995 byte[] familyName = Bytes.toBytes("familyName"); 4996 4997 // Create Table 4998 TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName) 4999 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(familyName) 5000 .setMaxVersions(Integer.MAX_VALUE).setBloomFilterType(BloomType.ROWCOL).build()) 5001 .build(); 5002 RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 5003 this.region = TEST_UTIL.createLocalHRegion(info, tableDescriptor); 5004 // Insert some data 5005 byte[] row = Bytes.toBytes("row1"); 5006 byte[] col = Bytes.toBytes("col1"); 5007 5008 Put put = new Put(row); 5009 put.addColumn(familyName, col, 1, Bytes.toBytes("SomeRandomValue")); 5010 region.put(put); 5011 region.flush(true); 5012 5013 Delete del = new Delete(row); 5014 region.delete(del); 5015 region.flush(true); 5016 5017 // Get remaining rows (should have none) 5018 Get get = new Get(row); 5019 get.addColumn(familyName, col); 5020 5021 Cell[] keyValues = region.get(get).rawCells(); 5022 assertEquals(0, keyValues.length); 5023 } 5024 5025 @Test 5026 public void testgetHDFSBlocksDistribution() throws Exception { 5027 HBaseTestingUtil htu = new HBaseTestingUtil(); 5028 // Why do we set the block size in this test? If we set it smaller than the kvs, then we'll 5029 // break up the file in to more pieces that can be distributed across the three nodes and we 5030 // won't be able to have the condition this test asserts; that at least one node has 5031 // a copy of all replicas -- if small block size, then blocks are spread evenly across the 5032 // the three nodes. hfilev3 with tags seems to put us over the block size. St.Ack. 5033 // final int DEFAULT_BLOCK_SIZE = 1024; 5034 // htu.getConfiguration().setLong("dfs.blocksize", DEFAULT_BLOCK_SIZE); 5035 htu.getConfiguration().setInt("dfs.replication", 2); 5036 5037 // set up a cluster with 3 nodes 5038 SingleProcessHBaseCluster cluster = null; 5039 String dataNodeHosts[] = new String[] { "host1", "host2", "host3" }; 5040 int regionServersCount = 3; 5041 5042 try { 5043 StartTestingClusterOption option = StartTestingClusterOption.builder() 5044 .numRegionServers(regionServersCount).dataNodeHosts(dataNodeHosts).build(); 5045 cluster = htu.startMiniCluster(option); 5046 byte[][] families = { fam1, fam2 }; 5047 Table ht = htu.createTable(tableName, families); 5048 5049 // Setting up region 5050 byte row[] = Bytes.toBytes("row1"); 5051 byte col[] = Bytes.toBytes("col1"); 5052 5053 Put put = new Put(row); 5054 put.addColumn(fam1, col, 1, Bytes.toBytes("test1")); 5055 put.addColumn(fam2, col, 1, Bytes.toBytes("test2")); 5056 ht.put(put); 5057 5058 HRegion firstRegion = htu.getHBaseCluster().getRegions(tableName).get(0); 5059 firstRegion.flush(true); 5060 HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution(); 5061 5062 // Given the default replication factor is 2 and we have 2 HFiles, 5063 // we will have total of 4 replica of blocks on 3 datanodes; thus there 5064 // must be at least one host that have replica for 2 HFiles. That host's 5065 // weight will be equal to the unique block weight. 5066 long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight(); 5067 StringBuilder sb = new StringBuilder(); 5068 for (String host : blocksDistribution1.getTopHosts()) { 5069 if (sb.length() > 0) sb.append(", "); 5070 sb.append(host); 5071 sb.append("="); 5072 sb.append(blocksDistribution1.getWeight(host)); 5073 } 5074 5075 String topHost = blocksDistribution1.getTopHosts().get(0); 5076 long topHostWeight = blocksDistribution1.getWeight(topHost); 5077 String msg = "uniqueBlocksWeight=" + uniqueBlocksWeight1 + ", topHostWeight=" + topHostWeight 5078 + ", topHost=" + topHost + "; " + sb.toString(); 5079 LOG.info(msg); 5080 assertTrue(msg, uniqueBlocksWeight1 == topHostWeight); 5081 5082 // use the static method to compute the value, it should be the same. 5083 // static method is used by load balancer or other components 5084 HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution( 5085 htu.getConfiguration(), firstRegion.getTableDescriptor(), firstRegion.getRegionInfo()); 5086 long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight(); 5087 5088 assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2); 5089 5090 ht.close(); 5091 } finally { 5092 if (cluster != null) { 5093 htu.shutdownMiniCluster(); 5094 } 5095 } 5096 } 5097 5098 /** 5099 * Testcase to check state of region initialization task set to ABORTED or not if any exceptions 5100 * during initialization 5101 */ 5102 @Test 5103 public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception { 5104 RegionInfo info; 5105 try { 5106 FileSystem fs = mock(FileSystem.class); 5107 when(fs.exists(any())).thenThrow(new IOException()); 5108 TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(tableName); 5109 ColumnFamilyDescriptor columnFamilyDescriptor = 5110 ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf")).build(); 5111 tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor); 5112 info = RegionInfoBuilder.newBuilder(tableName).build(); 5113 Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization"); 5114 region = HRegion.newHRegion(path, null, fs, CONF, info, tableDescriptorBuilder.build(), null); 5115 // region initialization throws IOException and set task state to ABORTED. 5116 region.initialize(); 5117 fail("Region initialization should fail due to IOException"); 5118 } catch (IOException io) { 5119 List<MonitoredTask> tasks = TaskMonitor.get().getTasks(); 5120 for (MonitoredTask monitoredTask : tasks) { 5121 if ( 5122 !(monitoredTask instanceof MonitoredRPCHandler) 5123 && monitoredTask.getDescription().contains(region.toString()) 5124 ) { 5125 assertTrue("Region state should be ABORTED.", 5126 monitoredTask.getState().equals(MonitoredTask.State.ABORTED)); 5127 break; 5128 } 5129 } 5130 } 5131 } 5132 5133 /** 5134 * Verifies that the .regioninfo file is written on region creation and that is recreated if 5135 * missing during region opening. 5136 */ 5137 @Test 5138 public void testRegionInfoFileCreation() throws IOException { 5139 Path rootDir = new Path(dir + "testRegionInfoFileCreation"); 5140 5141 TableDescriptorBuilder tableDescriptorBuilder = 5142 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())); 5143 ColumnFamilyDescriptor columnFamilyDescriptor = 5144 ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf")).build(); 5145 tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor); 5146 TableDescriptor tableDescriptor = tableDescriptorBuilder.build(); 5147 5148 RegionInfo hri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 5149 5150 // Create a region and skip the initialization (like CreateTableHandler) 5151 region = HBaseTestingUtil.createRegionAndWAL(hri, rootDir, CONF, tableDescriptor, false); 5152 Path regionDir = region.getRegionFileSystem().getRegionDir(); 5153 FileSystem fs = region.getRegionFileSystem().getFileSystem(); 5154 HBaseTestingUtil.closeRegionAndWAL(region); 5155 5156 Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE); 5157 5158 // Verify that the .regioninfo file is present 5159 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir", 5160 fs.exists(regionInfoFile)); 5161 5162 // Try to open the region 5163 region = HRegion.openHRegion(rootDir, hri, tableDescriptor, null, CONF); 5164 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir()); 5165 HBaseTestingUtil.closeRegionAndWAL(region); 5166 5167 // Verify that the .regioninfo file is still there 5168 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir", 5169 fs.exists(regionInfoFile)); 5170 5171 // Remove the .regioninfo file and verify is recreated on region open 5172 fs.delete(regionInfoFile, true); 5173 assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir", 5174 fs.exists(regionInfoFile)); 5175 5176 region = HRegion.openHRegion(rootDir, hri, tableDescriptor, null, CONF); 5177 // region = TEST_UTIL.openHRegion(hri, htd); 5178 assertEquals(regionDir, region.getRegionFileSystem().getRegionDir()); 5179 HBaseTestingUtil.closeRegionAndWAL(region); 5180 5181 // Verify that the .regioninfo file is still there 5182 assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir", 5183 fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE))); 5184 5185 region = null; 5186 } 5187 5188 /** 5189 * TestCase for increment 5190 */ 5191 private static class Incrementer implements Runnable { 5192 private HRegion region; 5193 private final static byte[] incRow = Bytes.toBytes("incRow"); 5194 private final static byte[] family = Bytes.toBytes("family"); 5195 private final static byte[] qualifier = Bytes.toBytes("qualifier"); 5196 private final static long ONE = 1L; 5197 private int incCounter; 5198 5199 public Incrementer(HRegion region, int incCounter) { 5200 this.region = region; 5201 this.incCounter = incCounter; 5202 } 5203 5204 @Override 5205 public void run() { 5206 int count = 0; 5207 while (count < incCounter) { 5208 Increment inc = new Increment(incRow); 5209 inc.addColumn(family, qualifier, ONE); 5210 count++; 5211 try { 5212 region.increment(inc); 5213 } catch (IOException e) { 5214 LOG.info("Count=" + count + ", " + e); 5215 break; 5216 } 5217 } 5218 } 5219 } 5220 5221 /** 5222 * Test case to check increment function with memstore flushing 5223 */ 5224 @Test 5225 public void testParallelIncrementWithMemStoreFlush() throws Exception { 5226 byte[] family = Incrementer.family; 5227 this.region = initHRegion(tableName, method, CONF, family); 5228 final HRegion region = this.region; 5229 final AtomicBoolean incrementDone = new AtomicBoolean(false); 5230 Runnable flusher = new Runnable() { 5231 @Override 5232 public void run() { 5233 while (!incrementDone.get()) { 5234 try { 5235 region.flush(true); 5236 } catch (Exception e) { 5237 e.printStackTrace(); 5238 } 5239 } 5240 } 5241 }; 5242 5243 // after all increment finished, the row will increment to 20*100 = 2000 5244 int threadNum = 20; 5245 int incCounter = 100; 5246 long expected = (long) threadNum * incCounter; 5247 Thread[] incrementers = new Thread[threadNum]; 5248 Thread flushThread = new Thread(flusher); 5249 for (int i = 0; i < threadNum; i++) { 5250 incrementers[i] = new Thread(new Incrementer(this.region, incCounter)); 5251 incrementers[i].start(); 5252 } 5253 flushThread.start(); 5254 for (int i = 0; i < threadNum; i++) { 5255 incrementers[i].join(); 5256 } 5257 5258 incrementDone.set(true); 5259 flushThread.join(); 5260 5261 Get get = new Get(Incrementer.incRow); 5262 get.addColumn(Incrementer.family, Incrementer.qualifier); 5263 get.readVersions(1); 5264 Result res = this.region.get(get); 5265 List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier); 5266 5267 // we just got the latest version 5268 assertEquals(1, kvs.size()); 5269 Cell kv = kvs.get(0); 5270 assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset())); 5271 } 5272 5273 /** 5274 * TestCase for append 5275 */ 5276 private static class Appender implements Runnable { 5277 private HRegion region; 5278 private final static byte[] appendRow = Bytes.toBytes("appendRow"); 5279 private final static byte[] family = Bytes.toBytes("family"); 5280 private final static byte[] qualifier = Bytes.toBytes("qualifier"); 5281 private final static byte[] CHAR = Bytes.toBytes("a"); 5282 private int appendCounter; 5283 5284 public Appender(HRegion region, int appendCounter) { 5285 this.region = region; 5286 this.appendCounter = appendCounter; 5287 } 5288 5289 @Override 5290 public void run() { 5291 int count = 0; 5292 while (count < appendCounter) { 5293 Append app = new Append(appendRow); 5294 app.addColumn(family, qualifier, CHAR); 5295 count++; 5296 try { 5297 region.append(app); 5298 } catch (IOException e) { 5299 LOG.info("Count=" + count + ", max=" + appendCounter + ", " + e); 5300 break; 5301 } 5302 } 5303 } 5304 } 5305 5306 /** 5307 * Test case to check append function with memstore flushing 5308 */ 5309 @Test 5310 public void testParallelAppendWithMemStoreFlush() throws Exception { 5311 byte[] family = Appender.family; 5312 this.region = initHRegion(tableName, method, CONF, family); 5313 final HRegion region = this.region; 5314 final AtomicBoolean appendDone = new AtomicBoolean(false); 5315 Runnable flusher = new Runnable() { 5316 @Override 5317 public void run() { 5318 while (!appendDone.get()) { 5319 try { 5320 region.flush(true); 5321 } catch (Exception e) { 5322 e.printStackTrace(); 5323 } 5324 } 5325 } 5326 }; 5327 5328 // After all append finished, the value will append to threadNum * 5329 // appendCounter Appender.CHAR 5330 int threadNum = 20; 5331 int appendCounter = 100; 5332 byte[] expected = new byte[threadNum * appendCounter]; 5333 for (int i = 0; i < threadNum * appendCounter; i++) { 5334 System.arraycopy(Appender.CHAR, 0, expected, i, 1); 5335 } 5336 Thread[] appenders = new Thread[threadNum]; 5337 Thread flushThread = new Thread(flusher); 5338 for (int i = 0; i < threadNum; i++) { 5339 appenders[i] = new Thread(new Appender(this.region, appendCounter)); 5340 appenders[i].start(); 5341 } 5342 flushThread.start(); 5343 for (int i = 0; i < threadNum; i++) { 5344 appenders[i].join(); 5345 } 5346 5347 appendDone.set(true); 5348 flushThread.join(); 5349 5350 Get get = new Get(Appender.appendRow); 5351 get.addColumn(Appender.family, Appender.qualifier); 5352 get.readVersions(1); 5353 Result res = this.region.get(get); 5354 List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier); 5355 5356 // we just got the latest version 5357 assertEquals(1, kvs.size()); 5358 Cell kv = kvs.get(0); 5359 byte[] appendResult = new byte[kv.getValueLength()]; 5360 System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength()); 5361 assertArrayEquals(expected, appendResult); 5362 } 5363 5364 /** 5365 * Test case to check put function with memstore flushing for same row, same ts 5366 */ 5367 @Test 5368 public void testPutWithMemStoreFlush() throws Exception { 5369 byte[] family = Bytes.toBytes("family"); 5370 byte[] qualifier = Bytes.toBytes("qualifier"); 5371 byte[] row = Bytes.toBytes("putRow"); 5372 byte[] value = null; 5373 this.region = initHRegion(tableName, method, CONF, family); 5374 Put put = null; 5375 Get get = null; 5376 List<Cell> kvs = null; 5377 Result res = null; 5378 5379 put = new Put(row); 5380 value = Bytes.toBytes("value0"); 5381 put.addColumn(family, qualifier, 1234567L, value); 5382 region.put(put); 5383 get = new Get(row); 5384 get.addColumn(family, qualifier); 5385 get.readAllVersions(); 5386 res = this.region.get(get); 5387 kvs = res.getColumnCells(family, qualifier); 5388 assertEquals(1, kvs.size()); 5389 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0))); 5390 5391 region.flush(true); 5392 get = new Get(row); 5393 get.addColumn(family, qualifier); 5394 get.readAllVersions(); 5395 res = this.region.get(get); 5396 kvs = res.getColumnCells(family, qualifier); 5397 assertEquals(1, kvs.size()); 5398 assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0))); 5399 5400 put = new Put(row); 5401 value = Bytes.toBytes("value1"); 5402 put.addColumn(family, qualifier, 1234567L, value); 5403 region.put(put); 5404 get = new Get(row); 5405 get.addColumn(family, qualifier); 5406 get.readAllVersions(); 5407 res = this.region.get(get); 5408 kvs = res.getColumnCells(family, qualifier); 5409 assertEquals(1, kvs.size()); 5410 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0))); 5411 5412 region.flush(true); 5413 get = new Get(row); 5414 get.addColumn(family, qualifier); 5415 get.readAllVersions(); 5416 res = this.region.get(get); 5417 kvs = res.getColumnCells(family, qualifier); 5418 assertEquals(1, kvs.size()); 5419 assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0))); 5420 } 5421 5422 /** 5423 * For this test,the spied {@link AsyncFSWAL} can not work properly because of a Mockito defect 5424 * that can not deal with classes which have a field of an inner class. See discussions in 5425 * HBASE-15536.When we reuse the code of {@link AsyncFSWAL} for {@link FSHLog}, this test could 5426 * not work for {@link FSHLog} also. 5427 */ 5428 @Test 5429 @Ignore 5430 public void testDurability() throws Exception { 5431 // there are 5 x 5 cases: 5432 // table durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT) x mutation 5433 // durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT) 5434 5435 // expected cases for append and sync wal 5436 durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false); 5437 durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false); 5438 durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false); 5439 5440 durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false); 5441 durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false); 5442 durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false); 5443 5444 durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false); 5445 durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false); 5446 5447 durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false); 5448 durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false); 5449 5450 durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false); 5451 durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false); 5452 durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false); 5453 5454 // expected cases for async wal 5455 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false); 5456 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false); 5457 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false); 5458 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false); 5459 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false); 5460 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false); 5461 5462 durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true); 5463 durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true); 5464 durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true); 5465 durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true); 5466 durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true); 5467 durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true); 5468 5469 // expect skip wal cases 5470 durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false); 5471 durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false); 5472 durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false); 5473 durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false); 5474 durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false); 5475 durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false); 5476 5477 } 5478 5479 private void durabilityTest(String method, Durability tableDurability, 5480 Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync, 5481 final boolean expectSyncFromLogSyncer) throws Exception { 5482 Configuration conf = HBaseConfiguration.create(CONF); 5483 conf.setLong(AbstractFSWAL.WAL_SHUTDOWN_WAIT_TIMEOUT_MS, 60 * 60 * 1000); 5484 method = method + "_" + tableDurability.name() + "_" + mutationDurability.name(); 5485 byte[] family = Bytes.toBytes("family"); 5486 Path logDir = new Path(new Path(dir + method), "log"); 5487 final Configuration walConf = new Configuration(conf); 5488 CommonFSUtils.setRootDir(walConf, logDir); 5489 // XXX: The spied AsyncFSWAL can not work properly because of a Mockito defect that can not 5490 // deal with classes which have a field of an inner class. See discussions in HBASE-15536. 5491 walConf.set(WALFactory.WAL_PROVIDER, "filesystem"); 5492 final WALFactory wals = new WALFactory(walConf, HBaseTestingUtil.getRandomUUID().toString()); 5493 final WAL wal = spy(wals.getWAL(RegionInfoBuilder.newBuilder(tableName).build())); 5494 this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF, 5495 false, tableDurability, wal, new byte[][] { family }); 5496 5497 Put put = new Put(Bytes.toBytes("r1")); 5498 put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); 5499 put.setDurability(mutationDurability); 5500 region.put(put); 5501 5502 // verify append called or not 5503 verify(wal, expectAppend ? times(1) : never()).appendData((RegionInfo) any(), 5504 (WALKeyImpl) any(), (WALEdit) any()); 5505 5506 // verify sync called or not 5507 if (expectSync || expectSyncFromLogSyncer) { 5508 TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() { 5509 @Override 5510 public boolean evaluate() throws Exception { 5511 try { 5512 if (expectSync) { 5513 verify(wal, times(1)).sync(anyLong()); // Hregion calls this one 5514 } else if (expectSyncFromLogSyncer) { 5515 verify(wal, times(1)).sync(); // wal syncer calls this one 5516 } 5517 } catch (Throwable ignore) { 5518 } 5519 return true; 5520 } 5521 }); 5522 } else { 5523 // verify(wal, never()).sync(anyLong()); 5524 verify(wal, never()).sync(); 5525 } 5526 5527 HBaseTestingUtil.closeRegionAndWAL(this.region); 5528 wals.close(); 5529 this.region = null; 5530 } 5531 5532 @Test 5533 public void testRegionReplicaSecondary() throws IOException { 5534 // create a primary region, load some data and flush 5535 // create a secondary region, and do a get against that 5536 Path rootDir = new Path(dir + name.getMethodName()); 5537 CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir); 5538 5539 byte[][] families = 5540 new byte[][] { Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3") }; 5541 byte[] cq = Bytes.toBytes("cq"); 5542 TableDescriptorBuilder builder = 5543 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())); 5544 for (byte[] family : families) { 5545 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)); 5546 } 5547 TableDescriptor tableDescriptor = builder.build(); 5548 long time = EnvironmentEdgeManager.currentTime(); 5549 RegionInfo primaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5550 .setRegionId(time).setReplicaId(0).build(); 5551 RegionInfo secondaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5552 .setRegionId(time).setReplicaId(1).build(); 5553 5554 HRegion primaryRegion = null, secondaryRegion = null; 5555 5556 try { 5557 primaryRegion = HBaseTestingUtil.createRegionAndWAL(primaryHri, rootDir, 5558 TEST_UTIL.getConfiguration(), tableDescriptor); 5559 5560 // load some data 5561 putData(primaryRegion, 0, 1000, cq, families); 5562 5563 // flush region 5564 primaryRegion.flush(true); 5565 5566 // open secondary region 5567 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, tableDescriptor, null, CONF); 5568 5569 verifyData(secondaryRegion, 0, 1000, cq, families); 5570 } finally { 5571 if (primaryRegion != null) { 5572 HBaseTestingUtil.closeRegionAndWAL(primaryRegion); 5573 } 5574 if (secondaryRegion != null) { 5575 HBaseTestingUtil.closeRegionAndWAL(secondaryRegion); 5576 } 5577 } 5578 } 5579 5580 @Test 5581 public void testRegionReplicaSecondaryIsReadOnly() throws IOException { 5582 // create a primary region, load some data and flush 5583 // create a secondary region, and do a put against that 5584 Path rootDir = new Path(dir + name.getMethodName()); 5585 CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir); 5586 5587 byte[][] families = 5588 new byte[][] { Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3") }; 5589 byte[] cq = Bytes.toBytes("cq"); 5590 TableDescriptorBuilder builder = 5591 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())); 5592 for (byte[] family : families) { 5593 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)); 5594 } 5595 TableDescriptor tableDescriptor = builder.build(); 5596 long time = EnvironmentEdgeManager.currentTime(); 5597 RegionInfo primaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5598 .setRegionId(time).setReplicaId(0).build(); 5599 RegionInfo secondaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5600 .setRegionId(time).setReplicaId(1).build(); 5601 5602 HRegion primaryRegion = null, secondaryRegion = null; 5603 5604 try { 5605 primaryRegion = HBaseTestingUtil.createRegionAndWAL(primaryHri, rootDir, 5606 TEST_UTIL.getConfiguration(), tableDescriptor); 5607 5608 // load some data 5609 putData(primaryRegion, 0, 1000, cq, families); 5610 5611 // flush region 5612 primaryRegion.flush(true); 5613 5614 // open secondary region 5615 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, tableDescriptor, null, CONF); 5616 5617 try { 5618 putData(secondaryRegion, 0, 1000, cq, families); 5619 fail("Should have thrown exception"); 5620 } catch (IOException ex) { 5621 // expected 5622 } 5623 } finally { 5624 if (primaryRegion != null) { 5625 HBaseTestingUtil.closeRegionAndWAL(primaryRegion); 5626 } 5627 if (secondaryRegion != null) { 5628 HBaseTestingUtil.closeRegionAndWAL(secondaryRegion); 5629 } 5630 } 5631 } 5632 5633 static WALFactory createWALFactory(Configuration conf, Path rootDir) throws IOException { 5634 Configuration confForWAL = new Configuration(conf); 5635 confForWAL.set(HConstants.HBASE_DIR, rootDir.toString()); 5636 return new WALFactory(confForWAL, "hregion-" + RandomStringUtils.randomNumeric(8)); 5637 } 5638 5639 @Test 5640 public void testCompactionFromPrimary() throws IOException { 5641 Path rootDir = new Path(dir + name.getMethodName()); 5642 CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir); 5643 5644 byte[][] families = 5645 new byte[][] { Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3") }; 5646 byte[] cq = Bytes.toBytes("cq"); 5647 TableDescriptorBuilder builder = 5648 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())); 5649 for (byte[] family : families) { 5650 builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)); 5651 } 5652 TableDescriptor tableDescriptor = builder.build(); 5653 long time = EnvironmentEdgeManager.currentTime(); 5654 RegionInfo primaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5655 .setRegionId(time).setReplicaId(0).build(); 5656 RegionInfo secondaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()) 5657 .setRegionId(time).setReplicaId(1).build(); 5658 5659 HRegion primaryRegion = null, secondaryRegion = null; 5660 5661 try { 5662 primaryRegion = HBaseTestingUtil.createRegionAndWAL(primaryHri, rootDir, 5663 TEST_UTIL.getConfiguration(), tableDescriptor); 5664 5665 // load some data 5666 putData(primaryRegion, 0, 1000, cq, families); 5667 5668 // flush region 5669 primaryRegion.flush(true); 5670 5671 // open secondary region 5672 secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, tableDescriptor, null, CONF); 5673 5674 // move the file of the primary region to the archive, simulating a compaction 5675 Collection<HStoreFile> storeFiles = primaryRegion.getStore(families[0]).getStorefiles(); 5676 primaryRegion.getRegionFileSystem().removeStoreFiles(Bytes.toString(families[0]), storeFiles); 5677 Collection<StoreFileInfo> storeFileInfos = 5678 primaryRegion.getRegionFileSystem().getStoreFiles(Bytes.toString(families[0])); 5679 Assert.assertTrue(storeFileInfos == null || storeFileInfos.isEmpty()); 5680 5681 verifyData(secondaryRegion, 0, 1000, cq, families); 5682 } finally { 5683 if (primaryRegion != null) { 5684 HBaseTestingUtil.closeRegionAndWAL(primaryRegion); 5685 } 5686 if (secondaryRegion != null) { 5687 HBaseTestingUtil.closeRegionAndWAL(secondaryRegion); 5688 } 5689 } 5690 } 5691 5692 private void putData(int startRow, int numRows, byte[] qf, byte[]... families) 5693 throws IOException { 5694 putData(this.region, startRow, numRows, qf, families); 5695 } 5696 5697 private void putData(HRegion region, int startRow, int numRows, byte[] qf, byte[]... families) 5698 throws IOException { 5699 putData(region, Durability.SKIP_WAL, startRow, numRows, qf, families); 5700 } 5701 5702 static void putData(HRegion region, Durability durability, int startRow, int numRows, byte[] qf, 5703 byte[]... families) throws IOException { 5704 for (int i = startRow; i < startRow + numRows; i++) { 5705 Put put = new Put(Bytes.toBytes("" + i)); 5706 put.setDurability(durability); 5707 for (byte[] family : families) { 5708 put.addColumn(family, qf, null); 5709 } 5710 region.put(put); 5711 LOG.info(put.toString()); 5712 } 5713 } 5714 5715 static void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families) 5716 throws IOException { 5717 for (int i = startRow; i < startRow + numRows; i++) { 5718 byte[] row = Bytes.toBytes("" + i); 5719 Get get = new Get(row); 5720 for (byte[] family : families) { 5721 get.addColumn(family, qf); 5722 } 5723 Result result = newReg.get(get); 5724 Cell[] raw = result.rawCells(); 5725 assertEquals(families.length, result.size()); 5726 for (int j = 0; j < families.length; j++) { 5727 assertTrue(CellUtil.matchingRows(raw[j], row)); 5728 assertTrue(CellUtil.matchingFamily(raw[j], families[j])); 5729 assertTrue(CellUtil.matchingQualifier(raw[j], qf)); 5730 } 5731 } 5732 } 5733 5734 static void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException { 5735 // Now I have k, get values out and assert they are as expected. 5736 Get get = new Get(k).addFamily(family).readAllVersions(); 5737 Cell[] results = r.get(get).rawCells(); 5738 for (int j = 0; j < results.length; j++) { 5739 byte[] tmp = CellUtil.cloneValue(results[j]); 5740 // Row should be equal to value every time. 5741 assertTrue(Bytes.equals(k, tmp)); 5742 } 5743 } 5744 5745 /* 5746 * Assert first value in the passed region is <code>firstValue</code>. 5747 */ 5748 protected void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue) 5749 throws IOException { 5750 byte[][] families = { fs }; 5751 Scan scan = new Scan(); 5752 for (int i = 0; i < families.length; i++) 5753 scan.addFamily(families[i]); 5754 InternalScanner s = r.getScanner(scan); 5755 try { 5756 List<Cell> curVals = new ArrayList<>(); 5757 boolean first = true; 5758 OUTER_LOOP: while (s.next(curVals)) { 5759 for (Cell kv : curVals) { 5760 byte[] val = CellUtil.cloneValue(kv); 5761 byte[] curval = val; 5762 if (first) { 5763 first = false; 5764 assertTrue(Bytes.compareTo(curval, firstValue) == 0); 5765 } else { 5766 // Not asserting anything. Might as well break. 5767 break OUTER_LOOP; 5768 } 5769 } 5770 } 5771 } finally { 5772 s.close(); 5773 } 5774 } 5775 5776 /** 5777 * Test that we get the expected flush results back 5778 */ 5779 @Test 5780 public void testFlushResult() throws IOException { 5781 byte[] family = Bytes.toBytes("family"); 5782 5783 this.region = initHRegion(tableName, method, family); 5784 5785 // empty memstore, flush doesn't run 5786 HRegion.FlushResult fr = region.flush(true); 5787 assertFalse(fr.isFlushSucceeded()); 5788 assertFalse(fr.isCompactionNeeded()); 5789 5790 // Flush enough files to get up to the threshold, doesn't need compactions 5791 for (int i = 0; i < 2; i++) { 5792 Put put = new Put(tableName.toBytes()).addColumn(family, family, tableName.toBytes()); 5793 region.put(put); 5794 fr = region.flush(true); 5795 assertTrue(fr.isFlushSucceeded()); 5796 assertFalse(fr.isCompactionNeeded()); 5797 } 5798 5799 // Two flushes after the threshold, compactions are needed 5800 for (int i = 0; i < 2; i++) { 5801 Put put = new Put(tableName.toBytes()).addColumn(family, family, tableName.toBytes()); 5802 region.put(put); 5803 fr = region.flush(true); 5804 assertTrue(fr.isFlushSucceeded()); 5805 assertTrue(fr.isCompactionNeeded()); 5806 } 5807 } 5808 5809 protected Configuration initSplit() { 5810 // Always compact if there is more than one store file. 5811 CONF.setInt("hbase.hstore.compactionThreshold", 2); 5812 5813 CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000); 5814 5815 // Increase the amount of time between client retries 5816 CONF.setLong("hbase.client.pause", 15 * 1000); 5817 5818 // This size should make it so we always split using the addContent 5819 // below. After adding all data, the first region is 1.3M 5820 CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128); 5821 return CONF; 5822 } 5823 5824 /** 5825 * @return A region on which you must call {@link HBaseTestingUtil#closeRegionAndWAL(HRegion)} 5826 * when done. 5827 */ 5828 protected HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf, 5829 byte[]... families) throws IOException { 5830 return initHRegion(tableName, callingMethod, conf, false, families); 5831 } 5832 5833 /** 5834 * @return A region on which you must call {@link HBaseTestingUtil#closeRegionAndWAL(HRegion)} 5835 * when done. 5836 */ 5837 protected HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf, 5838 boolean isReadOnly, byte[]... families) throws IOException { 5839 return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families); 5840 } 5841 5842 protected HRegion initHRegion(TableName tableName, byte[] startKey, byte[] stopKey, 5843 String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families) 5844 throws IOException { 5845 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(callingMethod + ".log"); 5846 RegionInfo hri = 5847 RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey).setEndKey(stopKey).build(); 5848 final WAL wal = HBaseTestingUtil.createWal(conf, logDir, hri); 5849 return initHRegion(tableName, startKey, stopKey, conf, isReadOnly, Durability.SYNC_WAL, wal, 5850 families); 5851 } 5852 5853 /** 5854 * @return A region on which you must call {@link HBaseTestingUtil#closeRegionAndWAL(HRegion)} 5855 * when done. 5856 */ 5857 public HRegion initHRegion(TableName tableName, byte[] startKey, byte[] stopKey, 5858 Configuration conf, boolean isReadOnly, Durability durability, WAL wal, byte[]... families) 5859 throws IOException { 5860 ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null, 5861 MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT); 5862 return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, conf, isReadOnly, durability, 5863 wal, families); 5864 } 5865 5866 /** 5867 * Assert that the passed in Cell has expected contents for the specified row, column & timestamp. 5868 */ 5869 private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) { 5870 String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts; 5871 assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx, 5872 Bytes.toString(CellUtil.cloneRow(kv))); 5873 assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf), 5874 Bytes.toString(CellUtil.cloneFamily(kv))); 5875 assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx, 5876 Bytes.toString(CellUtil.cloneQualifier(kv))); 5877 assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp()); 5878 assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts, 5879 Bytes.toString(CellUtil.cloneValue(kv))); 5880 } 5881 5882 @Test 5883 public void testReverseScanner_FromMemStore_SingleCF_Normal() throws IOException { 5884 byte[] rowC = Bytes.toBytes("rowC"); 5885 byte[] rowA = Bytes.toBytes("rowA"); 5886 byte[] rowB = Bytes.toBytes("rowB"); 5887 byte[] cf = Bytes.toBytes("CF"); 5888 byte[][] families = { cf }; 5889 byte[] col = Bytes.toBytes("C"); 5890 long ts = 1; 5891 this.region = initHRegion(tableName, method, families); 5892 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null); 5893 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, null); 5894 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null); 5895 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null); 5896 Put put = null; 5897 put = new Put(rowC); 5898 put.add(kv1); 5899 put.add(kv11); 5900 region.put(put); 5901 put = new Put(rowA); 5902 put.add(kv2); 5903 region.put(put); 5904 put = new Put(rowB); 5905 put.add(kv3); 5906 region.put(put); 5907 5908 Scan scan = new Scan().withStartRow(rowC); 5909 scan.readVersions(5); 5910 scan.setReversed(true); 5911 InternalScanner scanner = region.getScanner(scan); 5912 List<Cell> currRow = new ArrayList<>(); 5913 boolean hasNext = scanner.next(currRow); 5914 assertEquals(2, currRow.size()); 5915 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5916 currRow.get(0).getRowLength(), rowC, 0, rowC.length)); 5917 assertTrue(hasNext); 5918 currRow.clear(); 5919 hasNext = scanner.next(currRow); 5920 assertEquals(1, currRow.size()); 5921 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5922 currRow.get(0).getRowLength(), rowB, 0, rowB.length)); 5923 assertTrue(hasNext); 5924 currRow.clear(); 5925 hasNext = scanner.next(currRow); 5926 assertEquals(1, currRow.size()); 5927 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5928 currRow.get(0).getRowLength(), rowA, 0, rowA.length)); 5929 assertFalse(hasNext); 5930 scanner.close(); 5931 } 5932 5933 @Test 5934 public void testReverseScanner_FromMemStore_SingleCF_LargerKey() throws IOException { 5935 byte[] rowC = Bytes.toBytes("rowC"); 5936 byte[] rowA = Bytes.toBytes("rowA"); 5937 byte[] rowB = Bytes.toBytes("rowB"); 5938 byte[] rowD = Bytes.toBytes("rowD"); 5939 byte[] cf = Bytes.toBytes("CF"); 5940 byte[][] families = { cf }; 5941 byte[] col = Bytes.toBytes("C"); 5942 long ts = 1; 5943 this.region = initHRegion(tableName, method, families); 5944 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null); 5945 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, null); 5946 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null); 5947 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null); 5948 Put put = null; 5949 put = new Put(rowC); 5950 put.add(kv1); 5951 put.add(kv11); 5952 region.put(put); 5953 put = new Put(rowA); 5954 put.add(kv2); 5955 region.put(put); 5956 put = new Put(rowB); 5957 put.add(kv3); 5958 region.put(put); 5959 5960 Scan scan = new Scan().withStartRow(rowD); 5961 List<Cell> currRow = new ArrayList<>(); 5962 scan.setReversed(true); 5963 scan.readVersions(5); 5964 InternalScanner scanner = region.getScanner(scan); 5965 boolean hasNext = scanner.next(currRow); 5966 assertEquals(2, currRow.size()); 5967 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5968 currRow.get(0).getRowLength(), rowC, 0, rowC.length)); 5969 assertTrue(hasNext); 5970 currRow.clear(); 5971 hasNext = scanner.next(currRow); 5972 assertEquals(1, currRow.size()); 5973 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5974 currRow.get(0).getRowLength(), rowB, 0, rowB.length)); 5975 assertTrue(hasNext); 5976 currRow.clear(); 5977 hasNext = scanner.next(currRow); 5978 assertEquals(1, currRow.size()); 5979 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 5980 currRow.get(0).getRowLength(), rowA, 0, rowA.length)); 5981 assertFalse(hasNext); 5982 scanner.close(); 5983 } 5984 5985 @Test 5986 public void testReverseScanner_FromMemStore_SingleCF_FullScan() throws IOException { 5987 byte[] rowC = Bytes.toBytes("rowC"); 5988 byte[] rowA = Bytes.toBytes("rowA"); 5989 byte[] rowB = Bytes.toBytes("rowB"); 5990 byte[] cf = Bytes.toBytes("CF"); 5991 byte[][] families = { cf }; 5992 byte[] col = Bytes.toBytes("C"); 5993 long ts = 1; 5994 this.region = initHRegion(tableName, method, families); 5995 KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null); 5996 KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, null); 5997 KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null); 5998 KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null); 5999 Put put = null; 6000 put = new Put(rowC); 6001 put.add(kv1); 6002 put.add(kv11); 6003 region.put(put); 6004 put = new Put(rowA); 6005 put.add(kv2); 6006 region.put(put); 6007 put = new Put(rowB); 6008 put.add(kv3); 6009 region.put(put); 6010 Scan scan = new Scan(); 6011 List<Cell> currRow = new ArrayList<>(); 6012 scan.setReversed(true); 6013 InternalScanner scanner = region.getScanner(scan); 6014 boolean hasNext = scanner.next(currRow); 6015 assertEquals(1, currRow.size()); 6016 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6017 currRow.get(0).getRowLength(), rowC, 0, rowC.length)); 6018 assertTrue(hasNext); 6019 currRow.clear(); 6020 hasNext = scanner.next(currRow); 6021 assertEquals(1, currRow.size()); 6022 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6023 currRow.get(0).getRowLength(), rowB, 0, rowB.length)); 6024 assertTrue(hasNext); 6025 currRow.clear(); 6026 hasNext = scanner.next(currRow); 6027 assertEquals(1, currRow.size()); 6028 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6029 currRow.get(0).getRowLength(), rowA, 0, rowA.length)); 6030 assertFalse(hasNext); 6031 scanner.close(); 6032 } 6033 6034 @Test 6035 public void testReverseScanner_moreRowsMayExistAfter() throws IOException { 6036 // case for "INCLUDE_AND_SEEK_NEXT_ROW & SEEK_NEXT_ROW" endless loop 6037 byte[] rowA = Bytes.toBytes("rowA"); 6038 byte[] rowB = Bytes.toBytes("rowB"); 6039 byte[] rowC = Bytes.toBytes("rowC"); 6040 byte[] rowD = Bytes.toBytes("rowD"); 6041 byte[] rowE = Bytes.toBytes("rowE"); 6042 byte[] cf = Bytes.toBytes("CF"); 6043 byte[][] families = { cf }; 6044 byte[] col1 = Bytes.toBytes("col1"); 6045 byte[] col2 = Bytes.toBytes("col2"); 6046 long ts = 1; 6047 this.region = initHRegion(tableName, method, families); 6048 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null); 6049 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null); 6050 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null); 6051 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null); 6052 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null); 6053 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null); 6054 Put put = null; 6055 put = new Put(rowA); 6056 put.add(kv1); 6057 region.put(put); 6058 put = new Put(rowB); 6059 put.add(kv2); 6060 region.put(put); 6061 put = new Put(rowC); 6062 put.add(kv3); 6063 region.put(put); 6064 put = new Put(rowD); 6065 put.add(kv4_1); 6066 region.put(put); 6067 put = new Put(rowD); 6068 put.add(kv4_2); 6069 region.put(put); 6070 put = new Put(rowE); 6071 put.add(kv5); 6072 region.put(put); 6073 region.flush(true); 6074 Scan scan = new Scan().withStartRow(rowD).withStopRow(rowA); 6075 scan.addColumn(families[0], col1); 6076 scan.setReversed(true); 6077 List<Cell> currRow = new ArrayList<>(); 6078 InternalScanner scanner = region.getScanner(scan); 6079 boolean hasNext = scanner.next(currRow); 6080 assertEquals(1, currRow.size()); 6081 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6082 currRow.get(0).getRowLength(), rowD, 0, rowD.length)); 6083 assertTrue(hasNext); 6084 currRow.clear(); 6085 hasNext = scanner.next(currRow); 6086 assertEquals(1, currRow.size()); 6087 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6088 currRow.get(0).getRowLength(), rowC, 0, rowC.length)); 6089 assertTrue(hasNext); 6090 currRow.clear(); 6091 hasNext = scanner.next(currRow); 6092 assertEquals(1, currRow.size()); 6093 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6094 currRow.get(0).getRowLength(), rowB, 0, rowB.length)); 6095 assertFalse(hasNext); 6096 scanner.close(); 6097 6098 scan = new Scan().withStartRow(rowD).withStopRow(rowA); 6099 scan.addColumn(families[0], col2); 6100 scan.setReversed(true); 6101 currRow.clear(); 6102 scanner = region.getScanner(scan); 6103 hasNext = scanner.next(currRow); 6104 assertEquals(1, currRow.size()); 6105 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6106 currRow.get(0).getRowLength(), rowD, 0, rowD.length)); 6107 scanner.close(); 6108 } 6109 6110 @Test 6111 public void testReverseScanner_smaller_blocksize() throws IOException { 6112 // case to ensure no conflict with HFile index optimization 6113 byte[] rowA = Bytes.toBytes("rowA"); 6114 byte[] rowB = Bytes.toBytes("rowB"); 6115 byte[] rowC = Bytes.toBytes("rowC"); 6116 byte[] rowD = Bytes.toBytes("rowD"); 6117 byte[] rowE = Bytes.toBytes("rowE"); 6118 byte[] cf = Bytes.toBytes("CF"); 6119 byte[][] families = { cf }; 6120 byte[] col1 = Bytes.toBytes("col1"); 6121 byte[] col2 = Bytes.toBytes("col2"); 6122 long ts = 1; 6123 Configuration conf = new Configuration(CONF); 6124 conf.setInt("test.block.size", 1); 6125 this.region = initHRegion(tableName, method, conf, families); 6126 KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null); 6127 KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null); 6128 KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null); 6129 KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null); 6130 KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null); 6131 KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null); 6132 Put put = null; 6133 put = new Put(rowA); 6134 put.add(kv1); 6135 region.put(put); 6136 put = new Put(rowB); 6137 put.add(kv2); 6138 region.put(put); 6139 put = new Put(rowC); 6140 put.add(kv3); 6141 region.put(put); 6142 put = new Put(rowD); 6143 put.add(kv4_1); 6144 region.put(put); 6145 put = new Put(rowD); 6146 put.add(kv4_2); 6147 region.put(put); 6148 put = new Put(rowE); 6149 put.add(kv5); 6150 region.put(put); 6151 region.flush(true); 6152 Scan scan = new Scan().withStartRow(rowD).withStopRow(rowA); 6153 scan.addColumn(families[0], col1); 6154 scan.setReversed(true); 6155 List<Cell> currRow = new ArrayList<>(); 6156 InternalScanner scanner = region.getScanner(scan); 6157 boolean hasNext = scanner.next(currRow); 6158 assertEquals(1, currRow.size()); 6159 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6160 currRow.get(0).getRowLength(), rowD, 0, rowD.length)); 6161 assertTrue(hasNext); 6162 currRow.clear(); 6163 hasNext = scanner.next(currRow); 6164 assertEquals(1, currRow.size()); 6165 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6166 currRow.get(0).getRowLength(), rowC, 0, rowC.length)); 6167 assertTrue(hasNext); 6168 currRow.clear(); 6169 hasNext = scanner.next(currRow); 6170 assertEquals(1, currRow.size()); 6171 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6172 currRow.get(0).getRowLength(), rowB, 0, rowB.length)); 6173 assertFalse(hasNext); 6174 scanner.close(); 6175 6176 scan = new Scan().withStartRow(rowD).withStopRow(rowA); 6177 scan.addColumn(families[0], col2); 6178 scan.setReversed(true); 6179 currRow.clear(); 6180 scanner = region.getScanner(scan); 6181 hasNext = scanner.next(currRow); 6182 assertEquals(1, currRow.size()); 6183 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6184 currRow.get(0).getRowLength(), rowD, 0, rowD.length)); 6185 scanner.close(); 6186 } 6187 6188 @Test 6189 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1() throws IOException { 6190 byte[] row0 = Bytes.toBytes("row0"); // 1 kv 6191 byte[] row1 = Bytes.toBytes("row1"); // 2 kv 6192 byte[] row2 = Bytes.toBytes("row2"); // 4 kv 6193 byte[] row3 = Bytes.toBytes("row3"); // 2 kv 6194 byte[] row4 = Bytes.toBytes("row4"); // 5 kv 6195 byte[] row5 = Bytes.toBytes("row5"); // 2 kv 6196 byte[] cf1 = Bytes.toBytes("CF1"); 6197 byte[] cf2 = Bytes.toBytes("CF2"); 6198 byte[] cf3 = Bytes.toBytes("CF3"); 6199 byte[][] families = { cf1, cf2, cf3 }; 6200 byte[] col = Bytes.toBytes("C"); 6201 long ts = 1; 6202 Configuration conf = new Configuration(CONF); 6203 // disable compactions in this test. 6204 conf.setInt("hbase.hstore.compactionThreshold", 10000); 6205 this.region = initHRegion(tableName, method, conf, families); 6206 // kv naming style: kv(row number) totalKvCountInThisRow seq no 6207 KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put, null); 6208 KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put, null); 6209 KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1, KeyValue.Type.Put, null); 6210 KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null); 6211 KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put, null); 6212 KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put, null); 6213 KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4, KeyValue.Type.Put, null); 6214 KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put, null); 6215 KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4, KeyValue.Type.Put, null); 6216 KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put, null); 6217 KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put, null); 6218 KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5, KeyValue.Type.Put, null); 6219 KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put, null); 6220 KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3, KeyValue.Type.Put, null); 6221 KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put, null); 6222 KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put, null); 6223 // hfiles(cf1/cf2) :"row1"(1 kv) / "row2"(1 kv) / "row4"(2 kv) 6224 Put put = null; 6225 put = new Put(row1); 6226 put.add(kv1_2_1); 6227 region.put(put); 6228 put = new Put(row2); 6229 put.add(kv2_4_1); 6230 region.put(put); 6231 put = new Put(row4); 6232 put.add(kv4_5_4); 6233 put.add(kv4_5_5); 6234 region.put(put); 6235 region.flush(true); 6236 // hfiles(cf1/cf3) : "row1" (1 kvs) / "row2" (1 kv) / "row4" (2 kv) 6237 put = new Put(row4); 6238 put.add(kv4_5_1); 6239 put.add(kv4_5_3); 6240 region.put(put); 6241 put = new Put(row1); 6242 put.add(kv1_2_2); 6243 region.put(put); 6244 put = new Put(row2); 6245 put.add(kv2_4_4); 6246 region.put(put); 6247 region.flush(true); 6248 // hfiles(cf1/cf3) : "row2"(2 kv) / "row3"(1 kvs) / "row4" (1 kv) 6249 put = new Put(row4); 6250 put.add(kv4_5_2); 6251 region.put(put); 6252 put = new Put(row2); 6253 put.add(kv2_4_2); 6254 put.add(kv2_4_3); 6255 region.put(put); 6256 put = new Put(row3); 6257 put.add(kv3_2_2); 6258 region.put(put); 6259 region.flush(true); 6260 // memstore(cf1/cf2/cf3) : "row0" (1 kvs) / "row3" ( 1 kv) / "row5" (max) 6261 // ( 2 kv) 6262 put = new Put(row0); 6263 put.add(kv0_1_1); 6264 region.put(put); 6265 put = new Put(row3); 6266 put.add(kv3_2_1); 6267 region.put(put); 6268 put = new Put(row5); 6269 put.add(kv5_2_1); 6270 put.add(kv5_2_2); 6271 region.put(put); 6272 // scan range = ["row4", min), skip the max "row5" 6273 Scan scan = new Scan().withStartRow(row4); 6274 scan.readVersions(5); 6275 scan.setBatch(3); 6276 scan.setReversed(true); 6277 InternalScanner scanner = region.getScanner(scan); 6278 List<Cell> currRow = new ArrayList<>(); 6279 boolean hasNext = false; 6280 // 1. scan out "row4" (5 kvs), "row5" can't be scanned out since not 6281 // included in scan range 6282 // "row4" takes 2 next() calls since batch=3 6283 hasNext = scanner.next(currRow); 6284 assertEquals(3, currRow.size()); 6285 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6286 currRow.get(0).getRowLength(), row4, 0, row4.length)); 6287 assertTrue(hasNext); 6288 currRow.clear(); 6289 hasNext = scanner.next(currRow); 6290 assertEquals(2, currRow.size()); 6291 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6292 currRow.get(0).getRowLength(), row4, 0, row4.length)); 6293 assertTrue(hasNext); 6294 // 2. scan out "row3" (2 kv) 6295 currRow.clear(); 6296 hasNext = scanner.next(currRow); 6297 assertEquals(2, currRow.size()); 6298 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6299 currRow.get(0).getRowLength(), row3, 0, row3.length)); 6300 assertTrue(hasNext); 6301 // 3. scan out "row2" (4 kvs) 6302 // "row2" takes 2 next() calls since batch=3 6303 currRow.clear(); 6304 hasNext = scanner.next(currRow); 6305 assertEquals(3, currRow.size()); 6306 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6307 currRow.get(0).getRowLength(), row2, 0, row2.length)); 6308 assertTrue(hasNext); 6309 currRow.clear(); 6310 hasNext = scanner.next(currRow); 6311 assertEquals(1, currRow.size()); 6312 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6313 currRow.get(0).getRowLength(), row2, 0, row2.length)); 6314 assertTrue(hasNext); 6315 // 4. scan out "row1" (2 kv) 6316 currRow.clear(); 6317 hasNext = scanner.next(currRow); 6318 assertEquals(2, currRow.size()); 6319 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6320 currRow.get(0).getRowLength(), row1, 0, row1.length)); 6321 assertTrue(hasNext); 6322 // 5. scan out "row0" (1 kv) 6323 currRow.clear(); 6324 hasNext = scanner.next(currRow); 6325 assertEquals(1, currRow.size()); 6326 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6327 currRow.get(0).getRowLength(), row0, 0, row0.length)); 6328 assertFalse(hasNext); 6329 6330 scanner.close(); 6331 } 6332 6333 @Test 6334 public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2() throws IOException { 6335 byte[] row1 = Bytes.toBytes("row1"); 6336 byte[] row2 = Bytes.toBytes("row2"); 6337 byte[] row3 = Bytes.toBytes("row3"); 6338 byte[] row4 = Bytes.toBytes("row4"); 6339 byte[] cf1 = Bytes.toBytes("CF1"); 6340 byte[] cf2 = Bytes.toBytes("CF2"); 6341 byte[] cf3 = Bytes.toBytes("CF3"); 6342 byte[] cf4 = Bytes.toBytes("CF4"); 6343 byte[][] families = { cf1, cf2, cf3, cf4 }; 6344 byte[] col = Bytes.toBytes("C"); 6345 long ts = 1; 6346 Configuration conf = new Configuration(CONF); 6347 // disable compactions in this test. 6348 conf.setInt("hbase.hstore.compactionThreshold", 10000); 6349 this.region = initHRegion(tableName, method, conf, families); 6350 KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null); 6351 KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null); 6352 KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null); 6353 KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null); 6354 // storefile1 6355 Put put = new Put(row1); 6356 put.add(kv1); 6357 region.put(put); 6358 region.flush(true); 6359 // storefile2 6360 put = new Put(row2); 6361 put.add(kv2); 6362 region.put(put); 6363 region.flush(true); 6364 // storefile3 6365 put = new Put(row3); 6366 put.add(kv3); 6367 region.put(put); 6368 region.flush(true); 6369 // memstore 6370 put = new Put(row4); 6371 put.add(kv4); 6372 region.put(put); 6373 // scan range = ["row4", min) 6374 Scan scan = new Scan().withStartRow(row4); 6375 scan.setReversed(true); 6376 scan.setBatch(10); 6377 InternalScanner scanner = region.getScanner(scan); 6378 List<Cell> currRow = new ArrayList<>(); 6379 boolean hasNext = scanner.next(currRow); 6380 assertEquals(1, currRow.size()); 6381 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6382 currRow.get(0).getRowLength(), row4, 0, row4.length)); 6383 assertTrue(hasNext); 6384 currRow.clear(); 6385 hasNext = scanner.next(currRow); 6386 assertEquals(1, currRow.size()); 6387 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6388 currRow.get(0).getRowLength(), row3, 0, row3.length)); 6389 assertTrue(hasNext); 6390 currRow.clear(); 6391 hasNext = scanner.next(currRow); 6392 assertEquals(1, currRow.size()); 6393 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6394 currRow.get(0).getRowLength(), row2, 0, row2.length)); 6395 assertTrue(hasNext); 6396 currRow.clear(); 6397 hasNext = scanner.next(currRow); 6398 assertEquals(1, currRow.size()); 6399 assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(), 6400 currRow.get(0).getRowLength(), row1, 0, row1.length)); 6401 assertFalse(hasNext); 6402 } 6403 6404 /** 6405 * Test for HBASE-14497: Reverse Scan threw StackOverflow caused by readPt checking 6406 */ 6407 @Test 6408 public void testReverseScanner_StackOverflow() throws IOException { 6409 byte[] cf1 = Bytes.toBytes("CF1"); 6410 byte[][] families = { cf1 }; 6411 byte[] col = Bytes.toBytes("C"); 6412 Configuration conf = new Configuration(CONF); 6413 this.region = initHRegion(tableName, method, conf, families); 6414 // setup with one storefile and one memstore, to create scanner and get an earlier readPt 6415 Put put = new Put(Bytes.toBytes("19998")); 6416 put.addColumn(cf1, col, Bytes.toBytes("val")); 6417 region.put(put); 6418 region.flushcache(true, true, FlushLifeCycleTracker.DUMMY); 6419 Put put2 = new Put(Bytes.toBytes("19997")); 6420 put2.addColumn(cf1, col, Bytes.toBytes("val")); 6421 region.put(put2); 6422 6423 Scan scan = new Scan().withStartRow(Bytes.toBytes("19998")); 6424 scan.setReversed(true); 6425 InternalScanner scanner = region.getScanner(scan); 6426 6427 // create one storefile contains many rows will be skipped 6428 // to check StoreFileScanner.seekToPreviousRow 6429 for (int i = 10000; i < 20000; i++) { 6430 Put p = new Put(Bytes.toBytes("" + i)); 6431 p.addColumn(cf1, col, Bytes.toBytes("" + i)); 6432 region.put(p); 6433 } 6434 region.flushcache(true, true, FlushLifeCycleTracker.DUMMY); 6435 6436 // create one memstore contains many rows will be skipped 6437 // to check MemStoreScanner.seekToPreviousRow 6438 for (int i = 10000; i < 20000; i++) { 6439 Put p = new Put(Bytes.toBytes("" + i)); 6440 p.addColumn(cf1, col, Bytes.toBytes("" + i)); 6441 region.put(p); 6442 } 6443 6444 List<Cell> currRow = new ArrayList<>(); 6445 boolean hasNext; 6446 do { 6447 hasNext = scanner.next(currRow); 6448 } while (hasNext); 6449 assertEquals(2, currRow.size()); 6450 assertEquals("19998", Bytes.toString(currRow.get(0).getRowArray(), 6451 currRow.get(0).getRowOffset(), currRow.get(0).getRowLength())); 6452 assertEquals("19997", Bytes.toString(currRow.get(1).getRowArray(), 6453 currRow.get(1).getRowOffset(), currRow.get(1).getRowLength())); 6454 } 6455 6456 @Test 6457 public void testReverseScanShouldNotScanMemstoreIfReadPtLesser() throws Exception { 6458 byte[] cf1 = Bytes.toBytes("CF1"); 6459 byte[][] families = { cf1 }; 6460 byte[] col = Bytes.toBytes("C"); 6461 this.region = initHRegion(tableName, method, CONF, families); 6462 // setup with one storefile and one memstore, to create scanner and get an earlier readPt 6463 Put put = new Put(Bytes.toBytes("19996")); 6464 put.addColumn(cf1, col, Bytes.toBytes("val")); 6465 region.put(put); 6466 Put put2 = new Put(Bytes.toBytes("19995")); 6467 put2.addColumn(cf1, col, Bytes.toBytes("val")); 6468 region.put(put2); 6469 // create a reverse scan 6470 Scan scan = new Scan().withStartRow(Bytes.toBytes("19996")); 6471 scan.setReversed(true); 6472 RegionScannerImpl scanner = region.getScanner(scan); 6473 6474 // flush the cache. This will reset the store scanner 6475 region.flushcache(true, true, FlushLifeCycleTracker.DUMMY); 6476 6477 // create one memstore contains many rows will be skipped 6478 // to check MemStoreScanner.seekToPreviousRow 6479 for (int i = 10000; i < 20000; i++) { 6480 Put p = new Put(Bytes.toBytes("" + i)); 6481 p.addColumn(cf1, col, Bytes.toBytes("" + i)); 6482 region.put(p); 6483 } 6484 List<Cell> currRow = new ArrayList<>(); 6485 boolean hasNext; 6486 boolean assertDone = false; 6487 do { 6488 hasNext = scanner.next(currRow); 6489 // With HBASE-15871, after the scanner is reset the memstore scanner should not be 6490 // added here 6491 if (!assertDone) { 6492 StoreScanner current = (StoreScanner) (scanner.storeHeap).getCurrentForTesting(); 6493 List<KeyValueScanner> scanners = current.getAllScannersForTesting(); 6494 assertEquals("There should be only one scanner the store file scanner", 1, scanners.size()); 6495 assertDone = true; 6496 } 6497 } while (hasNext); 6498 assertEquals(2, currRow.size()); 6499 assertEquals("19996", Bytes.toString(currRow.get(0).getRowArray(), 6500 currRow.get(0).getRowOffset(), currRow.get(0).getRowLength())); 6501 assertEquals("19995", Bytes.toString(currRow.get(1).getRowArray(), 6502 currRow.get(1).getRowOffset(), currRow.get(1).getRowLength())); 6503 } 6504 6505 @Test 6506 public void testReverseScanWhenPutCellsAfterOpenReverseScan() throws Exception { 6507 byte[] cf1 = Bytes.toBytes("CF1"); 6508 byte[][] families = { cf1 }; 6509 byte[] col = Bytes.toBytes("C"); 6510 6511 this.region = initHRegion(tableName, method, CONF, families); 6512 6513 Put put = new Put(Bytes.toBytes("199996")); 6514 put.addColumn(cf1, col, Bytes.toBytes("val")); 6515 region.put(put); 6516 Put put2 = new Put(Bytes.toBytes("199995")); 6517 put2.addColumn(cf1, col, Bytes.toBytes("val")); 6518 region.put(put2); 6519 6520 // Create a reverse scan 6521 Scan scan = new Scan().withStartRow(Bytes.toBytes("199996")); 6522 scan.setReversed(true); 6523 RegionScannerImpl scanner = region.getScanner(scan); 6524 6525 // Put a lot of cells that have sequenceIDs grater than the readPt of the reverse scan 6526 for (int i = 100000; i < 200000; i++) { 6527 Put p = new Put(Bytes.toBytes("" + i)); 6528 p.addColumn(cf1, col, Bytes.toBytes("" + i)); 6529 region.put(p); 6530 } 6531 List<Cell> currRow = new ArrayList<>(); 6532 boolean hasNext; 6533 do { 6534 hasNext = scanner.next(currRow); 6535 } while (hasNext); 6536 6537 assertEquals(2, currRow.size()); 6538 assertEquals("199996", Bytes.toString(currRow.get(0).getRowArray(), 6539 currRow.get(0).getRowOffset(), currRow.get(0).getRowLength())); 6540 assertEquals("199995", Bytes.toString(currRow.get(1).getRowArray(), 6541 currRow.get(1).getRowOffset(), currRow.get(1).getRowLength())); 6542 } 6543 6544 @Test 6545 public void testWriteRequestsCounter() throws IOException { 6546 byte[] fam = Bytes.toBytes("info"); 6547 byte[][] families = { fam }; 6548 this.region = initHRegion(tableName, method, CONF, families); 6549 6550 Assert.assertEquals(0L, region.getWriteRequestsCount()); 6551 6552 Put put = new Put(row); 6553 put.addColumn(fam, fam, fam); 6554 6555 Assert.assertEquals(0L, region.getWriteRequestsCount()); 6556 region.put(put); 6557 Assert.assertEquals(1L, region.getWriteRequestsCount()); 6558 region.put(put); 6559 Assert.assertEquals(2L, region.getWriteRequestsCount()); 6560 region.put(put); 6561 Assert.assertEquals(3L, region.getWriteRequestsCount()); 6562 6563 region.delete(new Delete(row)); 6564 Assert.assertEquals(4L, region.getWriteRequestsCount()); 6565 } 6566 6567 @Test 6568 public void testOpenRegionWrittenToWAL() throws Exception { 6569 final ServerName serverName = ServerName.valueOf(name.getMethodName(), 100, 42); 6570 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName)); 6571 6572 TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) 6573 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)) 6574 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam2)).build(); 6575 RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build(); 6576 6577 // open the region w/o rss and wal and flush some files 6578 region = HBaseTestingUtil.createRegionAndWAL(hri, TEST_UTIL.getDataTestDir(), 6579 TEST_UTIL.getConfiguration(), htd); 6580 assertNotNull(region); 6581 6582 // create a file in fam1 for the region before opening in OpenRegionHandler 6583 region.put(new Put(Bytes.toBytes("a")).addColumn(fam1, fam1, fam1)); 6584 region.flush(true); 6585 HBaseTestingUtil.closeRegionAndWAL(region); 6586 6587 ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class); 6588 6589 // capture append() calls 6590 WAL wal = mockWAL(); 6591 when(rss.getWAL(any(RegionInfo.class))).thenReturn(wal); 6592 6593 region = 6594 HRegion.openHRegion(hri, htd, rss.getWAL(hri), TEST_UTIL.getConfiguration(), rss, null); 6595 6596 verify(wal, times(1)).appendMarker(any(RegionInfo.class), any(WALKeyImpl.class), 6597 editCaptor.capture()); 6598 6599 WALEdit edit = editCaptor.getValue(); 6600 assertNotNull(edit); 6601 assertNotNull(edit.getCells()); 6602 assertEquals(1, edit.getCells().size()); 6603 RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0)); 6604 assertNotNull(desc); 6605 6606 LOG.info("RegionEventDescriptor from WAL: " + desc); 6607 6608 assertEquals(RegionEventDescriptor.EventType.REGION_OPEN, desc.getEventType()); 6609 assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getTableName().toBytes())); 6610 assertTrue( 6611 Bytes.equals(desc.getEncodedRegionName().toByteArray(), hri.getEncodedNameAsBytes())); 6612 assertTrue(desc.getLogSequenceNumber() > 0); 6613 assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer())); 6614 assertEquals(2, desc.getStoresCount()); 6615 6616 StoreDescriptor store = desc.getStores(0); 6617 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1)); 6618 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1)); 6619 assertEquals(1, store.getStoreFileCount()); // 1store file 6620 assertFalse(store.getStoreFile(0).contains("/")); // ensure path is relative 6621 6622 store = desc.getStores(1); 6623 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2)); 6624 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2)); 6625 assertEquals(0, store.getStoreFileCount()); // no store files 6626 } 6627 6628 // Helper for test testOpenRegionWrittenToWALForLogReplay 6629 static class HRegionWithSeqId extends HRegion { 6630 public HRegionWithSeqId(final Path tableDir, final WAL wal, final FileSystem fs, 6631 final Configuration confParam, final RegionInfo regionInfo, final TableDescriptor htd, 6632 final RegionServerServices rsServices) { 6633 super(tableDir, wal, fs, confParam, regionInfo, htd, rsServices); 6634 } 6635 6636 @Override 6637 protected long getNextSequenceId(WAL wal) throws IOException { 6638 return 42; 6639 } 6640 } 6641 6642 @Test 6643 public void testFlushedFileWithNoTags() throws Exception { 6644 final TableName tableName = TableName.valueOf(name.getMethodName()); 6645 TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName) 6646 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)).build(); 6647 RegionInfo info = RegionInfoBuilder.newBuilder(tableName).build(); 6648 Path path = TEST_UTIL.getDataTestDir(getClass().getSimpleName()); 6649 region = HBaseTestingUtil.createRegionAndWAL(info, path, TEST_UTIL.getConfiguration(), 6650 tableDescriptor); 6651 Put put = new Put(Bytes.toBytes("a-b-0-0")); 6652 put.addColumn(fam1, qual1, Bytes.toBytes("c1-value")); 6653 region.put(put); 6654 region.flush(true); 6655 HStore store = region.getStore(fam1); 6656 Collection<HStoreFile> storefiles = store.getStorefiles(); 6657 for (HStoreFile sf : storefiles) { 6658 assertFalse("Tags should not be present ", 6659 sf.getReader().getHFileReader().getFileContext().isIncludesTags()); 6660 } 6661 } 6662 6663 /** 6664 * Utility method to setup a WAL mock. 6665 * <p/> 6666 * Needs to do the bit where we close latch on the WALKeyImpl on append else test hangs. 6667 * @return a mock WAL 6668 */ 6669 private WAL mockWAL() throws IOException { 6670 WAL wal = mock(WAL.class); 6671 when(wal.appendData(any(RegionInfo.class), any(WALKeyImpl.class), any(WALEdit.class))) 6672 .thenAnswer(new Answer<Long>() { 6673 @Override 6674 public Long answer(InvocationOnMock invocation) throws Throwable { 6675 WALKeyImpl key = invocation.getArgument(1); 6676 MultiVersionConcurrencyControl.WriteEntry we = key.getMvcc().begin(); 6677 key.setWriteEntry(we); 6678 return 1L; 6679 } 6680 }); 6681 when(wal.appendMarker(any(RegionInfo.class), any(WALKeyImpl.class), any(WALEdit.class))) 6682 .thenAnswer(new Answer<Long>() { 6683 @Override 6684 public Long answer(InvocationOnMock invocation) throws Throwable { 6685 WALKeyImpl key = invocation.getArgument(1); 6686 MultiVersionConcurrencyControl.WriteEntry we = key.getMvcc().begin(); 6687 key.setWriteEntry(we); 6688 return 1L; 6689 } 6690 }); 6691 return wal; 6692 } 6693 6694 @Test 6695 public void testCloseRegionWrittenToWAL() throws Exception { 6696 Path rootDir = new Path(dir + name.getMethodName()); 6697 CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir); 6698 6699 final ServerName serverName = ServerName.valueOf("testCloseRegionWrittenToWAL", 100, 42); 6700 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName)); 6701 6702 TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) 6703 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)) 6704 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam2)).build(); 6705 RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build(); 6706 6707 ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class); 6708 6709 // capture append() calls 6710 WAL wal = mockWAL(); 6711 when(rss.getWAL(any(RegionInfo.class))).thenReturn(wal); 6712 6713 // create and then open a region first so that it can be closed later 6714 region = 6715 HRegion.createHRegion(hri, rootDir, TEST_UTIL.getConfiguration(), htd, rss.getWAL(hri)); 6716 region = 6717 HRegion.openHRegion(hri, htd, rss.getWAL(hri), TEST_UTIL.getConfiguration(), rss, null); 6718 6719 // close the region 6720 region.close(false); 6721 6722 // 2 times, one for region open, the other close region 6723 verify(wal, times(2)).appendMarker(any(RegionInfo.class), (WALKeyImpl) any(WALKeyImpl.class), 6724 editCaptor.capture()); 6725 6726 WALEdit edit = editCaptor.getAllValues().get(1); 6727 assertNotNull(edit); 6728 assertNotNull(edit.getCells()); 6729 assertEquals(1, edit.getCells().size()); 6730 RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0)); 6731 assertNotNull(desc); 6732 6733 LOG.info("RegionEventDescriptor from WAL: " + desc); 6734 6735 assertEquals(RegionEventDescriptor.EventType.REGION_CLOSE, desc.getEventType()); 6736 assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getTableName().toBytes())); 6737 assertTrue( 6738 Bytes.equals(desc.getEncodedRegionName().toByteArray(), hri.getEncodedNameAsBytes())); 6739 assertTrue(desc.getLogSequenceNumber() > 0); 6740 assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer())); 6741 assertEquals(2, desc.getStoresCount()); 6742 6743 StoreDescriptor store = desc.getStores(0); 6744 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1)); 6745 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1)); 6746 assertEquals(0, store.getStoreFileCount()); // no store files 6747 6748 store = desc.getStores(1); 6749 assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2)); 6750 assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2)); 6751 assertEquals(0, store.getStoreFileCount()); // no store files 6752 } 6753 6754 /** 6755 * Test RegionTooBusyException thrown when region is busy 6756 */ 6757 @Test 6758 public void testRegionTooBusy() throws IOException { 6759 byte[] family = Bytes.toBytes("family"); 6760 long defaultBusyWaitDuration = 6761 CONF.getLong("hbase.busy.wait.duration", HRegion.DEFAULT_BUSY_WAIT_DURATION); 6762 CONF.setLong("hbase.busy.wait.duration", 1000); 6763 region = initHRegion(tableName, method, CONF, family); 6764 final AtomicBoolean stopped = new AtomicBoolean(true); 6765 Thread t = new Thread(new Runnable() { 6766 @Override 6767 public void run() { 6768 try { 6769 region.lock.writeLock().lock(); 6770 stopped.set(false); 6771 while (!stopped.get()) { 6772 Thread.sleep(100); 6773 } 6774 } catch (InterruptedException ie) { 6775 } finally { 6776 region.lock.writeLock().unlock(); 6777 } 6778 } 6779 }); 6780 t.start(); 6781 Get get = new Get(row); 6782 try { 6783 while (stopped.get()) { 6784 Thread.sleep(100); 6785 } 6786 region.get(get); 6787 fail("Should throw RegionTooBusyException"); 6788 } catch (InterruptedException ie) { 6789 fail("test interrupted"); 6790 } catch (RegionTooBusyException e) { 6791 // Good, expected 6792 } finally { 6793 stopped.set(true); 6794 try { 6795 t.join(); 6796 } catch (Throwable e) { 6797 } 6798 6799 HBaseTestingUtil.closeRegionAndWAL(region); 6800 region = null; 6801 CONF.setLong("hbase.busy.wait.duration", defaultBusyWaitDuration); 6802 } 6803 } 6804 6805 @Test 6806 public void testCellTTLs() throws IOException { 6807 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge(); 6808 EnvironmentEdgeManager.injectEdge(edge); 6809 6810 final byte[] row = Bytes.toBytes("testRow"); 6811 final byte[] q1 = Bytes.toBytes("q1"); 6812 final byte[] q2 = Bytes.toBytes("q2"); 6813 final byte[] q3 = Bytes.toBytes("q3"); 6814 final byte[] q4 = Bytes.toBytes("q4"); 6815 6816 // 10 seconds 6817 TableDescriptor tableDescriptor = 6818 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) 6819 .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam1).setTimeToLive(10).build()) 6820 .build(); 6821 6822 Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); 6823 conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS); 6824 6825 region = HBaseTestingUtil.createRegionAndWAL( 6826 RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(), 6827 TEST_UTIL.getDataTestDir(), conf, tableDescriptor); 6828 assertNotNull(region); 6829 long now = EnvironmentEdgeManager.currentTime(); 6830 // Add a cell that will expire in 5 seconds via cell TTL 6831 region.put(new Put(row).add(new KeyValue(row, fam1, q1, now, HConstants.EMPTY_BYTE_ARRAY, 6832 new ArrayBackedTag[] { 6833 // TTL tags specify ts in milliseconds 6834 new ArrayBackedTag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) }))); 6835 // Add a cell that will expire after 10 seconds via family setting 6836 region.put(new Put(row).addColumn(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY)); 6837 // Add a cell that will expire in 15 seconds via cell TTL 6838 region.put(new Put(row).add(new KeyValue(row, fam1, q3, now + 10000 - 1, 6839 HConstants.EMPTY_BYTE_ARRAY, new ArrayBackedTag[] { 6840 // TTL tags specify ts in milliseconds 6841 new ArrayBackedTag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) }))); 6842 // Add a cell that will expire in 20 seconds via family setting 6843 region.put(new Put(row).addColumn(fam1, q4, now + 10000 - 1, HConstants.EMPTY_BYTE_ARRAY)); 6844 6845 // Flush so we are sure store scanning gets this right 6846 region.flush(true); 6847 6848 // A query at time T+0 should return all cells 6849 Result r = region.get(new Get(row)); 6850 assertNotNull(r.getValue(fam1, q1)); 6851 assertNotNull(r.getValue(fam1, q2)); 6852 assertNotNull(r.getValue(fam1, q3)); 6853 assertNotNull(r.getValue(fam1, q4)); 6854 6855 // Increment time to T+5 seconds 6856 edge.incrementTime(5000); 6857 6858 r = region.get(new Get(row)); 6859 assertNull(r.getValue(fam1, q1)); 6860 assertNotNull(r.getValue(fam1, q2)); 6861 assertNotNull(r.getValue(fam1, q3)); 6862 assertNotNull(r.getValue(fam1, q4)); 6863 6864 // Increment time to T+10 seconds 6865 edge.incrementTime(5000); 6866 6867 r = region.get(new Get(row)); 6868 assertNull(r.getValue(fam1, q1)); 6869 assertNull(r.getValue(fam1, q2)); 6870 assertNotNull(r.getValue(fam1, q3)); 6871 assertNotNull(r.getValue(fam1, q4)); 6872 6873 // Increment time to T+15 seconds 6874 edge.incrementTime(5000); 6875 6876 r = region.get(new Get(row)); 6877 assertNull(r.getValue(fam1, q1)); 6878 assertNull(r.getValue(fam1, q2)); 6879 assertNull(r.getValue(fam1, q3)); 6880 assertNotNull(r.getValue(fam1, q4)); 6881 6882 // Increment time to T+20 seconds 6883 edge.incrementTime(10000); 6884 6885 r = region.get(new Get(row)); 6886 assertNull(r.getValue(fam1, q1)); 6887 assertNull(r.getValue(fam1, q2)); 6888 assertNull(r.getValue(fam1, q3)); 6889 assertNull(r.getValue(fam1, q4)); 6890 6891 // Fun with disappearing increments 6892 6893 // Start at 1 6894 region.put(new Put(row).addColumn(fam1, q1, Bytes.toBytes(1L))); 6895 r = region.get(new Get(row)); 6896 byte[] val = r.getValue(fam1, q1); 6897 assertNotNull(val); 6898 assertEquals(1L, Bytes.toLong(val)); 6899 6900 // Increment with a TTL of 5 seconds 6901 Increment incr = new Increment(row).addColumn(fam1, q1, 1L); 6902 incr.setTTL(5000); 6903 region.increment(incr); // 2 6904 6905 // New value should be 2 6906 r = region.get(new Get(row)); 6907 val = r.getValue(fam1, q1); 6908 assertNotNull(val); 6909 assertEquals(2L, Bytes.toLong(val)); 6910 6911 // Increment time to T+25 seconds 6912 edge.incrementTime(5000); 6913 6914 // Value should be back to 1 6915 r = region.get(new Get(row)); 6916 val = r.getValue(fam1, q1); 6917 assertNotNull(val); 6918 assertEquals(1L, Bytes.toLong(val)); 6919 6920 // Increment time to T+30 seconds 6921 edge.incrementTime(5000); 6922 6923 // Original value written at T+20 should be gone now via family TTL 6924 r = region.get(new Get(row)); 6925 assertNull(r.getValue(fam1, q1)); 6926 } 6927 6928 @Test 6929 public void testTTLsUsingSmallHeartBeatCells() throws IOException { 6930 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge(); 6931 EnvironmentEdgeManager.injectEdge(edge); 6932 6933 final byte[] row = Bytes.toBytes("testRow"); 6934 final byte[] q1 = Bytes.toBytes("q1"); 6935 final byte[] q2 = Bytes.toBytes("q2"); 6936 final byte[] q3 = Bytes.toBytes("q3"); 6937 final byte[] q4 = Bytes.toBytes("q4"); 6938 final byte[] q5 = Bytes.toBytes("q5"); 6939 final byte[] q6 = Bytes.toBytes("q6"); 6940 final byte[] q7 = Bytes.toBytes("q7"); 6941 final byte[] q8 = Bytes.toBytes("q8"); 6942 6943 // 10 seconds 6944 int ttlSecs = 10; 6945 TableDescriptor tableDescriptor = 6946 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily( 6947 ColumnFamilyDescriptorBuilder.newBuilder(fam1).setTimeToLive(ttlSecs).build()).build(); 6948 6949 Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); 6950 conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS); 6951 // using small heart beat cells 6952 conf.setLong(StoreScanner.HBASE_CELLS_SCANNED_PER_HEARTBEAT_CHECK, 2); 6953 6954 region = HBaseTestingUtil.createRegionAndWAL( 6955 RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(), 6956 TEST_UTIL.getDataTestDir(), conf, tableDescriptor); 6957 assertNotNull(region); 6958 long now = EnvironmentEdgeManager.currentTime(); 6959 // Add a cell that will expire in 5 seconds via cell TTL 6960 region.put(new Put(row).addColumn(fam1, q1, now, HConstants.EMPTY_BYTE_ARRAY)); 6961 region.put(new Put(row).addColumn(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY)); 6962 region.put(new Put(row).addColumn(fam1, q3, now, HConstants.EMPTY_BYTE_ARRAY)); 6963 // Add a cell that will expire after 10 seconds via family setting 6964 region 6965 .put(new Put(row).addColumn(fam1, q4, now + ttlSecs * 1000 + 1, HConstants.EMPTY_BYTE_ARRAY)); 6966 region 6967 .put(new Put(row).addColumn(fam1, q5, now + ttlSecs * 1000 + 1, HConstants.EMPTY_BYTE_ARRAY)); 6968 6969 region.put(new Put(row).addColumn(fam1, q6, now, HConstants.EMPTY_BYTE_ARRAY)); 6970 region.put(new Put(row).addColumn(fam1, q7, now, HConstants.EMPTY_BYTE_ARRAY)); 6971 region 6972 .put(new Put(row).addColumn(fam1, q8, now + ttlSecs * 1000 + 1, HConstants.EMPTY_BYTE_ARRAY)); 6973 6974 // Flush so we are sure store scanning gets this right 6975 region.flush(true); 6976 6977 // A query at time T+0 should return all cells 6978 checkScan(8); 6979 region.delete(new Delete(row).addColumn(fam1, q8)); 6980 6981 // Increment time to T+ttlSecs seconds 6982 edge.incrementTime(ttlSecs * 1000); 6983 checkScan(2); 6984 } 6985 6986 private void checkScan(int expectCellSize) throws IOException { 6987 Scan s = new Scan().withStartRow(row); 6988 ScannerContext.Builder contextBuilder = ScannerContext.newBuilder(true); 6989 ScannerContext scannerContext = contextBuilder.build(); 6990 RegionScanner scanner = region.getScanner(s); 6991 List<Cell> kvs = new ArrayList<>(); 6992 scanner.next(kvs, scannerContext); 6993 assertEquals(expectCellSize, kvs.size()); 6994 scanner.close(); 6995 } 6996 6997 @Test 6998 public void testIncrementTimestampsAreMonotonic() throws IOException { 6999 region = initHRegion(tableName, method, CONF, fam1); 7000 ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); 7001 EnvironmentEdgeManager.injectEdge(edge); 7002 7003 edge.setValue(10); 7004 Increment inc = new Increment(row); 7005 inc.setDurability(Durability.SKIP_WAL); 7006 inc.addColumn(fam1, qual1, 1L); 7007 region.increment(inc); 7008 7009 Result result = region.get(new Get(row)); 7010 Cell c = result.getColumnLatestCell(fam1, qual1); 7011 assertNotNull(c); 7012 assertEquals(10L, c.getTimestamp()); 7013 7014 edge.setValue(1); // clock goes back 7015 region.increment(inc); 7016 result = region.get(new Get(row)); 7017 c = result.getColumnLatestCell(fam1, qual1); 7018 assertEquals(11L, c.getTimestamp()); 7019 assertEquals(2L, Bytes.toLong(c.getValueArray(), c.getValueOffset(), c.getValueLength())); 7020 } 7021 7022 @Test 7023 public void testAppendTimestampsAreMonotonic() throws IOException { 7024 region = initHRegion(tableName, method, CONF, fam1); 7025 ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); 7026 EnvironmentEdgeManager.injectEdge(edge); 7027 7028 edge.setValue(10); 7029 Append a = new Append(row); 7030 a.setDurability(Durability.SKIP_WAL); 7031 a.addColumn(fam1, qual1, qual1); 7032 region.append(a); 7033 7034 Result result = region.get(new Get(row)); 7035 Cell c = result.getColumnLatestCell(fam1, qual1); 7036 assertNotNull(c); 7037 assertEquals(10L, c.getTimestamp()); 7038 7039 edge.setValue(1); // clock goes back 7040 region.append(a); 7041 result = region.get(new Get(row)); 7042 c = result.getColumnLatestCell(fam1, qual1); 7043 assertEquals(11L, c.getTimestamp()); 7044 7045 byte[] expected = new byte[qual1.length * 2]; 7046 System.arraycopy(qual1, 0, expected, 0, qual1.length); 7047 System.arraycopy(qual1, 0, expected, qual1.length, qual1.length); 7048 7049 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(), expected, 0, 7050 expected.length)); 7051 } 7052 7053 @Test 7054 public void testCheckAndMutateTimestampsAreMonotonic() throws IOException { 7055 region = initHRegion(tableName, method, CONF, fam1); 7056 ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); 7057 EnvironmentEdgeManager.injectEdge(edge); 7058 7059 edge.setValue(10); 7060 Put p = new Put(row); 7061 p.setDurability(Durability.SKIP_WAL); 7062 p.addColumn(fam1, qual1, qual1); 7063 region.put(p); 7064 7065 Result result = region.get(new Get(row)); 7066 Cell c = result.getColumnLatestCell(fam1, qual1); 7067 assertNotNull(c); 7068 assertEquals(10L, c.getTimestamp()); 7069 7070 edge.setValue(1); // clock goes back 7071 p = new Put(row); 7072 p.setDurability(Durability.SKIP_WAL); 7073 p.addColumn(fam1, qual1, qual2); 7074 region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(qual1), p); 7075 result = region.get(new Get(row)); 7076 c = result.getColumnLatestCell(fam1, qual1); 7077 assertEquals(10L, c.getTimestamp()); 7078 7079 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(), qual2, 0, 7080 qual2.length)); 7081 } 7082 7083 @Test 7084 public void testBatchMutateWithWrongRegionException() throws Exception { 7085 final byte[] a = Bytes.toBytes("a"); 7086 final byte[] b = Bytes.toBytes("b"); 7087 final byte[] c = Bytes.toBytes("c"); // exclusive 7088 7089 int prevLockTimeout = CONF.getInt("hbase.rowlock.wait.duration", 30000); 7090 CONF.setInt("hbase.rowlock.wait.duration", 1000); 7091 region = initHRegion(tableName, a, c, method, CONF, false, fam1); 7092 7093 Mutation[] mutations = new Mutation[] { 7094 new Put(a).add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a) 7095 .setFamily(fam1).setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), 7096 // this is outside the region boundary 7097 new Put(c).add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(c) 7098 .setFamily(fam1).setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Type.Put).build()), 7099 new Put(b) 7100 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1) 7101 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()) }; 7102 7103 OperationStatus[] status = region.batchMutate(mutations); 7104 assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode()); 7105 assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, status[1].getOperationStatusCode()); 7106 assertEquals(OperationStatusCode.SUCCESS, status[2].getOperationStatusCode()); 7107 7108 // test with a row lock held for a long time 7109 final CountDownLatch obtainedRowLock = new CountDownLatch(1); 7110 ExecutorService exec = Executors.newFixedThreadPool(2); 7111 Future<Void> f1 = exec.submit(new Callable<Void>() { 7112 @Override 7113 public Void call() throws Exception { 7114 LOG.info("Acquiring row lock"); 7115 RowLock rl = region.getRowLock(b); 7116 obtainedRowLock.countDown(); 7117 LOG.info("Waiting for 5 seconds before releasing lock"); 7118 Threads.sleep(5000); 7119 LOG.info("Releasing row lock"); 7120 rl.release(); 7121 return null; 7122 } 7123 }); 7124 obtainedRowLock.await(30, TimeUnit.SECONDS); 7125 7126 Future<Void> f2 = exec.submit(new Callable<Void>() { 7127 @Override 7128 public Void call() throws Exception { 7129 Mutation[] mutations = new Mutation[] { 7130 new Put(a) 7131 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a).setFamily(fam1) 7132 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), 7133 new Put(b) 7134 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1) 7135 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), }; 7136 7137 // this will wait for the row lock, and it will eventually succeed 7138 OperationStatus[] status = region.batchMutate(mutations); 7139 assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode()); 7140 assertEquals(OperationStatusCode.SUCCESS, status[1].getOperationStatusCode()); 7141 return null; 7142 } 7143 }); 7144 7145 f1.get(); 7146 f2.get(); 7147 7148 CONF.setInt("hbase.rowlock.wait.duration", prevLockTimeout); 7149 } 7150 7151 @Test 7152 public void testBatchMutateWithZeroRowLockWait() throws Exception { 7153 final byte[] a = Bytes.toBytes("a"); 7154 final byte[] b = Bytes.toBytes("b"); 7155 final byte[] c = Bytes.toBytes("c"); // exclusive 7156 7157 Configuration conf = new Configuration(CONF); 7158 conf.setInt("hbase.rowlock.wait.duration", 0); 7159 final RegionInfo hri = 7160 RegionInfoBuilder.newBuilder(tableName).setStartKey(a).setEndKey(c).build(); 7161 final TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName) 7162 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)).build(); 7163 region = HRegion.createHRegion(hri, TEST_UTIL.getDataTestDir(), conf, htd, 7164 HBaseTestingUtil.createWal(conf, TEST_UTIL.getDataTestDirOnTestFS(method + ".log"), hri)); 7165 7166 Mutation[] mutations = new Mutation[] { 7167 new Put(a).add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a) 7168 .setFamily(fam1).setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), 7169 new Put(b) 7170 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1) 7171 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()) }; 7172 7173 OperationStatus[] status = region.batchMutate(mutations); 7174 assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode()); 7175 assertEquals(OperationStatusCode.SUCCESS, status[1].getOperationStatusCode()); 7176 7177 // test with a row lock held for a long time 7178 final CountDownLatch obtainedRowLock = new CountDownLatch(1); 7179 ExecutorService exec = Executors.newFixedThreadPool(2); 7180 Future<Void> f1 = exec.submit(new Callable<Void>() { 7181 @Override 7182 public Void call() throws Exception { 7183 LOG.info("Acquiring row lock"); 7184 RowLock rl = region.getRowLock(b); 7185 obtainedRowLock.countDown(); 7186 LOG.info("Waiting for 5 seconds before releasing lock"); 7187 Threads.sleep(5000); 7188 LOG.info("Releasing row lock"); 7189 rl.release(); 7190 return null; 7191 } 7192 }); 7193 obtainedRowLock.await(30, TimeUnit.SECONDS); 7194 7195 Future<Void> f2 = exec.submit(new Callable<Void>() { 7196 @Override 7197 public Void call() throws Exception { 7198 Mutation[] mutations = new Mutation[] { 7199 new Put(a) 7200 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a).setFamily(fam1) 7201 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), 7202 new Put(b) 7203 .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1) 7204 .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), }; 7205 // when handling row b we are going to spin on the failure to get the row lock 7206 // until the lock above is released, but we will still succeed so long as that 7207 // takes less time then the test time out. 7208 OperationStatus[] status = region.batchMutate(mutations); 7209 assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode()); 7210 assertEquals(OperationStatusCode.SUCCESS, status[1].getOperationStatusCode()); 7211 return null; 7212 } 7213 }); 7214 7215 f1.get(); 7216 f2.get(); 7217 } 7218 7219 @Test 7220 public void testCheckAndRowMutateTimestampsAreMonotonic() throws IOException { 7221 region = initHRegion(tableName, method, CONF, fam1); 7222 ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); 7223 EnvironmentEdgeManager.injectEdge(edge); 7224 7225 edge.setValue(10); 7226 Put p = new Put(row); 7227 p.setDurability(Durability.SKIP_WAL); 7228 p.addColumn(fam1, qual1, qual1); 7229 region.put(p); 7230 7231 Result result = region.get(new Get(row)); 7232 Cell c = result.getColumnLatestCell(fam1, qual1); 7233 assertNotNull(c); 7234 assertEquals(10L, c.getTimestamp()); 7235 7236 edge.setValue(1); // clock goes back 7237 p = new Put(row); 7238 p.setDurability(Durability.SKIP_WAL); 7239 p.addColumn(fam1, qual1, qual2); 7240 RowMutations rm = new RowMutations(row); 7241 rm.add(p); 7242 assertTrue(region.checkAndRowMutate(row, fam1, qual1, CompareOperator.EQUAL, 7243 new BinaryComparator(qual1), rm)); 7244 result = region.get(new Get(row)); 7245 c = result.getColumnLatestCell(fam1, qual1); 7246 assertEquals(10L, c.getTimestamp()); 7247 LOG.info( 7248 "c value " + Bytes.toStringBinary(c.getValueArray(), c.getValueOffset(), c.getValueLength())); 7249 7250 assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(), qual2, 0, 7251 qual2.length)); 7252 } 7253 7254 HRegion initHRegion(TableName tableName, String callingMethod, byte[]... families) 7255 throws IOException { 7256 return initHRegion(tableName, callingMethod, HBaseConfiguration.create(), families); 7257 } 7258 7259 /** 7260 * HBASE-16429 Make sure no stuck if roll writer when ring buffer is filled with appends 7261 * @throws IOException if IO error occurred during test 7262 */ 7263 @Test 7264 public void testWritesWhileRollWriter() throws IOException { 7265 int testCount = 10; 7266 int numRows = 1024; 7267 int numFamilies = 2; 7268 int numQualifiers = 2; 7269 final byte[][] families = new byte[numFamilies][]; 7270 for (int i = 0; i < numFamilies; i++) { 7271 families[i] = Bytes.toBytes("family" + i); 7272 } 7273 final byte[][] qualifiers = new byte[numQualifiers][]; 7274 for (int i = 0; i < numQualifiers; i++) { 7275 qualifiers[i] = Bytes.toBytes("qual" + i); 7276 } 7277 7278 CONF.setInt("hbase.regionserver.wal.disruptor.event.count", 2); 7279 this.region = initHRegion(tableName, method, CONF, families); 7280 try { 7281 List<Thread> threads = new ArrayList<>(); 7282 for (int i = 0; i < numRows; i++) { 7283 final int count = i; 7284 Thread t = new Thread(new Runnable() { 7285 7286 @Override 7287 public void run() { 7288 byte[] row = Bytes.toBytes("row" + count); 7289 Put put = new Put(row); 7290 put.setDurability(Durability.SYNC_WAL); 7291 byte[] value = Bytes.toBytes(String.valueOf(count)); 7292 for (byte[] family : families) { 7293 for (byte[] qualifier : qualifiers) { 7294 put.addColumn(family, qualifier, count, value); 7295 } 7296 } 7297 try { 7298 region.put(put); 7299 } catch (IOException e) { 7300 throw new RuntimeException(e); 7301 } 7302 } 7303 }); 7304 threads.add(t); 7305 } 7306 for (Thread t : threads) { 7307 t.start(); 7308 } 7309 7310 for (int i = 0; i < testCount; i++) { 7311 region.getWAL().rollWriter(); 7312 Thread.yield(); 7313 } 7314 } finally { 7315 try { 7316 HBaseTestingUtil.closeRegionAndWAL(this.region); 7317 CONF.setInt("hbase.regionserver.wal.disruptor.event.count", 16 * 1024); 7318 } catch (DroppedSnapshotException dse) { 7319 // We could get this on way out because we interrupt the background flusher and it could 7320 // fail anywhere causing a DSE over in the background flusher... only it is not properly 7321 // dealt with so could still be memory hanging out when we get to here -- memory we can't 7322 // flush because the accounting is 'off' since original DSE. 7323 } 7324 this.region = null; 7325 } 7326 } 7327 7328 @Test 7329 public void testMutateRow() throws Exception { 7330 final byte[] row = Bytes.toBytes("row"); 7331 final byte[] q1 = Bytes.toBytes("q1"); 7332 final byte[] q2 = Bytes.toBytes("q2"); 7333 final byte[] q3 = Bytes.toBytes("q3"); 7334 final byte[] q4 = Bytes.toBytes("q4"); 7335 final String v1 = "v1"; 7336 7337 region = initHRegion(tableName, method, CONF, fam1); 7338 7339 // Initial values 7340 region 7341 .batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q2, Bytes.toBytes("toBeDeleted")), 7342 new Put(row).addColumn(fam1, q3, Bytes.toBytes(5L)), 7343 new Put(row).addColumn(fam1, q4, Bytes.toBytes("a")), }); 7344 7345 // Do mutateRow 7346 Result result = region.mutateRow( 7347 new RowMutations(row).add(Arrays.asList(new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1)), 7348 new Delete(row).addColumns(fam1, q2), new Increment(row).addColumn(fam1, q3, 1), 7349 new Append(row).addColumn(fam1, q4, Bytes.toBytes("b"))))); 7350 7351 assertNotNull(result); 7352 assertEquals(6L, Bytes.toLong(result.getValue(fam1, q3))); 7353 assertEquals("ab", Bytes.toString(result.getValue(fam1, q4))); 7354 7355 // Verify the value 7356 result = region.get(new Get(row)); 7357 assertEquals(v1, Bytes.toString(result.getValue(fam1, q1))); 7358 assertNull(result.getValue(fam1, q2)); 7359 assertEquals(6L, Bytes.toLong(result.getValue(fam1, q3))); 7360 assertEquals("ab", Bytes.toString(result.getValue(fam1, q4))); 7361 } 7362 7363 @Test 7364 public void testMutateRowInParallel() throws Exception { 7365 final int numReaderThreads = 100; 7366 final CountDownLatch latch = new CountDownLatch(numReaderThreads); 7367 7368 final byte[] row = Bytes.toBytes("row"); 7369 final byte[] q1 = Bytes.toBytes("q1"); 7370 final byte[] q2 = Bytes.toBytes("q2"); 7371 final byte[] q3 = Bytes.toBytes("q3"); 7372 final byte[] q4 = Bytes.toBytes("q4"); 7373 final String v1 = "v1"; 7374 final String v2 = "v2"; 7375 7376 // We need to ensure the timestamp of the delete operation is more than the previous one 7377 final AtomicLong deleteTimestamp = new AtomicLong(); 7378 7379 region = initHRegion(tableName, method, CONF, fam1); 7380 7381 // Initial values 7382 region.batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1)) 7383 .addColumn(fam1, q2, deleteTimestamp.getAndIncrement(), Bytes.toBytes(v2)) 7384 .addColumn(fam1, q3, Bytes.toBytes(1L)).addColumn(fam1, q4, Bytes.toBytes("a")) }); 7385 7386 final AtomicReference<AssertionError> assertionError = new AtomicReference<>(); 7387 7388 // Writer thread 7389 Thread writerThread = new Thread(() -> { 7390 try { 7391 while (true) { 7392 // If all the reader threads finish, then stop the writer thread 7393 if (latch.await(0, TimeUnit.MILLISECONDS)) { 7394 return; 7395 } 7396 7397 // Execute the mutations. This should be done atomically 7398 region.mutateRow(new RowMutations(row) 7399 .add(Arrays.asList(new Put(row).addColumn(fam1, q1, Bytes.toBytes(v2)), 7400 new Delete(row).addColumns(fam1, q2, deleteTimestamp.getAndIncrement()), 7401 new Increment(row).addColumn(fam1, q3, 1L), 7402 new Append(row).addColumn(fam1, q4, Bytes.toBytes("b"))))); 7403 7404 // We need to ensure the timestamps of the Increment/Append operations are more than the 7405 // previous ones 7406 Result result = region.get(new Get(row).addColumn(fam1, q3).addColumn(fam1, q4)); 7407 long tsIncrement = result.getColumnLatestCell(fam1, q3).getTimestamp(); 7408 long tsAppend = result.getColumnLatestCell(fam1, q4).getTimestamp(); 7409 7410 // Put the initial values 7411 region.batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1)) 7412 .addColumn(fam1, q2, deleteTimestamp.getAndIncrement(), Bytes.toBytes(v2)) 7413 .addColumn(fam1, q3, tsIncrement + 1, Bytes.toBytes(1L)) 7414 .addColumn(fam1, q4, tsAppend + 1, Bytes.toBytes("a")) }); 7415 } 7416 } catch (Exception e) { 7417 assertionError.set(new AssertionError(e)); 7418 } 7419 }); 7420 writerThread.start(); 7421 7422 // Reader threads 7423 for (int i = 0; i < numReaderThreads; i++) { 7424 new Thread(() -> { 7425 try { 7426 for (int j = 0; j < 10000; j++) { 7427 // Verify the values 7428 Result result = region.get(new Get(row)); 7429 7430 // The values should be equals to either the initial values or the values after 7431 // executing the mutations 7432 String q1Value = Bytes.toString(result.getValue(fam1, q1)); 7433 if (v1.equals(q1Value)) { 7434 assertEquals(v2, Bytes.toString(result.getValue(fam1, q2))); 7435 assertEquals(1L, Bytes.toLong(result.getValue(fam1, q3))); 7436 assertEquals("a", Bytes.toString(result.getValue(fam1, q4))); 7437 } else if (v2.equals(q1Value)) { 7438 assertNull(Bytes.toString(result.getValue(fam1, q2))); 7439 assertEquals(2L, Bytes.toLong(result.getValue(fam1, q3))); 7440 assertEquals("ab", Bytes.toString(result.getValue(fam1, q4))); 7441 } else { 7442 fail("the qualifier " + Bytes.toString(q1) + " should be " + v1 + " or " + v2 7443 + ", but " + q1Value); 7444 } 7445 } 7446 } catch (Exception e) { 7447 assertionError.set(new AssertionError(e)); 7448 } catch (AssertionError e) { 7449 assertionError.set(e); 7450 } 7451 7452 latch.countDown(); 7453 }).start(); 7454 } 7455 7456 writerThread.join(); 7457 7458 if (assertionError.get() != null) { 7459 throw assertionError.get(); 7460 } 7461 } 7462 7463 @Test 7464 public void testMutateRow_WriteRequestCount() throws Exception { 7465 byte[] row1 = Bytes.toBytes("row1"); 7466 byte[] fam1 = Bytes.toBytes("fam1"); 7467 byte[] qf1 = Bytes.toBytes("qualifier"); 7468 byte[] val1 = Bytes.toBytes("value1"); 7469 7470 RowMutations rm = new RowMutations(row1); 7471 Put put = new Put(row1); 7472 put.addColumn(fam1, qf1, val1); 7473 rm.add(put); 7474 7475 this.region = initHRegion(tableName, method, CONF, fam1); 7476 long wrcBeforeMutate = this.region.writeRequestsCount.longValue(); 7477 this.region.mutateRow(rm); 7478 long wrcAfterMutate = this.region.writeRequestsCount.longValue(); 7479 Assert.assertEquals(wrcBeforeMutate + rm.getMutations().size(), wrcAfterMutate); 7480 } 7481 7482 @Test 7483 public void testBulkLoadReplicationEnabled() throws IOException { 7484 TEST_UTIL.getConfiguration().setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true); 7485 final ServerName serverName = ServerName.valueOf(name.getMethodName(), 100, 42); 7486 final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName)); 7487 7488 TableDescriptor tableDescriptor = 7489 TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) 7490 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)).build(); 7491 RegionInfo hri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(); 7492 region = HRegion.openHRegion(hri, tableDescriptor, rss.getWAL(hri), 7493 TEST_UTIL.getConfiguration(), rss, null); 7494 7495 assertTrue(region.conf.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, false)); 7496 String plugins = region.conf.get(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, ""); 7497 String replicationCoprocessorClass = ReplicationObserver.class.getCanonicalName(); 7498 assertTrue(plugins.contains(replicationCoprocessorClass)); 7499 assertTrue(region.getCoprocessorHost().getCoprocessors() 7500 .contains(ReplicationObserver.class.getSimpleName())); 7501 } 7502 7503 /** 7504 * The same as HRegion class, the only difference is that instantiateHStore will create a 7505 * different HStore - HStoreForTesting. [HBASE-8518] 7506 */ 7507 public static class HRegionForTesting extends HRegion { 7508 7509 public HRegionForTesting(final Path tableDir, final WAL wal, final FileSystem fs, 7510 final Configuration confParam, final RegionInfo regionInfo, final TableDescriptor htd, 7511 final RegionServerServices rsServices) { 7512 this(new HRegionFileSystem(confParam, fs, tableDir, regionInfo), wal, confParam, htd, 7513 rsServices); 7514 } 7515 7516 public HRegionForTesting(HRegionFileSystem fs, WAL wal, Configuration confParam, 7517 TableDescriptor htd, RegionServerServices rsServices) { 7518 super(fs, wal, confParam, htd, rsServices); 7519 } 7520 7521 /** 7522 * Create HStore instance. 7523 * @return If Mob is enabled, return HMobStore, otherwise return HStoreForTesting. 7524 */ 7525 @Override 7526 protected HStore instantiateHStore(final ColumnFamilyDescriptor family, boolean warmup) 7527 throws IOException { 7528 if (family.isMobEnabled()) { 7529 if (HFile.getFormatVersion(this.conf) < HFile.MIN_FORMAT_VERSION_WITH_TAGS) { 7530 throw new IOException("A minimum HFile version of " + HFile.MIN_FORMAT_VERSION_WITH_TAGS 7531 + " is required for MOB feature. Consider setting " + HFile.FORMAT_VERSION_KEY 7532 + " accordingly."); 7533 } 7534 return new HMobStore(this, family, this.conf, warmup); 7535 } 7536 return new HStoreForTesting(this, family, this.conf, warmup); 7537 } 7538 } 7539 7540 /** 7541 * HStoreForTesting is merely the same as HStore, the difference is in the doCompaction method of 7542 * HStoreForTesting there is a checkpoint "hbase.hstore.compaction.complete" which doesn't let 7543 * hstore compaction complete. In the former edition, this config is set in HStore class inside 7544 * compact method, though this is just for testing, otherwise it doesn't do any help. In 7545 * HBASE-8518, we try to get rid of all "hbase.hstore.compaction.complete" config (except for 7546 * testing code). 7547 */ 7548 public static class HStoreForTesting extends HStore { 7549 7550 protected HStoreForTesting(final HRegion region, final ColumnFamilyDescriptor family, 7551 final Configuration confParam, boolean warmup) throws IOException { 7552 super(region, family, confParam, warmup); 7553 } 7554 7555 @Override 7556 protected List<HStoreFile> doCompaction(CompactionRequestImpl cr, 7557 Collection<HStoreFile> filesToCompact, User user, long compactionStartTime, 7558 List<Path> newFiles) throws IOException { 7559 // let compaction incomplete. 7560 if (!this.conf.getBoolean("hbase.hstore.compaction.complete", true)) { 7561 LOG.warn("hbase.hstore.compaction.complete is set to false"); 7562 List<HStoreFile> sfs = new ArrayList<>(newFiles.size()); 7563 final boolean evictOnClose = 7564 getCacheConfig() != null ? getCacheConfig().shouldEvictOnClose() : true; 7565 for (Path newFile : newFiles) { 7566 // Create storefile around what we wrote with a reader on it. 7567 HStoreFile sf = storeEngine.createStoreFileAndReader(newFile); 7568 sf.closeStoreFile(evictOnClose); 7569 sfs.add(sf); 7570 } 7571 return sfs; 7572 } 7573 return super.doCompaction(cr, filesToCompact, user, compactionStartTime, newFiles); 7574 } 7575 } 7576 7577 @Test 7578 public void testCloseNoInterrupt() throws Exception { 7579 byte[] cf1 = Bytes.toBytes("CF1"); 7580 byte[][] families = { cf1 }; 7581 final int SLEEP_TIME = 10 * 1000; 7582 7583 Configuration conf = new Configuration(CONF); 7584 // Disable close thread interrupt and server abort behavior 7585 conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, false); 7586 conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, 1000); 7587 region = initHRegion(tableName, method, conf, families); 7588 7589 final CountDownLatch latch = new CountDownLatch(1); 7590 final AtomicBoolean holderInterrupted = new AtomicBoolean(); 7591 Thread holder = new Thread(new Runnable() { 7592 @Override 7593 public void run() { 7594 try { 7595 LOG.info("Starting region operation holder"); 7596 region.startRegionOperation(Operation.SCAN); 7597 latch.countDown(); 7598 try { 7599 Thread.sleep(SLEEP_TIME); 7600 } catch (InterruptedException e) { 7601 LOG.info("Interrupted"); 7602 holderInterrupted.set(true); 7603 } 7604 } catch (Exception e) { 7605 throw new RuntimeException(e); 7606 } finally { 7607 try { 7608 region.closeRegionOperation(); 7609 } catch (IOException e) { 7610 } 7611 LOG.info("Stopped region operation holder"); 7612 } 7613 } 7614 }); 7615 7616 holder.start(); 7617 latch.await(); 7618 region.close(); 7619 region = null; 7620 holder.join(); 7621 7622 assertFalse("Region lock holder should not have been interrupted", holderInterrupted.get()); 7623 } 7624 7625 @Test 7626 public void testCloseInterrupt() throws Exception { 7627 byte[] cf1 = Bytes.toBytes("CF1"); 7628 byte[][] families = { cf1 }; 7629 final int SLEEP_TIME = 10 * 1000; 7630 7631 Configuration conf = new Configuration(CONF); 7632 // Enable close thread interrupt and server abort behavior 7633 conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, true); 7634 // Speed up the unit test, no need to wait default 10 seconds. 7635 conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, 1000); 7636 region = initHRegion(tableName, method, conf, families); 7637 7638 final CountDownLatch latch = new CountDownLatch(1); 7639 final AtomicBoolean holderInterrupted = new AtomicBoolean(); 7640 Thread holder = new Thread(new Runnable() { 7641 @Override 7642 public void run() { 7643 try { 7644 LOG.info("Starting region operation holder"); 7645 region.startRegionOperation(Operation.SCAN); 7646 latch.countDown(); 7647 try { 7648 Thread.sleep(SLEEP_TIME); 7649 } catch (InterruptedException e) { 7650 LOG.info("Interrupted"); 7651 holderInterrupted.set(true); 7652 } 7653 } catch (Exception e) { 7654 throw new RuntimeException(e); 7655 } finally { 7656 try { 7657 region.closeRegionOperation(); 7658 } catch (IOException e) { 7659 } 7660 LOG.info("Stopped region operation holder"); 7661 } 7662 } 7663 }); 7664 7665 holder.start(); 7666 latch.await(); 7667 region.close(); 7668 region = null; 7669 holder.join(); 7670 7671 assertTrue("Region lock holder was not interrupted", holderInterrupted.get()); 7672 } 7673 7674 @Test 7675 public void testCloseAbort() throws Exception { 7676 byte[] cf1 = Bytes.toBytes("CF1"); 7677 byte[][] families = { cf1 }; 7678 final int SLEEP_TIME = 10 * 1000; 7679 7680 Configuration conf = new Configuration(CONF); 7681 // Enable close thread interrupt and server abort behavior. 7682 conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, true); 7683 // Set the abort interval to a fraction of sleep time so we are guaranteed to be aborted. 7684 conf.setInt(HRegion.CLOSE_WAIT_TIME, SLEEP_TIME / 2); 7685 // Set the wait interval to a fraction of sleep time so we are guaranteed to be interrupted. 7686 conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, SLEEP_TIME / 4); 7687 region = initHRegion(tableName, method, conf, families); 7688 RegionServerServices rsServices = mock(RegionServerServices.class); 7689 when(rsServices.getServerName()).thenReturn(ServerName.valueOf("localhost", 1000, 1000)); 7690 region.rsServices = rsServices; 7691 7692 final CountDownLatch latch = new CountDownLatch(1); 7693 Thread holder = new Thread(new Runnable() { 7694 @Override 7695 public void run() { 7696 try { 7697 LOG.info("Starting region operation holder"); 7698 region.startRegionOperation(Operation.SCAN); 7699 latch.countDown(); 7700 // Hold the lock for SLEEP_TIME seconds no matter how many times we are interrupted. 7701 int timeRemaining = SLEEP_TIME; 7702 while (timeRemaining > 0) { 7703 long start = EnvironmentEdgeManager.currentTime(); 7704 try { 7705 Thread.sleep(timeRemaining); 7706 } catch (InterruptedException e) { 7707 LOG.info("Interrupted"); 7708 } 7709 long end = EnvironmentEdgeManager.currentTime(); 7710 timeRemaining -= end - start; 7711 if (timeRemaining < 0) { 7712 timeRemaining = 0; 7713 } 7714 if (timeRemaining > 0) { 7715 LOG.info("Sleeping again, remaining time " + timeRemaining + " ms"); 7716 } 7717 } 7718 } catch (Exception e) { 7719 throw new RuntimeException(e); 7720 } finally { 7721 try { 7722 region.closeRegionOperation(); 7723 } catch (IOException e) { 7724 } 7725 LOG.info("Stopped region operation holder"); 7726 } 7727 } 7728 }); 7729 7730 holder.start(); 7731 latch.await(); 7732 try { 7733 region.close(); 7734 } catch (IOException e) { 7735 LOG.info("Caught expected exception", e); 7736 } 7737 region = null; 7738 holder.join(); 7739 7740 // Verify the region tried to abort the server 7741 verify(rsServices, atLeast(1)).abort(anyString(), any()); 7742 } 7743 7744 @Test 7745 public void testInterruptProtection() throws Exception { 7746 byte[] cf1 = Bytes.toBytes("CF1"); 7747 byte[][] families = { cf1 }; 7748 final int SLEEP_TIME = 10 * 1000; 7749 7750 Configuration conf = new Configuration(CONF); 7751 // Enable close thread interrupt and server abort behavior. 7752 conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, true); 7753 conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, 1000); 7754 region = initHRegion(tableName, method, conf, families); 7755 7756 final CountDownLatch latch = new CountDownLatch(1); 7757 final AtomicBoolean holderInterrupted = new AtomicBoolean(); 7758 Thread holder = new Thread(new Runnable() { 7759 @Override 7760 public void run() { 7761 try { 7762 LOG.info("Starting region operation holder"); 7763 region.startRegionOperation(Operation.SCAN); 7764 LOG.info("Protecting against interrupts"); 7765 region.disableInterrupts(); 7766 try { 7767 latch.countDown(); 7768 try { 7769 Thread.sleep(SLEEP_TIME); 7770 } catch (InterruptedException e) { 7771 LOG.info("Interrupted"); 7772 holderInterrupted.set(true); 7773 } 7774 } finally { 7775 region.enableInterrupts(); 7776 } 7777 } catch (Exception e) { 7778 throw new RuntimeException(e); 7779 } finally { 7780 try { 7781 region.closeRegionOperation(); 7782 } catch (IOException e) { 7783 } 7784 LOG.info("Stopped region operation holder"); 7785 } 7786 } 7787 }); 7788 7789 holder.start(); 7790 latch.await(); 7791 region.close(); 7792 region = null; 7793 holder.join(); 7794 7795 assertFalse("Region lock holder should not have been interrupted", holderInterrupted.get()); 7796 } 7797 7798 @Test 7799 public void testRegionOnCoprocessorsChange() throws IOException { 7800 byte[] cf1 = Bytes.toBytes("CF1"); 7801 byte[][] families = { cf1 }; 7802 7803 Configuration conf = new Configuration(CONF); 7804 region = initHRegion(tableName, method, conf, families); 7805 assertNull(region.getCoprocessorHost()); 7806 7807 // set and verify the system coprocessors for region and user region 7808 Configuration newConf = new Configuration(conf); 7809 newConf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, MetaTableMetrics.class.getName()); 7810 newConf.set(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, 7811 NoOpRegionCoprocessor.class.getName()); 7812 // trigger configuration change 7813 region.onConfigurationChange(newConf); 7814 assertTrue(region.getCoprocessorHost() != null); 7815 Set<String> coprocessors = region.getCoprocessorHost().getCoprocessors(); 7816 assertTrue(coprocessors.size() == 2); 7817 assertTrue(region.getCoprocessorHost().getCoprocessors() 7818 .contains(MetaTableMetrics.class.getSimpleName())); 7819 assertTrue(region.getCoprocessorHost().getCoprocessors() 7820 .contains(NoOpRegionCoprocessor.class.getSimpleName())); 7821 7822 // remove region coprocessor and keep only user region coprocessor 7823 newConf.unset(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY); 7824 region.onConfigurationChange(newConf); 7825 assertTrue(region.getCoprocessorHost() != null); 7826 coprocessors = region.getCoprocessorHost().getCoprocessors(); 7827 assertTrue(coprocessors.size() == 1); 7828 assertTrue(region.getCoprocessorHost().getCoprocessors() 7829 .contains(NoOpRegionCoprocessor.class.getSimpleName())); 7830 } 7831 7832 @Test 7833 public void testRegionOnCoprocessorsWithoutChange() throws IOException { 7834 byte[] cf1 = Bytes.toBytes("CF1"); 7835 byte[][] families = { cf1 }; 7836 7837 Configuration conf = new Configuration(CONF); 7838 conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, 7839 MetaTableMetrics.class.getCanonicalName()); 7840 region = initHRegion(tableName, method, conf, families); 7841 // region service is null in unit test, we need to load the coprocessor once 7842 region.setCoprocessorHost(new RegionCoprocessorHost(region, null, conf)); 7843 RegionCoprocessor regionCoprocessor = 7844 region.getCoprocessorHost().findCoprocessor(MetaTableMetrics.class.getName()); 7845 7846 // simulate when other configuration may have changed and onConfigurationChange execute once 7847 region.onConfigurationChange(conf); 7848 RegionCoprocessor regionCoprocessorAfterOnConfigurationChange = 7849 region.getCoprocessorHost().findCoprocessor(MetaTableMetrics.class.getName()); 7850 assertEquals(regionCoprocessor, regionCoprocessorAfterOnConfigurationChange); 7851 } 7852 7853 public static class NoOpRegionCoprocessor implements RegionCoprocessor, RegionObserver { 7854 // a empty region coprocessor class 7855 } 7856}