001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.HBaseTestingUtil.COLUMNS;
021import static org.apache.hadoop.hbase.HBaseTestingUtil.fam1;
022import static org.apache.hadoop.hbase.HBaseTestingUtil.fam2;
023import static org.apache.hadoop.hbase.HBaseTestingUtil.fam3;
024import static org.junit.Assert.assertArrayEquals;
025import static org.junit.Assert.assertEquals;
026import static org.junit.Assert.assertFalse;
027import static org.junit.Assert.assertNotNull;
028import static org.junit.Assert.assertNull;
029import static org.junit.Assert.assertThrows;
030import static org.junit.Assert.assertTrue;
031import static org.junit.Assert.fail;
032import static org.mockito.ArgumentMatchers.any;
033import static org.mockito.ArgumentMatchers.anyLong;
034import static org.mockito.ArgumentMatchers.anyString;
035import static org.mockito.ArgumentMatchers.isA;
036import static org.mockito.Mockito.atLeast;
037import static org.mockito.Mockito.doAnswer;
038import static org.mockito.Mockito.doThrow;
039import static org.mockito.Mockito.mock;
040import static org.mockito.Mockito.never;
041import static org.mockito.Mockito.spy;
042import static org.mockito.Mockito.times;
043import static org.mockito.Mockito.verify;
044import static org.mockito.Mockito.when;
045
046import java.io.IOException;
047import java.io.InterruptedIOException;
048import java.math.BigDecimal;
049import java.security.PrivilegedExceptionAction;
050import java.util.ArrayList;
051import java.util.Arrays;
052import java.util.Collection;
053import java.util.List;
054import java.util.Map;
055import java.util.NavigableMap;
056import java.util.Objects;
057import java.util.Set;
058import java.util.TreeMap;
059import java.util.concurrent.Callable;
060import java.util.concurrent.CountDownLatch;
061import java.util.concurrent.ExecutorService;
062import java.util.concurrent.Executors;
063import java.util.concurrent.Future;
064import java.util.concurrent.TimeUnit;
065import java.util.concurrent.atomic.AtomicBoolean;
066import java.util.concurrent.atomic.AtomicInteger;
067import java.util.concurrent.atomic.AtomicLong;
068import java.util.concurrent.atomic.AtomicReference;
069import java.util.stream.Collectors;
070import org.apache.commons.lang3.RandomStringUtils;
071import org.apache.hadoop.conf.Configuration;
072import org.apache.hadoop.fs.FSDataOutputStream;
073import org.apache.hadoop.fs.FileStatus;
074import org.apache.hadoop.fs.FileSystem;
075import org.apache.hadoop.fs.Path;
076import org.apache.hadoop.hbase.ArrayBackedTag;
077import org.apache.hadoop.hbase.Cell;
078import org.apache.hadoop.hbase.Cell.Type;
079import org.apache.hadoop.hbase.CellBuilderFactory;
080import org.apache.hadoop.hbase.CellBuilderType;
081import org.apache.hadoop.hbase.CellUtil;
082import org.apache.hadoop.hbase.CompareOperator;
083import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
084import org.apache.hadoop.hbase.DoNotRetryIOException;
085import org.apache.hadoop.hbase.DroppedSnapshotException;
086import org.apache.hadoop.hbase.ExtendedCell;
087import org.apache.hadoop.hbase.ExtendedCellBuilderFactory;
088import org.apache.hadoop.hbase.HBaseClassTestRule;
089import org.apache.hadoop.hbase.HBaseConfiguration;
090import org.apache.hadoop.hbase.HBaseTestingUtil;
091import org.apache.hadoop.hbase.HConstants;
092import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
093import org.apache.hadoop.hbase.HDFSBlocksDistribution;
094import org.apache.hadoop.hbase.KeyValue;
095import org.apache.hadoop.hbase.MultithreadedTestUtil;
096import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
097import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
098import org.apache.hadoop.hbase.NotServingRegionException;
099import org.apache.hadoop.hbase.PrivateCellUtil;
100import org.apache.hadoop.hbase.RegionTooBusyException;
101import org.apache.hadoop.hbase.ServerName;
102import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
103import org.apache.hadoop.hbase.StartTestingClusterOption;
104import org.apache.hadoop.hbase.TableName;
105import org.apache.hadoop.hbase.TagType;
106import org.apache.hadoop.hbase.Waiter;
107import org.apache.hadoop.hbase.client.Append;
108import org.apache.hadoop.hbase.client.CheckAndMutate;
109import org.apache.hadoop.hbase.client.CheckAndMutateResult;
110import org.apache.hadoop.hbase.client.ClientInternalHelper;
111import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
112import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
113import org.apache.hadoop.hbase.client.Delete;
114import org.apache.hadoop.hbase.client.Durability;
115import org.apache.hadoop.hbase.client.Get;
116import org.apache.hadoop.hbase.client.Increment;
117import org.apache.hadoop.hbase.client.Mutation;
118import org.apache.hadoop.hbase.client.Put;
119import org.apache.hadoop.hbase.client.RegionInfo;
120import org.apache.hadoop.hbase.client.RegionInfoBuilder;
121import org.apache.hadoop.hbase.client.Result;
122import org.apache.hadoop.hbase.client.RowMutations;
123import org.apache.hadoop.hbase.client.Scan;
124import org.apache.hadoop.hbase.client.Table;
125import org.apache.hadoop.hbase.client.TableDescriptor;
126import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
127import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
128import org.apache.hadoop.hbase.coprocessor.MetaTableMetrics;
129import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
130import org.apache.hadoop.hbase.coprocessor.RegionObserver;
131import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
132import org.apache.hadoop.hbase.filter.BigDecimalComparator;
133import org.apache.hadoop.hbase.filter.BinaryComparator;
134import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
135import org.apache.hadoop.hbase.filter.Filter;
136import org.apache.hadoop.hbase.filter.FilterBase;
137import org.apache.hadoop.hbase.filter.FilterList;
138import org.apache.hadoop.hbase.filter.NullComparator;
139import org.apache.hadoop.hbase.filter.PrefixFilter;
140import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
141import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
142import org.apache.hadoop.hbase.filter.SubstringComparator;
143import org.apache.hadoop.hbase.filter.ValueFilter;
144import org.apache.hadoop.hbase.io.TimeRange;
145import org.apache.hadoop.hbase.io.hfile.HFile;
146import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
147import org.apache.hadoop.hbase.monitoring.MonitoredTask;
148import org.apache.hadoop.hbase.monitoring.TaskMonitor;
149import org.apache.hadoop.hbase.regionserver.Region.Operation;
150import org.apache.hadoop.hbase.regionserver.Region.RowLock;
151import org.apache.hadoop.hbase.regionserver.TestHStore.FaultyFileSystem;
152import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl;
153import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
154import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
155import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL;
156import org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL;
157import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
158import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
159import org.apache.hadoop.hbase.regionserver.wal.WALUtil;
160import org.apache.hadoop.hbase.replication.regionserver.ReplicationObserver;
161import org.apache.hadoop.hbase.security.User;
162import org.apache.hadoop.hbase.test.MetricsAssertHelper;
163import org.apache.hadoop.hbase.testclassification.LargeTests;
164import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests;
165import org.apache.hadoop.hbase.util.Bytes;
166import org.apache.hadoop.hbase.util.CommonFSUtils;
167import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
168import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
169import org.apache.hadoop.hbase.util.HFileArchiveUtil;
170import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
171import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
172import org.apache.hadoop.hbase.util.Threads;
173import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
174import org.apache.hadoop.hbase.wal.FaultyFSLog;
175import org.apache.hadoop.hbase.wal.NettyAsyncFSWALConfigHelper;
176import org.apache.hadoop.hbase.wal.WAL;
177import org.apache.hadoop.hbase.wal.WALEdit;
178import org.apache.hadoop.hbase.wal.WALEditInternalHelper;
179import org.apache.hadoop.hbase.wal.WALFactory;
180import org.apache.hadoop.hbase.wal.WALKeyImpl;
181import org.apache.hadoop.hbase.wal.WALProvider;
182import org.apache.hadoop.hbase.wal.WALProvider.Writer;
183import org.apache.hadoop.hbase.wal.WALSplitUtil;
184import org.apache.hadoop.hbase.wal.WALStreamReader;
185import org.junit.After;
186import org.junit.Assert;
187import org.junit.Before;
188import org.junit.ClassRule;
189import org.junit.Ignore;
190import org.junit.Rule;
191import org.junit.Test;
192import org.junit.experimental.categories.Category;
193import org.junit.rules.ExpectedException;
194import org.junit.rules.TestName;
195import org.mockito.ArgumentCaptor;
196import org.mockito.ArgumentMatcher;
197import org.mockito.invocation.InvocationOnMock;
198import org.mockito.stubbing.Answer;
199import org.slf4j.Logger;
200import org.slf4j.LoggerFactory;
201
202import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
203import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
204import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
205import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup;
206import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel;
207
208import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
209import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor;
210import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor;
211import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.FlushAction;
212import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor;
213import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor;
214import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.StoreDescriptor;
215
216/**
217 * Basic stand-alone testing of HRegion. No clusters! A lot of the meta information for an HRegion
218 * now lives inside other HRegions or in the HBaseMaster, so only basic testing is possible.
219 */
220@Category({ VerySlowRegionServerTests.class, LargeTests.class })
221@SuppressWarnings("deprecation")
222public class TestHRegion {
223
224  @ClassRule
225  public static final HBaseClassTestRule CLASS_RULE =
226    HBaseClassTestRule.forClass(TestHRegion.class);
227
228  // Do not spin up clusters in here. If you need to spin up a cluster, do it
229  // over in TestHRegionOnCluster.
230  private static final Logger LOG = LoggerFactory.getLogger(TestHRegion.class);
231  @Rule
232  public TestName name = new TestName();
233  @Rule
234  public final ExpectedException thrown = ExpectedException.none();
235
236  private static final String COLUMN_FAMILY = "MyCF";
237  private static final byte[] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);
238  private static final EventLoopGroup GROUP = new NioEventLoopGroup();
239
240  HRegion region = null;
241  // Do not run unit tests in parallel (? Why not? It don't work? Why not? St.Ack)
242  protected static HBaseTestingUtil TEST_UTIL;
243  public static Configuration CONF;
244  private String dir;
245  private final int MAX_VERSIONS = 2;
246
247  // Test names
248  protected TableName tableName;
249  protected String method;
250  protected final byte[] qual = Bytes.toBytes("qual");
251  protected final byte[] qual1 = Bytes.toBytes("qual1");
252  protected final byte[] qual2 = Bytes.toBytes("qual2");
253  protected final byte[] qual3 = Bytes.toBytes("qual3");
254  protected final byte[] value = Bytes.toBytes("value");
255  protected final byte[] value1 = Bytes.toBytes("value1");
256  protected final byte[] value2 = Bytes.toBytes("value2");
257  protected final byte[] row = Bytes.toBytes("rowA");
258  protected final byte[] row2 = Bytes.toBytes("rowB");
259
260  protected final MetricsAssertHelper metricsAssertHelper =
261    CompatibilitySingletonFactory.getInstance(MetricsAssertHelper.class);
262
263  @Before
264  public void setup() throws IOException {
265    TEST_UTIL = new HBaseTestingUtil();
266    CONF = TEST_UTIL.getConfiguration();
267    NettyAsyncFSWALConfigHelper.setEventLoopConfig(CONF, GROUP, NioSocketChannel.class);
268    dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
269    method = name.getMethodName();
270    tableName = TableName.valueOf(method);
271    CONF.set(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, String.valueOf(0.09));
272    CONF.setLong(AbstractFSWAL.WAL_SYNC_TIMEOUT_MS, 10000);
273  }
274
275  @After
276  public void tearDown() throws IOException {
277    // Region may have been closed, but it is still no harm if we close it again here using HTU.
278    HBaseTestingUtil.closeRegionAndWAL(region);
279    EnvironmentEdgeManagerTestHelper.reset();
280    LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
281    TEST_UTIL.cleanupTestDir();
282  }
283
284  /**
285   * Test that I can use the max flushed sequence id after the close.
286   */
287  @Test
288  public void testSequenceId() throws IOException {
289    region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES);
290    assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId());
291    // Weird. This returns 0 if no store files or no edits. Afraid to change it.
292    assertEquals(0, (long) region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES));
293    HBaseTestingUtil.closeRegionAndWAL(this.region);
294    assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId());
295    assertEquals(0, (long) region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES));
296    HRegion oldRegion = region;
297    try {
298      // Open region again.
299      region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES);
300      byte[] value = Bytes.toBytes(method);
301      // Make a random put against our cf.
302      Put put = new Put(value);
303      put.addColumn(COLUMN_FAMILY_BYTES, null, value);
304      region.put(put);
305      // No flush yet so init numbers should still be in place.
306      assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId());
307      assertEquals(0, (long) region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES));
308      region.flush(true);
309      long max = region.getMaxFlushedSeqId();
310      HBaseTestingUtil.closeRegionAndWAL(this.region);
311      assertEquals(max, region.getMaxFlushedSeqId());
312      this.region = null;
313    } finally {
314      HBaseTestingUtil.closeRegionAndWAL(oldRegion);
315    }
316  }
317
318  /**
319   * Test for Bug 2 of HBASE-10466. "Bug 2: Conditions for the first flush of region close
320   * (so-called pre-flush) If memstoreSize is smaller than a certain value, or when region close
321   * starts a flush is ongoing, the first flush is skipped and only the second flush takes place.
322   * However, two flushes are required in case previous flush fails and leaves some data in
323   * snapshot. The bug could cause loss of data in current memstore. The fix is removing all
324   * conditions except abort check so we ensure 2 flushes for region close."
325   */
326  @Test
327  public void testCloseCarryingSnapshot() throws IOException {
328    region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES);
329    HStore store = region.getStore(COLUMN_FAMILY_BYTES);
330    // Get some random bytes.
331    byte[] value = Bytes.toBytes(method);
332    // Make a random put against our cf.
333    Put put = new Put(value);
334    put.addColumn(COLUMN_FAMILY_BYTES, null, value);
335    // First put something in current memstore, which will be in snapshot after flusher.prepare()
336    region.put(put);
337    StoreFlushContext storeFlushCtx = store.createFlushContext(12345, FlushLifeCycleTracker.DUMMY);
338    storeFlushCtx.prepare();
339    // Second put something in current memstore
340    put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
341    region.put(put);
342    // Close with something in memstore and something in the snapshot. Make sure all is cleared.
343    HBaseTestingUtil.closeRegionAndWAL(region);
344    assertEquals(0, region.getMemStoreDataSize());
345    region = null;
346  }
347
348  /*
349   * This test is for verifying memstore snapshot size is correctly updated in case of rollback See
350   * HBASE-10845
351   */
352  @Test
353  public void testMemstoreSnapshotSize() throws IOException {
354    class MyFaultyFSLog extends FaultyFSLog {
355      StoreFlushContext storeFlushCtx;
356
357      public MyFaultyFSLog(FileSystem fs, Path rootDir, String logName, Configuration conf)
358        throws IOException {
359        super(fs, rootDir, logName, conf);
360      }
361
362      void setStoreFlushCtx(StoreFlushContext storeFlushCtx) {
363        this.storeFlushCtx = storeFlushCtx;
364      }
365
366      @Override
367      protected void doSync(long txid, boolean forceSync) throws IOException {
368        storeFlushCtx.prepare();
369        super.doSync(txid, forceSync);
370      }
371    }
372
373    FileSystem fs = FileSystem.get(CONF);
374    Path rootDir = new Path(dir + "testMemstoreSnapshotSize");
375    MyFaultyFSLog faultyLog = new MyFaultyFSLog(fs, rootDir, "testMemstoreSnapshotSize", CONF);
376    faultyLog.init();
377    region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, faultyLog,
378      COLUMN_FAMILY_BYTES);
379
380    HStore store = region.getStore(COLUMN_FAMILY_BYTES);
381    // Get some random bytes.
382    byte[] value = Bytes.toBytes(method);
383    faultyLog.setStoreFlushCtx(store.createFlushContext(12345, FlushLifeCycleTracker.DUMMY));
384
385    Put put = new Put(value);
386    put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
387    faultyLog.setFailureType(FaultyFSLog.FailureType.SYNC);
388    boolean threwIOE = false;
389    try {
390      region.put(put);
391    } catch (IOException ioe) {
392      threwIOE = true;
393    } finally {
394      assertTrue("The regionserver should have thrown an exception", threwIOE);
395    }
396    MemStoreSize mss = store.getFlushableSize();
397    assertTrue("flushable size should be zero, but it is " + mss, mss.getDataSize() == 0);
398  }
399
400  /**
401   * Create a WAL outside of the usual helper in
402   * {@link HBaseTestingUtil#createWal(Configuration, Path, RegionInfo)} because that method doesn't
403   * play nicely with FaultyFileSystem. Call this method before overriding {@code fs.file.impl}.
404   * @param callingMethod a unique component for the path, probably the name of the test method.
405   */
406  private static WAL createWALCompatibleWithFaultyFileSystem(String callingMethod,
407    Configuration conf, TableName tableName) throws IOException {
408    final Path logDir = TEST_UTIL.getDataTestDirOnTestFS(callingMethod + ".log");
409    final Configuration walConf = new Configuration(conf);
410    CommonFSUtils.setRootDir(walConf, logDir);
411    return new WALFactory(walConf, callingMethod)
412      .getWAL(RegionInfoBuilder.newBuilder(tableName).build());
413  }
414
415  @Test
416  public void testMemstoreSizeAccountingWithFailedPostBatchMutate() throws IOException {
417    String testName = "testMemstoreSizeAccountingWithFailedPostBatchMutate";
418    FileSystem fs = FileSystem.get(CONF);
419    Path rootDir = new Path(dir + testName);
420    FSHLog hLog = new FSHLog(fs, rootDir, testName, CONF);
421    hLog.init();
422    region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, hLog,
423      COLUMN_FAMILY_BYTES);
424    HStore store = region.getStore(COLUMN_FAMILY_BYTES);
425    assertEquals(0, region.getMemStoreDataSize());
426
427    // Put one value
428    byte[] value = Bytes.toBytes(method);
429    Put put = new Put(value);
430    put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
431    region.put(put);
432    long onePutSize = region.getMemStoreDataSize();
433    assertTrue(onePutSize > 0);
434
435    RegionCoprocessorHost mockedCPHost = mock(RegionCoprocessorHost.class);
436    doThrow(new IOException()).when(mockedCPHost).postBatchMutate(any());
437    region.setCoprocessorHost(mockedCPHost);
438
439    put = new Put(value);
440    put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("dfg"), value);
441    try {
442      region.put(put);
443      fail("Should have failed with IOException");
444    } catch (IOException expected) {
445    }
446    long expectedSize = onePutSize * 2;
447    assertEquals("memstoreSize should be incremented", expectedSize, region.getMemStoreDataSize());
448    assertEquals("flushable size should be incremented", expectedSize,
449      store.getFlushableSize().getDataSize());
450
451    region.setCoprocessorHost(null);
452  }
453
454  /**
455   * A test case of HBASE-21041
456   */
457  @Test
458  public void testFlushAndMemstoreSizeCounting() throws Exception {
459    byte[] family = Bytes.toBytes("family");
460    this.region = initHRegion(tableName, method, CONF, family);
461    for (byte[] row : HBaseTestingUtil.ROWS) {
462      Put put = new Put(row);
463      put.addColumn(family, family, row);
464      region.put(put);
465    }
466    region.flush(true);
467    // After flush, data size should be zero
468    assertEquals(0, region.getMemStoreDataSize());
469    // After flush, a new active mutable segment is created, so the heap size
470    // should equal to MutableSegment.DEEP_OVERHEAD
471    assertEquals(MutableSegment.DEEP_OVERHEAD, region.getMemStoreHeapSize());
472    // After flush, offheap should be zero
473    assertEquals(0, region.getMemStoreOffHeapSize());
474  }
475
476  /**
477   * Test we do not lose data if we fail a flush and then close. Part of HBase-10466. Tests the
478   * following from the issue description: "Bug 1: Wrong calculation of HRegion.memstoreSize: When a
479   * flush fails, data to be flushed is kept in each MemStore's snapshot and wait for next flush
480   * attempt to continue on it. But when the next flush succeeds, the counter of total memstore size
481   * in HRegion is always deduced by the sum of current memstore sizes instead of snapshots left
482   * from previous failed flush. This calculation is problematic that almost every time there is
483   * failed flush, HRegion.memstoreSize gets reduced by a wrong value. If region flush could not
484   * proceed for a couple cycles, the size in current memstore could be much larger than the
485   * snapshot. It's likely to drift memstoreSize much smaller than expected. In extreme case, if the
486   * error accumulates to even bigger than HRegion's memstore size limit, any further flush is
487   * skipped because flush does not do anything if memstoreSize is not larger than 0."
488   */
489  @Test
490  public void testFlushSizeAccounting() throws Exception {
491    final Configuration conf = HBaseConfiguration.create(CONF);
492    final WAL wal = createWALCompatibleWithFaultyFileSystem(method, conf, tableName);
493    // Only retry once.
494    conf.setInt("hbase.hstore.flush.retries.number", 1);
495    final User user = User.createUserForTesting(conf, method, new String[] { "foo" });
496    // Inject our faulty LocalFileSystem
497    conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
498    user.runAs(new PrivilegedExceptionAction<Object>() {
499      @Override
500      public Object run() throws Exception {
501        // Make sure it worked (above is sensitive to caching details in hadoop core)
502        FileSystem fs = FileSystem.get(conf);
503        Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
504        FaultyFileSystem ffs = (FaultyFileSystem) fs;
505        HRegion region = null;
506        try {
507          // Initialize region
508          region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, wal,
509            COLUMN_FAMILY_BYTES);
510          long size = region.getMemStoreDataSize();
511          Assert.assertEquals(0, size);
512          // Put one item into memstore. Measure the size of one item in memstore.
513          Put p1 = new Put(row);
514          p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[]) null));
515          region.put(p1);
516          final long sizeOfOnePut = region.getMemStoreDataSize();
517          // Fail a flush which means the current memstore will hang out as memstore 'snapshot'.
518          try {
519            LOG.info("Flushing");
520            region.flush(true);
521            Assert.fail("Didn't bubble up IOE!");
522          } catch (DroppedSnapshotException dse) {
523            // What we are expecting
524            region.closing.set(false); // this is needed for the rest of the test to work
525          }
526          // Make it so all writes succeed from here on out
527          ffs.fault.set(false);
528          // Check sizes. Should still be the one entry.
529          Assert.assertEquals(sizeOfOnePut, region.getMemStoreDataSize());
530          // Now add two entries so that on this next flush that fails, we can see if we
531          // subtract the right amount, the snapshot size only.
532          Put p2 = new Put(row);
533          p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[]) null));
534          p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[]) null));
535          region.put(p2);
536          long expectedSize = sizeOfOnePut * 3;
537          Assert.assertEquals(expectedSize, region.getMemStoreDataSize());
538          // Do a successful flush. It will clear the snapshot only. Thats how flushes work.
539          // If already a snapshot, we clear it else we move the memstore to be snapshot and flush
540          // it
541          region.flush(true);
542          // Make sure our memory accounting is right.
543          Assert.assertEquals(sizeOfOnePut * 2, region.getMemStoreDataSize());
544        } finally {
545          HBaseTestingUtil.closeRegionAndWAL(region);
546        }
547        return null;
548      }
549    });
550    FileSystem.closeAllForUGI(user.getUGI());
551  }
552
553  @Test
554  public void testCloseWithFailingFlush() throws Exception {
555    final Configuration conf = HBaseConfiguration.create(CONF);
556    final WAL wal = createWALCompatibleWithFaultyFileSystem(method, conf, tableName);
557    // Only retry once.
558    conf.setInt("hbase.hstore.flush.retries.number", 1);
559    final User user = User.createUserForTesting(conf, this.method, new String[] { "foo" });
560    // Inject our faulty LocalFileSystem
561    conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
562    user.runAs(new PrivilegedExceptionAction<Object>() {
563      @Override
564      public Object run() throws Exception {
565        // Make sure it worked (above is sensitive to caching details in hadoop core)
566        FileSystem fs = FileSystem.get(conf);
567        Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
568        FaultyFileSystem ffs = (FaultyFileSystem) fs;
569        HRegion region = null;
570        try {
571          // Initialize region
572          region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, wal,
573            COLUMN_FAMILY_BYTES);
574          long size = region.getMemStoreDataSize();
575          Assert.assertEquals(0, size);
576          // Put one item into memstore. Measure the size of one item in memstore.
577          Put p1 = new Put(row);
578          p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[]) null));
579          region.put(p1);
580          // Manufacture an outstanding snapshot -- fake a failed flush by doing prepare step only.
581          HStore store = region.getStore(COLUMN_FAMILY_BYTES);
582          StoreFlushContext storeFlushCtx =
583            store.createFlushContext(12345, FlushLifeCycleTracker.DUMMY);
584          storeFlushCtx.prepare();
585          // Now add two entries to the foreground memstore.
586          Put p2 = new Put(row);
587          p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[]) null));
588          p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[]) null));
589          region.put(p2);
590          // Now try close on top of a failing flush.
591          HBaseTestingUtil.closeRegionAndWAL(region);
592          region = null;
593          fail();
594        } catch (DroppedSnapshotException dse) {
595          // Expected
596          LOG.info("Expected DroppedSnapshotException");
597        } finally {
598          // Make it so all writes succeed from here on out so can close clean
599          ffs.fault.set(false);
600          HBaseTestingUtil.closeRegionAndWAL(region);
601        }
602        return null;
603      }
604    });
605    FileSystem.closeAllForUGI(user.getUGI());
606  }
607
608  @Test
609  public void testCompactionAffectedByScanners() throws Exception {
610    byte[] family = Bytes.toBytes("family");
611    this.region = initHRegion(tableName, method, CONF, family);
612
613    Put put = new Put(Bytes.toBytes("r1"));
614    put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
615    region.put(put);
616    region.flush(true);
617
618    Scan scan = new Scan();
619    scan.readVersions(3);
620    // open the first scanner
621    try (RegionScanner scanner1 = region.getScanner(scan)) {
622      Delete delete = new Delete(Bytes.toBytes("r1"));
623      region.delete(delete);
624      region.flush(true);
625      // open the second scanner
626      try (RegionScanner scanner2 = region.getScanner(scan)) {
627        List<Cell> results = new ArrayList<>();
628
629        LOG.info("Smallest read point:" + region.getSmallestReadPoint());
630
631        // make a major compaction
632        region.compact(true);
633
634        // open the third scanner
635        try (RegionScanner scanner3 = region.getScanner(scan)) {
636          // get data from scanner 1, 2, 3 after major compaction
637          scanner1.next(results);
638          LOG.info(results.toString());
639          assertEquals(1, results.size());
640
641          results.clear();
642          scanner2.next(results);
643          LOG.info(results.toString());
644          assertEquals(0, results.size());
645
646          results.clear();
647          scanner3.next(results);
648          LOG.info(results.toString());
649          assertEquals(0, results.size());
650        }
651      }
652    }
653  }
654
655  @Test
656  public void testToShowNPEOnRegionScannerReseek() throws Exception {
657    byte[] family = Bytes.toBytes("family");
658    this.region = initHRegion(tableName, method, CONF, family);
659
660    Put put = new Put(Bytes.toBytes("r1"));
661    put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
662    region.put(put);
663    put = new Put(Bytes.toBytes("r2"));
664    put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
665    region.put(put);
666    region.flush(true);
667
668    Scan scan = new Scan();
669    scan.readVersions(3);
670    // open the first scanner
671    try (RegionScanner scanner1 = region.getScanner(scan)) {
672      LOG.info("Smallest read point:" + region.getSmallestReadPoint());
673
674      region.compact(true);
675
676      scanner1.reseek(Bytes.toBytes("r2"));
677      List<Cell> results = new ArrayList<>();
678      scanner1.next(results);
679      Cell keyValue = results.get(0);
680      assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
681      scanner1.close();
682    }
683  }
684
685  @Test
686  public void testArchiveRecoveredEditsReplay() throws Exception {
687    byte[] family = Bytes.toBytes("family");
688    this.region = initHRegion(tableName, method, CONF, family);
689    final WALFactory wals = new WALFactory(CONF, method);
690    try {
691      Path regiondir = region.getRegionFileSystem().getRegionDir();
692      FileSystem fs = region.getRegionFileSystem().getFileSystem();
693      byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
694
695      Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir);
696
697      long maxSeqId = 1050;
698      long minSeqId = 1000;
699
700      for (long i = minSeqId; i <= maxSeqId; i += 10) {
701        Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
702        fs.create(recoveredEdits);
703        WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
704
705        long time = System.nanoTime();
706        WALEdit edit = new WALEdit();
707        WALEditInternalHelper.addExtendedCell(edit,
708          new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i)));
709        writer.append(new WAL.Entry(
710          new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit));
711
712        writer.close();
713      }
714      MonitoredTask status = TaskMonitor.get().createStatus(method);
715      Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);
716      for (HStore store : region.getStores()) {
717        maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId - 1);
718      }
719      CONF.set("hbase.region.archive.recovered.edits", "true");
720      CONF.set(CommonFSUtils.HBASE_WAL_DIR, "/custom_wal_dir");
721      long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status);
722      assertEquals(maxSeqId, seqId);
723      region.getMVCC().advanceTo(seqId);
724      String fakeFamilyName = recoveredEditsDir.getName();
725      Path rootDir = new Path(CONF.get(HConstants.HBASE_DIR));
726      Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePathForRootDir(rootDir,
727        region.getRegionInfo(), Bytes.toBytes(fakeFamilyName));
728      FileStatus[] list = TEST_UTIL.getTestFileSystem().listStatus(storeArchiveDir);
729      assertEquals(6, list.length);
730    } finally {
731      CONF.set("hbase.region.archive.recovered.edits", "false");
732      CONF.set(CommonFSUtils.HBASE_WAL_DIR, "");
733      HBaseTestingUtil.closeRegionAndWAL(this.region);
734      this.region = null;
735      wals.close();
736    }
737  }
738
739  @Test
740  public void testSkipRecoveredEditsReplay() throws Exception {
741    byte[] family = Bytes.toBytes("family");
742    this.region = initHRegion(tableName, method, CONF, family);
743    final WALFactory wals = new WALFactory(CONF, method);
744    try {
745      Path regiondir = region.getRegionFileSystem().getRegionDir();
746      FileSystem fs = region.getRegionFileSystem().getFileSystem();
747      byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
748
749      Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir);
750
751      long maxSeqId = 1050;
752      long minSeqId = 1000;
753
754      for (long i = minSeqId; i <= maxSeqId; i += 10) {
755        Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
756        fs.create(recoveredEdits);
757        WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
758
759        long time = System.nanoTime();
760        WALEdit edit = new WALEdit();
761        WALEditInternalHelper.addExtendedCell(edit,
762          new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i)));
763        writer.append(new WAL.Entry(
764          new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit));
765
766        writer.close();
767      }
768      MonitoredTask status = TaskMonitor.get().createStatus(method);
769      Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);
770      for (HStore store : region.getStores()) {
771        maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId - 1);
772      }
773      long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status);
774      assertEquals(maxSeqId, seqId);
775      region.getMVCC().advanceTo(seqId);
776      Get get = new Get(row);
777      Result result = region.get(get);
778      for (long i = minSeqId; i <= maxSeqId; i += 10) {
779        List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
780        assertEquals(1, kvs.size());
781        assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
782      }
783    } finally {
784      HBaseTestingUtil.closeRegionAndWAL(this.region);
785      this.region = null;
786      wals.close();
787    }
788  }
789
790  @Test
791  public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
792    byte[] family = Bytes.toBytes("family");
793    this.region = initHRegion(tableName, method, CONF, family);
794    final WALFactory wals = new WALFactory(CONF, method);
795    try {
796      Path regiondir = region.getRegionFileSystem().getRegionDir();
797      FileSystem fs = region.getRegionFileSystem().getFileSystem();
798      byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
799
800      Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir);
801
802      long maxSeqId = 1050;
803      long minSeqId = 1000;
804
805      for (long i = minSeqId; i <= maxSeqId; i += 10) {
806        Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
807        fs.create(recoveredEdits);
808        WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
809
810        long time = System.nanoTime();
811        WALEdit edit = new WALEdit();
812        WALEditInternalHelper.addExtendedCell(edit,
813          new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i)));
814        writer.append(new WAL.Entry(
815          new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit));
816
817        writer.close();
818      }
819      long recoverSeqId = 1030;
820      MonitoredTask status = TaskMonitor.get().createStatus(method);
821      Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);
822      for (HStore store : region.getStores()) {
823        maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), recoverSeqId - 1);
824      }
825      long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status);
826      assertEquals(maxSeqId, seqId);
827      region.getMVCC().advanceTo(seqId);
828      Get get = new Get(row);
829      Result result = region.get(get);
830      for (long i = minSeqId; i <= maxSeqId; i += 10) {
831        List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
832        if (i < recoverSeqId) {
833          assertEquals(0, kvs.size());
834        } else {
835          assertEquals(1, kvs.size());
836          assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
837        }
838      }
839    } finally {
840      HBaseTestingUtil.closeRegionAndWAL(this.region);
841      this.region = null;
842      wals.close();
843    }
844  }
845
846  @Test
847  public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
848    byte[] family = Bytes.toBytes("family");
849    this.region = initHRegion(tableName, method, CONF, family);
850    Path regiondir = region.getRegionFileSystem().getRegionDir();
851    FileSystem fs = region.getRegionFileSystem().getFileSystem();
852
853    Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir);
854    for (int i = 1000; i < 1050; i += 10) {
855      Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
856      FSDataOutputStream dos = fs.create(recoveredEdits);
857      dos.writeInt(i);
858      dos.close();
859    }
860    long minSeqId = 2000;
861    Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
862    FSDataOutputStream dos = fs.create(recoveredEdits);
863    dos.close();
864
865    Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);
866    for (HStore store : region.getStores()) {
867      maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId);
868    }
869    long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, null);
870    assertEquals(minSeqId, seqId);
871  }
872
873  @Test
874  public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception {
875    byte[] family = Bytes.toBytes("family");
876    this.region = initHRegion(tableName, method, CONF, family);
877    final WALFactory wals = new WALFactory(CONF, method);
878    try {
879      Path regiondir = region.getRegionFileSystem().getRegionDir();
880      FileSystem fs = region.getRegionFileSystem().getFileSystem();
881      byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
882      byte[][] columns = region.getTableDescriptor().getColumnFamilyNames().toArray(new byte[0][]);
883
884      assertEquals(0, region.getStoreFileList(columns).size());
885
886      Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir);
887
888      long maxSeqId = 1050;
889      long minSeqId = 1000;
890
891      for (long i = minSeqId; i <= maxSeqId; i += 10) {
892        Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
893        fs.create(recoveredEdits);
894        WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
895
896        long time = System.nanoTime();
897        WALEdit edit = null;
898        if (i == maxSeqId) {
899          edit = WALEdit.createCompaction(region.getRegionInfo(),
900            CompactionDescriptor.newBuilder().setTableName(ByteString.copyFrom(tableName.getName()))
901              .setFamilyName(ByteString.copyFrom(regionName))
902              .setEncodedRegionName(ByteString.copyFrom(regionName))
903              .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString())))
904              .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName())).build());
905        } else {
906          edit = new WALEdit();
907          WALEditInternalHelper.addExtendedCell(edit,
908            new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i)));
909        }
910        writer.append(new WAL.Entry(
911          new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit));
912        writer.close();
913      }
914
915      long recoverSeqId = 1030;
916      Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);
917      MonitoredTask status = TaskMonitor.get().createStatus(method);
918      for (HStore store : region.getStores()) {
919        maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), recoverSeqId - 1);
920      }
921      long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status);
922      assertEquals(maxSeqId, seqId);
923
924      // assert that the files are flushed
925      assertEquals(1, region.getStoreFileList(columns).size());
926
927    } finally {
928      HBaseTestingUtil.closeRegionAndWAL(this.region);
929      this.region = null;
930      wals.close();
931    }
932  }
933
934  @Test
935  public void testRecoveredEditsReplayCompaction() throws Exception {
936    testRecoveredEditsReplayCompaction(false);
937    testRecoveredEditsReplayCompaction(true);
938  }
939
940  public void testRecoveredEditsReplayCompaction(boolean mismatchedRegionName) throws Exception {
941    CONF.setClass(HConstants.REGION_IMPL, HRegionForTesting.class, Region.class);
942    byte[] family = Bytes.toBytes("family");
943    this.region = initHRegion(tableName, method, CONF, family);
944    final WALFactory wals = new WALFactory(CONF, method);
945    try {
946      Path regiondir = region.getRegionFileSystem().getRegionDir();
947      FileSystem fs = region.getRegionFileSystem().getFileSystem();
948      byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
949
950      long maxSeqId = 3;
951      long minSeqId = 0;
952
953      for (long i = minSeqId; i < maxSeqId; i++) {
954        Put put = new Put(Bytes.toBytes(i));
955        put.addColumn(family, Bytes.toBytes(i), Bytes.toBytes(i));
956        region.put(put);
957        region.flush(true);
958      }
959
960      // this will create a region with 3 files
961      assertEquals(3, region.getStore(family).getStorefilesCount());
962      List<Path> storeFiles = new ArrayList<>(3);
963      for (HStoreFile sf : region.getStore(family).getStorefiles()) {
964        storeFiles.add(sf.getPath());
965      }
966
967      // disable compaction completion
968      CONF.setBoolean("hbase.hstore.compaction.complete", false);
969      region.compactStores();
970
971      // ensure that nothing changed
972      assertEquals(3, region.getStore(family).getStorefilesCount());
973
974      // now find the compacted file, and manually add it to the recovered edits
975      Path tmpDir = new Path(region.getRegionFileSystem().getTempDir(), Bytes.toString(family));
976      FileStatus[] files = CommonFSUtils.listStatus(fs, tmpDir);
977      String errorMsg = "Expected to find 1 file in the region temp directory "
978        + "from the compaction, could not find any";
979      assertNotNull(errorMsg, files);
980      assertEquals(errorMsg, 1, files.length);
981      // move the file inside region dir
982      Path newFile =
983        region.getRegionFileSystem().commitStoreFile(Bytes.toString(family), files[0].getPath());
984
985      byte[] encodedNameAsBytes = this.region.getRegionInfo().getEncodedNameAsBytes();
986      byte[] fakeEncodedNameAsBytes = new byte[encodedNameAsBytes.length];
987      for (int i = 0; i < encodedNameAsBytes.length; i++) {
988        // Mix the byte array to have a new encodedName
989        fakeEncodedNameAsBytes[i] = (byte) (encodedNameAsBytes[i] + 1);
990      }
991
992      CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(
993        this.region.getRegionInfo(), mismatchedRegionName ? fakeEncodedNameAsBytes : null, family,
994        storeFiles, Lists.newArrayList(newFile),
995        region.getRegionFileSystem().getStoreDir(Bytes.toString(family)));
996
997      WALUtil.writeCompactionMarker(region.getWAL(), this.region.getReplicationScope(),
998        this.region.getRegionInfo(), compactionDescriptor, region.getMVCC(), null);
999
1000      Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir);
1001
1002      Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
1003      fs.create(recoveredEdits);
1004      WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
1005
1006      long time = System.nanoTime();
1007
1008      writer.append(new WAL.Entry(
1009        new WALKeyImpl(regionName, tableName, 10, time, HConstants.DEFAULT_CLUSTER_ID),
1010        WALEdit.createCompaction(region.getRegionInfo(), compactionDescriptor)));
1011      writer.close();
1012
1013      // close the region now, and reopen again
1014      region.getTableDescriptor();
1015      region.getRegionInfo();
1016      HBaseTestingUtil.closeRegionAndWAL(this.region);
1017      try {
1018        region = HRegion.openHRegion(region, null);
1019      } catch (WrongRegionException wre) {
1020        fail("Matching encoded region name should not have produced WrongRegionException");
1021      }
1022
1023      // now check whether we have only one store file, the compacted one
1024      Collection<HStoreFile> sfs = region.getStore(family).getStorefiles();
1025      for (HStoreFile sf : sfs) {
1026        LOG.info(Objects.toString(sf.getPath()));
1027      }
1028      if (!mismatchedRegionName) {
1029        assertEquals(1, region.getStore(family).getStorefilesCount());
1030      }
1031      files = CommonFSUtils.listStatus(fs, tmpDir);
1032      assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);
1033
1034      for (long i = minSeqId; i < maxSeqId; i++) {
1035        Get get = new Get(Bytes.toBytes(i));
1036        Result result = region.get(get);
1037        byte[] value = result.getValue(family, Bytes.toBytes(i));
1038        assertArrayEquals(Bytes.toBytes(i), value);
1039      }
1040    } finally {
1041      HBaseTestingUtil.closeRegionAndWAL(this.region);
1042      this.region = null;
1043      wals.close();
1044      CONF.setClass(HConstants.REGION_IMPL, HRegion.class, Region.class);
1045    }
1046  }
1047
1048  @Test
1049  public void testFlushMarkers() throws Exception {
1050    // tests that flush markers are written to WAL and handled at recovered edits
1051    byte[] family = Bytes.toBytes("family");
1052    Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + ".log");
1053    final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration());
1054    CommonFSUtils.setRootDir(walConf, logDir);
1055    final WALFactory wals = new WALFactory(walConf, method);
1056    final WAL wal = wals.getWAL(RegionInfoBuilder.newBuilder(tableName).build());
1057
1058    this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF,
1059      false, Durability.USE_DEFAULT, wal, family);
1060    try {
1061      Path regiondir = region.getRegionFileSystem().getRegionDir();
1062      FileSystem fs = region.getRegionFileSystem().getFileSystem();
1063      byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
1064
1065      long maxSeqId = 3;
1066      long minSeqId = 0;
1067
1068      for (long i = minSeqId; i < maxSeqId; i++) {
1069        Put put = new Put(Bytes.toBytes(i));
1070        put.addColumn(family, Bytes.toBytes(i), Bytes.toBytes(i));
1071        region.put(put);
1072        region.flush(true);
1073      }
1074
1075      // this will create a region with 3 files from flush
1076      assertEquals(3, region.getStore(family).getStorefilesCount());
1077      List<String> storeFiles = new ArrayList<>(3);
1078      for (HStoreFile sf : region.getStore(family).getStorefiles()) {
1079        storeFiles.add(sf.getPath().getName());
1080      }
1081
1082      // now verify that the flush markers are written
1083      wal.shutdown();
1084      WALStreamReader reader = WALFactory.createStreamReader(fs,
1085        AbstractFSWALProvider.getCurrentFileName(wal), TEST_UTIL.getConfiguration());
1086      try {
1087        List<WAL.Entry> flushDescriptors = new ArrayList<>();
1088        long lastFlushSeqId = -1;
1089        while (true) {
1090          WAL.Entry entry = reader.next();
1091          if (entry == null) {
1092            break;
1093          }
1094          Cell cell = entry.getEdit().getCells().get(0);
1095          if (WALEdit.isMetaEditFamily(cell)) {
1096            FlushDescriptor flushDesc = WALEdit.getFlushDescriptor(cell);
1097            assertNotNull(flushDesc);
1098            assertArrayEquals(tableName.getName(), flushDesc.getTableName().toByteArray());
1099            if (flushDesc.getAction() == FlushAction.START_FLUSH) {
1100              assertTrue(flushDesc.getFlushSequenceNumber() > lastFlushSeqId);
1101            } else if (flushDesc.getAction() == FlushAction.COMMIT_FLUSH) {
1102              assertTrue(flushDesc.getFlushSequenceNumber() == lastFlushSeqId);
1103            }
1104            lastFlushSeqId = flushDesc.getFlushSequenceNumber();
1105            assertArrayEquals(regionName, flushDesc.getEncodedRegionName().toByteArray());
1106            assertEquals(1, flushDesc.getStoreFlushesCount()); // only one store
1107            StoreFlushDescriptor storeFlushDesc = flushDesc.getStoreFlushes(0);
1108            assertArrayEquals(family, storeFlushDesc.getFamilyName().toByteArray());
1109            assertEquals("family", storeFlushDesc.getStoreHomeDir());
1110            if (flushDesc.getAction() == FlushAction.START_FLUSH) {
1111              assertEquals(0, storeFlushDesc.getFlushOutputCount());
1112            } else {
1113              assertEquals(1, storeFlushDesc.getFlushOutputCount()); // only one file from flush
1114              assertTrue(storeFiles.contains(storeFlushDesc.getFlushOutput(0)));
1115            }
1116
1117            flushDescriptors.add(entry);
1118          }
1119        }
1120
1121        assertEquals(3 * 2, flushDescriptors.size()); // START_FLUSH and COMMIT_FLUSH per flush
1122
1123        // now write those markers to the recovered edits again.
1124
1125        Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir);
1126
1127        Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
1128        fs.create(recoveredEdits);
1129        WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
1130
1131        for (WAL.Entry entry : flushDescriptors) {
1132          writer.append(entry);
1133        }
1134        writer.close();
1135      } finally {
1136        reader.close();
1137      }
1138
1139      // close the region now, and reopen again
1140      HBaseTestingUtil.closeRegionAndWAL(this.region);
1141      region = HRegion.openHRegion(region, null);
1142
1143      // now check whether we have can read back the data from region
1144      for (long i = minSeqId; i < maxSeqId; i++) {
1145        Get get = new Get(Bytes.toBytes(i));
1146        Result result = region.get(get);
1147        byte[] value = result.getValue(family, Bytes.toBytes(i));
1148        assertArrayEquals(Bytes.toBytes(i), value);
1149      }
1150    } finally {
1151      HBaseTestingUtil.closeRegionAndWAL(this.region);
1152      this.region = null;
1153      wals.close();
1154    }
1155  }
1156
1157  static class IsFlushWALMarker implements ArgumentMatcher<WALEdit> {
1158    volatile FlushAction[] actions;
1159
1160    public IsFlushWALMarker(FlushAction... actions) {
1161      this.actions = actions;
1162    }
1163
1164    @Override
1165    public boolean matches(WALEdit edit) {
1166      List<Cell> cells = edit.getCells();
1167      if (cells.isEmpty()) {
1168        return false;
1169      }
1170      if (WALEdit.isMetaEditFamily(cells.get(0))) {
1171        FlushDescriptor desc;
1172        try {
1173          desc = WALEdit.getFlushDescriptor(cells.get(0));
1174        } catch (IOException e) {
1175          LOG.warn(e.toString(), e);
1176          return false;
1177        }
1178        if (desc != null) {
1179          for (FlushAction action : actions) {
1180            if (desc.getAction() == action) {
1181              return true;
1182            }
1183          }
1184        }
1185      }
1186      return false;
1187    }
1188
1189    public IsFlushWALMarker set(FlushAction... actions) {
1190      this.actions = actions;
1191      return this;
1192    }
1193  }
1194
1195  @Test
1196  public void testFlushMarkersWALFail() throws Exception {
1197    // test the cases where the WAL append for flush markers fail.
1198    byte[] family = Bytes.toBytes("family");
1199
1200    // spy an actual WAL implementation to throw exception (was not able to mock)
1201    Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + "log");
1202
1203    final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration());
1204    CommonFSUtils.setRootDir(walConf, logDir);
1205    // Make up a WAL that we can manipulate at append time.
1206    class FailAppendFlushMarkerWAL extends FSHLog {
1207      volatile FlushAction[] flushActions = null;
1208
1209      public FailAppendFlushMarkerWAL(FileSystem fs, Path root, String logDir, Configuration conf)
1210        throws IOException {
1211        super(fs, root, logDir, conf);
1212      }
1213
1214      @Override
1215      protected Writer createWriterInstance(FileSystem fs, Path path) throws IOException {
1216        final Writer w = super.createWriterInstance(fs, path);
1217        return new Writer() {
1218          @Override
1219          public void close() throws IOException {
1220            w.close();
1221          }
1222
1223          @Override
1224          public void sync(boolean forceSync) throws IOException {
1225            w.sync(forceSync);
1226          }
1227
1228          @Override
1229          public void append(Entry entry) throws IOException {
1230            List<Cell> cells = entry.getEdit().getCells();
1231            if (WALEdit.isMetaEditFamily(cells.get(0))) {
1232              FlushDescriptor desc = WALEdit.getFlushDescriptor(cells.get(0));
1233              if (desc != null) {
1234                for (FlushAction flushAction : flushActions) {
1235                  if (desc.getAction().equals(flushAction)) {
1236                    throw new IOException("Failed to append flush marker! " + flushAction);
1237                  }
1238                }
1239              }
1240            }
1241            w.append(entry);
1242          }
1243
1244          @Override
1245          public long getLength() {
1246            return w.getLength();
1247          }
1248
1249          @Override
1250          public long getSyncedLength() {
1251            return w.getSyncedLength();
1252          }
1253        };
1254      }
1255    }
1256    FailAppendFlushMarkerWAL wal = new FailAppendFlushMarkerWAL(FileSystem.get(walConf),
1257      CommonFSUtils.getRootDir(walConf), method, walConf);
1258    wal.init();
1259    this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF,
1260      false, Durability.USE_DEFAULT, wal, family);
1261    int i = 0;
1262    Put put = new Put(Bytes.toBytes(i));
1263    put.setDurability(Durability.SKIP_WAL); // have to skip mocked wal
1264    put.addColumn(family, Bytes.toBytes(i), Bytes.toBytes(i));
1265    region.put(put);
1266
1267    // 1. Test case where START_FLUSH throws exception
1268    wal.flushActions = new FlushAction[] { FlushAction.START_FLUSH };
1269
1270    // start cache flush will throw exception
1271    try {
1272      region.flush(true);
1273      fail("This should have thrown exception");
1274    } catch (DroppedSnapshotException unexpected) {
1275      // this should not be a dropped snapshot exception. Meaning that RS will not abort
1276      throw unexpected;
1277    } catch (IOException expected) {
1278      // expected
1279    }
1280    // The WAL is hosed now. It has two edits appended. We cannot roll the log without it
1281    // throwing a DroppedSnapshotException to force an abort. Just clean up the mess.
1282    region.close(true);
1283    wal.close();
1284    // release the snapshot and active segment, so netty will not report memory leak
1285    for (HStore store : region.getStores()) {
1286      AbstractMemStore memstore = (AbstractMemStore) store.memstore;
1287      memstore.doClearSnapShot();
1288      memstore.close();
1289    }
1290
1291    // 2. Test case where START_FLUSH succeeds but COMMIT_FLUSH will throw exception
1292    wal.flushActions = new FlushAction[] { FlushAction.COMMIT_FLUSH };
1293    wal = new FailAppendFlushMarkerWAL(FileSystem.get(walConf), CommonFSUtils.getRootDir(walConf),
1294      method, walConf);
1295    wal.init();
1296    this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF,
1297      false, Durability.USE_DEFAULT, wal, family);
1298    region.put(put);
1299    // 3. Test case where ABORT_FLUSH will throw exception.
1300    // Even if ABORT_FLUSH throws exception, we should not fail with IOE, but continue with
1301    // DroppedSnapshotException. Below COMMIT_FLUSH will cause flush to abort
1302    wal.flushActions = new FlushAction[] { FlushAction.COMMIT_FLUSH, FlushAction.ABORT_FLUSH };
1303
1304    // we expect this exception, since we were able to write the snapshot, but failed to
1305    // write the flush marker to WAL
1306    assertThrows(DroppedSnapshotException.class, () -> region.flush(true));
1307
1308    region.close(true);
1309    // release the snapshot and active segment, so netty will not report memory leak
1310    for (HStore store : region.getStores()) {
1311      AbstractMemStore memstore = (AbstractMemStore) store.memstore;
1312      memstore.doClearSnapShot();
1313      memstore.close();
1314    }
1315    region = null;
1316  }
1317
1318  @Test
1319  public void testGetWhileRegionClose() throws IOException {
1320    Configuration hc = initSplit();
1321    int numRows = 100;
1322    byte[][] families = { fam1, fam2, fam3 };
1323
1324    // Setting up region
1325    this.region = initHRegion(tableName, method, hc, families);
1326    // Put data in region
1327    final int startRow = 100;
1328    putData(startRow, numRows, qual1, families);
1329    putData(startRow, numRows, qual2, families);
1330    putData(startRow, numRows, qual3, families);
1331    final AtomicBoolean done = new AtomicBoolean(false);
1332    final AtomicInteger gets = new AtomicInteger(0);
1333    GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
1334    try {
1335      // Set ten threads running concurrently getting from the region.
1336      for (int i = 0; i < threads.length / 2; i++) {
1337        threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
1338        threads[i].setDaemon(true);
1339        threads[i].start();
1340      }
1341      // Artificially make the condition by setting closing flag explicitly.
1342      // I can't make the issue happen with a call to region.close().
1343      this.region.closing.set(true);
1344      for (int i = threads.length / 2; i < threads.length; i++) {
1345        threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
1346        threads[i].setDaemon(true);
1347        threads[i].start();
1348      }
1349    } finally {
1350      if (this.region != null) {
1351        HBaseTestingUtil.closeRegionAndWAL(this.region);
1352        this.region = null;
1353      }
1354    }
1355    done.set(true);
1356    for (GetTillDoneOrException t : threads) {
1357      try {
1358        t.join();
1359      } catch (InterruptedException e) {
1360        e.printStackTrace();
1361      }
1362      if (t.e != null) {
1363        LOG.info("Exception=" + t.e);
1364        assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
1365      }
1366    }
1367  }
1368
1369  /*
1370   * Thread that does get on single row until 'done' flag is flipped. If an exception causes us to
1371   * fail, it records it.
1372   */
1373  class GetTillDoneOrException extends Thread {
1374    private final Get g;
1375    private final AtomicBoolean done;
1376    private final AtomicInteger count;
1377    private Exception e;
1378
1379    GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d,
1380      final AtomicInteger c) {
1381      super("getter." + i);
1382      this.g = new Get(r);
1383      this.done = d;
1384      this.count = c;
1385    }
1386
1387    @Override
1388    public void run() {
1389      while (!this.done.get()) {
1390        try {
1391          assertTrue(region.get(g).size() > 0);
1392          this.count.incrementAndGet();
1393        } catch (Exception e) {
1394          this.e = e;
1395          break;
1396        }
1397      }
1398    }
1399  }
1400
1401  /*
1402   * An involved filter test. Has multiple column families and deletes in mix.
1403   */
1404  @Test
1405  public void testWeirdCacheBehaviour() throws Exception {
1406    final TableName tableName = TableName.valueOf(name.getMethodName());
1407    byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
1408      Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
1409    this.region = initHRegion(tableName, method, CONF, FAMILIES);
1410    String value = "this is the value";
1411    String value2 = "this is some other value";
1412    String keyPrefix1 = "prefix1";
1413    String keyPrefix2 = "prefix2";
1414    String keyPrefix3 = "prefix3";
1415    putRows(this.region, 3, value, keyPrefix1);
1416    putRows(this.region, 3, value, keyPrefix2);
1417    putRows(this.region, 3, value, keyPrefix3);
1418    putRows(this.region, 3, value2, keyPrefix1);
1419    putRows(this.region, 3, value2, keyPrefix2);
1420    putRows(this.region, 3, value2, keyPrefix3);
1421    System.out.println("Checking values for key: " + keyPrefix1);
1422    assertEquals("Got back incorrect number of rows from scan", 3,
1423      getNumberOfRows(keyPrefix1, value2, this.region));
1424    System.out.println("Checking values for key: " + keyPrefix2);
1425    assertEquals("Got back incorrect number of rows from scan", 3,
1426      getNumberOfRows(keyPrefix2, value2, this.region));
1427    System.out.println("Checking values for key: " + keyPrefix3);
1428    assertEquals("Got back incorrect number of rows from scan", 3,
1429      getNumberOfRows(keyPrefix3, value2, this.region));
1430    deleteColumns(this.region, value2, keyPrefix1);
1431    deleteColumns(this.region, value2, keyPrefix2);
1432    deleteColumns(this.region, value2, keyPrefix3);
1433    System.out.println("Starting important checks.....");
1434    assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
1435      getNumberOfRows(keyPrefix1, value2, this.region));
1436    assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
1437      getNumberOfRows(keyPrefix2, value2, this.region));
1438    assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
1439      getNumberOfRows(keyPrefix3, value2, this.region));
1440  }
1441
1442  @Test
1443  public void testAppendWithReadOnlyTable() throws Exception {
1444    final TableName tableName = TableName.valueOf(name.getMethodName());
1445    this.region = initHRegion(tableName, method, CONF, true, Bytes.toBytes("somefamily"));
1446    boolean exceptionCaught = false;
1447    Append append = new Append(Bytes.toBytes("somerow"));
1448    append.setDurability(Durability.SKIP_WAL);
1449    append.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
1450      Bytes.toBytes("somevalue"));
1451    try {
1452      region.append(append);
1453    } catch (IOException e) {
1454      exceptionCaught = true;
1455    }
1456    assertTrue(exceptionCaught == true);
1457  }
1458
1459  @Test
1460  public void testIncrWithReadOnlyTable() throws Exception {
1461    final TableName tableName = TableName.valueOf(name.getMethodName());
1462    this.region = initHRegion(tableName, method, CONF, true, Bytes.toBytes("somefamily"));
1463    boolean exceptionCaught = false;
1464    Increment inc = new Increment(Bytes.toBytes("somerow"));
1465    inc.setDurability(Durability.SKIP_WAL);
1466    inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
1467    try {
1468      region.increment(inc);
1469    } catch (IOException e) {
1470      exceptionCaught = true;
1471    }
1472    assertTrue(exceptionCaught == true);
1473  }
1474
1475  private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
1476    int count = 0;
1477    try (InternalScanner scanner = buildScanner(keyPrefix, value, r)) {
1478      boolean more = false;
1479      List<Cell> results = new ArrayList<>();
1480      do {
1481        more = scanner.next(results);
1482        if (results != null && !results.isEmpty()) {
1483          count++;
1484        } else {
1485          break;
1486        }
1487        Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
1488        delete.addColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
1489        r.delete(delete);
1490        results.clear();
1491      } while (more);
1492    }
1493    assertEquals("Did not perform correct number of deletes", 3, count);
1494  }
1495
1496  private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
1497    try (InternalScanner resultScanner = buildScanner(keyPrefix, value, r)) {
1498      int numberOfResults = 0;
1499      List<Cell> results = new ArrayList<>();
1500      boolean more = false;
1501      do {
1502        more = resultScanner.next(results);
1503        if (results != null && !results.isEmpty()) {
1504          numberOfResults++;
1505        } else {
1506          break;
1507        }
1508        for (Cell kv : results) {
1509          LOG.info("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
1510        }
1511        results.clear();
1512      } while (more);
1513      return numberOfResults;
1514    }
1515  }
1516
1517  private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
1518    throws IOException {
1519    // Defaults FilterList.Operator.MUST_PASS_ALL.
1520    FilterList allFilters = new FilterList();
1521    allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
1522    // Only return rows where this column value exists in the row.
1523    SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
1524      Bytes.toBytes("qual2"), CompareOperator.EQUAL, Bytes.toBytes(value));
1525    filter.setFilterIfMissing(true);
1526    allFilters.addFilter(filter);
1527    Scan scan = new Scan();
1528    scan.addFamily(Bytes.toBytes("trans-blob"));
1529    scan.addFamily(Bytes.toBytes("trans-type"));
1530    scan.addFamily(Bytes.toBytes("trans-date"));
1531    scan.addFamily(Bytes.toBytes("trans-tags"));
1532    scan.addFamily(Bytes.toBytes("trans-group"));
1533    scan.setFilter(allFilters);
1534    return r.getScanner(scan);
1535  }
1536
1537  private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
1538    for (int i = 0; i < numRows; i++) {
1539      String row = key + "_" + i/* UUID.randomUUID().toString() */;
1540      System.out.println(String.format("Saving row: %s, with value %s", row, value));
1541      Put put = new Put(Bytes.toBytes(row));
1542      put.setDurability(Durability.SKIP_WAL);
1543      put.addColumn(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
1544      put.addColumn(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
1545      put.addColumn(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
1546      put.addColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
1547      put.addColumn(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
1548      r.put(put);
1549    }
1550  }
1551
1552  @Test
1553  public void testFamilyWithAndWithoutColon() throws Exception {
1554    byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1555    this.region = initHRegion(tableName, method, CONF, cf);
1556    Put p = new Put(tableName.toBytes());
1557    byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
1558    p.addColumn(cfwithcolon, cfwithcolon, cfwithcolon);
1559    boolean exception = false;
1560    try {
1561      this.region.put(p);
1562    } catch (NoSuchColumnFamilyException e) {
1563      exception = true;
1564    }
1565    assertTrue(exception);
1566  }
1567
1568  @Test
1569  public void testBatchPut_whileNoRowLocksHeld() throws IOException {
1570    final Put[] puts = new Put[10];
1571    MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1572    long syncs = prepareRegionForBachPut(puts, source, false);
1573
1574    OperationStatus[] codes = this.region.batchMutate(puts);
1575    assertEquals(10, codes.length);
1576    for (int i = 0; i < 10; i++) {
1577      assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
1578    }
1579    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
1580
1581    LOG.info("Next a batch put with one invalid family");
1582    puts[5].addColumn(Bytes.toBytes("BAD_CF"), qual, value);
1583    codes = this.region.batchMutate(puts);
1584    assertEquals(10, codes.length);
1585    for (int i = 0; i < 10; i++) {
1586      assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1587        codes[i].getOperationStatusCode());
1588    }
1589
1590    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
1591  }
1592
1593  @Test
1594  public void testBatchPut_whileMultipleRowLocksHeld() throws Exception {
1595    final Put[] puts = new Put[10];
1596    MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1597    long syncs = prepareRegionForBachPut(puts, source, false);
1598
1599    puts[5].addColumn(Bytes.toBytes("BAD_CF"), qual, value);
1600
1601    LOG.info("batchPut will have to break into four batches to avoid row locks");
1602    RowLock rowLock1 = region.getRowLock(Bytes.toBytes("row_2"));
1603    RowLock rowLock2 = region.getRowLock(Bytes.toBytes("row_1"));
1604    RowLock rowLock3 = region.getRowLock(Bytes.toBytes("row_3"));
1605    RowLock rowLock4 = region.getRowLock(Bytes.toBytes("row_3"), true);
1606
1607    MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
1608    final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<>();
1609    final CountDownLatch startingPuts = new CountDownLatch(1);
1610    final CountDownLatch startingClose = new CountDownLatch(1);
1611    TestThread putter = new TestThread(ctx) {
1612      @Override
1613      public void doWork() throws IOException {
1614        startingPuts.countDown();
1615        retFromThread.set(region.batchMutate(puts));
1616      }
1617    };
1618    LOG.info("...starting put thread while holding locks");
1619    ctx.addThread(putter);
1620    ctx.startThreads();
1621
1622    // Now attempt to close the region from another thread. Prior to HBASE-12565
1623    // this would cause the in-progress batchMutate operation to to fail with
1624    // exception because it use to release and re-acquire the close-guard lock
1625    // between batches. Caller then didn't get status indicating which writes succeeded.
1626    // We now expect this thread to block until the batchMutate call finishes.
1627    Thread regionCloseThread = new TestThread(ctx) {
1628      @Override
1629      public void doWork() {
1630        try {
1631          startingPuts.await();
1632          // Give some time for the batch mutate to get in.
1633          // We don't want to race with the mutate
1634          Thread.sleep(10);
1635          startingClose.countDown();
1636          HBaseTestingUtil.closeRegionAndWAL(region);
1637          region = null;
1638        } catch (IOException e) {
1639          throw new RuntimeException(e);
1640        } catch (InterruptedException e) {
1641          throw new RuntimeException(e);
1642        }
1643      }
1644    };
1645    regionCloseThread.start();
1646
1647    startingClose.await();
1648    startingPuts.await();
1649    Thread.sleep(100);
1650    LOG.info("...releasing row lock 1, which should let put thread continue");
1651    rowLock1.release();
1652    rowLock2.release();
1653    rowLock3.release();
1654    waitForCounter(source, "syncTimeNumOps", syncs + 1);
1655
1656    LOG.info("...joining on put thread");
1657    ctx.stop();
1658    regionCloseThread.join();
1659
1660    OperationStatus[] codes = retFromThread.get();
1661    for (int i = 0; i < codes.length; i++) {
1662      assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1663        codes[i].getOperationStatusCode());
1664    }
1665    rowLock4.release();
1666  }
1667
1668  private void waitForCounter(MetricsWALSource source, String metricName, long expectedCount)
1669    throws InterruptedException {
1670    long startWait = EnvironmentEdgeManager.currentTime();
1671    long currentCount;
1672    while ((currentCount = metricsAssertHelper.getCounter(metricName, source)) < expectedCount) {
1673      Thread.sleep(100);
1674      if (EnvironmentEdgeManager.currentTime() - startWait > 10000) {
1675        fail(String.format("Timed out waiting for '%s' >= '%s', currentCount=%s", metricName,
1676          expectedCount, currentCount));
1677      }
1678    }
1679  }
1680
1681  @Test
1682  public void testAtomicBatchPut() throws IOException {
1683    final Put[] puts = new Put[10];
1684    MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1685    long syncs = prepareRegionForBachPut(puts, source, false);
1686
1687    // 1. Straight forward case, should succeed
1688    OperationStatus[] codes = this.region.batchMutate(puts, true);
1689    assertEquals(10, codes.length);
1690    for (int i = 0; i < 10; i++) {
1691      assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
1692    }
1693    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
1694
1695    // 2. Failed to get lock
1696    RowLock lock = region.getRowLock(Bytes.toBytes("row_" + 3));
1697    // Method {@link HRegion#getRowLock(byte[])} is reentrant. As 'row_3' is locked in this
1698    // thread, need to run {@link HRegion#batchMutate(HRegion.BatchOperation)} in different thread
1699    MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
1700    final AtomicReference<IOException> retFromThread = new AtomicReference<>();
1701    final CountDownLatch finishedPuts = new CountDownLatch(1);
1702    TestThread putter = new TestThread(ctx) {
1703      @Override
1704      public void doWork() throws IOException {
1705        try {
1706          region.batchMutate(puts, true);
1707        } catch (IOException ioe) {
1708          LOG.error("test failed!", ioe);
1709          retFromThread.set(ioe);
1710        }
1711        finishedPuts.countDown();
1712      }
1713    };
1714    LOG.info("...starting put thread while holding locks");
1715    ctx.addThread(putter);
1716    ctx.startThreads();
1717    LOG.info("...waiting for batch puts while holding locks");
1718    try {
1719      finishedPuts.await();
1720    } catch (InterruptedException e) {
1721      LOG.error("Interrupted!", e);
1722    } finally {
1723      if (lock != null) {
1724        lock.release();
1725      }
1726    }
1727    assertNotNull(retFromThread.get());
1728    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
1729
1730    // 3. Exception thrown in validation
1731    LOG.info("Next a batch put with one invalid family");
1732    puts[5].addColumn(Bytes.toBytes("BAD_CF"), qual, value);
1733    thrown.expect(NoSuchColumnFamilyException.class);
1734    this.region.batchMutate(puts, true);
1735  }
1736
1737  @Test
1738  public void testBatchPutWithTsSlop() throws Exception {
1739    // add data with a timestamp that is too recent for range. Ensure assert
1740    CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1741    final Put[] puts = new Put[10];
1742    MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1743
1744    long syncs = prepareRegionForBachPut(puts, source, true);
1745
1746    OperationStatus[] codes = this.region.batchMutate(puts);
1747    assertEquals(10, codes.length);
1748    for (int i = 0; i < 10; i++) {
1749      assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
1750    }
1751    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1752  }
1753
1754  /** Returns syncs initial syncTimeNumOps */
1755  private long prepareRegionForBachPut(final Put[] puts, final MetricsWALSource source,
1756    boolean slop) throws IOException {
1757    this.region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES);
1758
1759    LOG.info("First a batch put with all valid puts");
1760    for (int i = 0; i < puts.length; i++) {
1761      puts[i] = slop
1762        ? new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100)
1763        : new Put(Bytes.toBytes("row_" + i));
1764      puts[i].addColumn(COLUMN_FAMILY_BYTES, qual, value);
1765    }
1766
1767    long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1768    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1769    return syncs;
1770  }
1771
1772  // ////////////////////////////////////////////////////////////////////////////
1773  // checkAndMutate tests
1774  // ////////////////////////////////////////////////////////////////////////////
1775  @Test
1776  @Deprecated
1777  public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
1778    byte[] row1 = Bytes.toBytes("row1");
1779    byte[] fam1 = Bytes.toBytes("fam1");
1780    byte[] qf1 = Bytes.toBytes("qualifier");
1781    byte[] emptyVal = new byte[] {};
1782    byte[] val1 = Bytes.toBytes("value1");
1783    byte[] val2 = Bytes.toBytes("value2");
1784
1785    // Setting up region
1786    this.region = initHRegion(tableName, method, CONF, fam1);
1787    // Putting empty data in key
1788    Put put = new Put(row1);
1789    put.addColumn(fam1, qf1, emptyVal);
1790
1791    // checkAndPut with empty value
1792    boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
1793      new BinaryComparator(emptyVal), put);
1794    assertTrue(res);
1795
1796    // Putting data in key
1797    put = new Put(row1);
1798    put.addColumn(fam1, qf1, val1);
1799
1800    // checkAndPut with correct value
1801    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
1802      new BinaryComparator(emptyVal), put);
1803    assertTrue(res);
1804
1805    // not empty anymore
1806    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
1807      new BinaryComparator(emptyVal), put);
1808    assertFalse(res);
1809
1810    Delete delete = new Delete(row1);
1811    delete.addColumn(fam1, qf1);
1812    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
1813      new BinaryComparator(emptyVal), delete);
1814    assertFalse(res);
1815
1816    put = new Put(row1);
1817    put.addColumn(fam1, qf1, val2);
1818    // checkAndPut with correct value
1819    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val1),
1820      put);
1821    assertTrue(res);
1822
1823    // checkAndDelete with correct value
1824    delete = new Delete(row1);
1825    delete.addColumn(fam1, qf1);
1826    delete.addColumn(fam1, qf1);
1827    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val2),
1828      delete);
1829    assertTrue(res);
1830
1831    delete = new Delete(row1);
1832    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
1833      new BinaryComparator(emptyVal), delete);
1834    assertTrue(res);
1835
1836    // checkAndPut looking for a null value
1837    put = new Put(row1);
1838    put.addColumn(fam1, qf1, val1);
1839
1840    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new NullComparator(), put);
1841    assertTrue(res);
1842  }
1843
1844  @Test
1845  @Deprecated
1846  public void testCheckAndMutate_WithWrongValue() throws IOException {
1847    byte[] row1 = Bytes.toBytes("row1");
1848    byte[] fam1 = Bytes.toBytes("fam1");
1849    byte[] qf1 = Bytes.toBytes("qualifier");
1850    byte[] val1 = Bytes.toBytes("value1");
1851    byte[] val2 = Bytes.toBytes("value2");
1852    BigDecimal bd1 = new BigDecimal(Double.MAX_VALUE);
1853    BigDecimal bd2 = new BigDecimal(Double.MIN_VALUE);
1854
1855    // Setting up region
1856    this.region = initHRegion(tableName, method, CONF, fam1);
1857    // Putting data in key
1858    Put put = new Put(row1);
1859    put.addColumn(fam1, qf1, val1);
1860    region.put(put);
1861
1862    // checkAndPut with wrong value
1863    boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
1864      new BinaryComparator(val2), put);
1865    assertEquals(false, res);
1866
1867    // checkAndDelete with wrong value
1868    Delete delete = new Delete(row1);
1869    delete.addFamily(fam1);
1870    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val2),
1871      put);
1872    assertEquals(false, res);
1873
1874    // Putting data in key
1875    put = new Put(row1);
1876    put.addColumn(fam1, qf1, Bytes.toBytes(bd1));
1877    region.put(put);
1878
1879    // checkAndPut with wrong value
1880    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
1881      new BigDecimalComparator(bd2), put);
1882    assertEquals(false, res);
1883
1884    // checkAndDelete with wrong value
1885    delete = new Delete(row1);
1886    delete.addFamily(fam1);
1887    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
1888      new BigDecimalComparator(bd2), put);
1889    assertEquals(false, res);
1890  }
1891
1892  @Test
1893  @Deprecated
1894  public void testCheckAndMutate_WithCorrectValue() throws IOException {
1895    byte[] row1 = Bytes.toBytes("row1");
1896    byte[] fam1 = Bytes.toBytes("fam1");
1897    byte[] qf1 = Bytes.toBytes("qualifier");
1898    byte[] val1 = Bytes.toBytes("value1");
1899    BigDecimal bd1 = new BigDecimal(Double.MIN_VALUE);
1900
1901    // Setting up region
1902    this.region = initHRegion(tableName, method, CONF, fam1);
1903    // Putting data in key
1904    long now = EnvironmentEdgeManager.currentTime();
1905    Put put = new Put(row1);
1906    put.addColumn(fam1, qf1, now, val1);
1907    region.put(put);
1908
1909    // checkAndPut with correct value
1910    boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
1911      new BinaryComparator(val1), put);
1912    assertEquals("First", true, res);
1913
1914    // checkAndDelete with correct value
1915    Delete delete = new Delete(row1, now + 1);
1916    delete.addColumn(fam1, qf1);
1917    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val1),
1918      delete);
1919    assertEquals("Delete", true, res);
1920
1921    // Putting data in key
1922    put = new Put(row1);
1923    put.addColumn(fam1, qf1, now + 2, Bytes.toBytes(bd1));
1924    region.put(put);
1925
1926    // checkAndPut with correct value
1927    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
1928      new BigDecimalComparator(bd1), put);
1929    assertEquals("Second put", true, res);
1930
1931    // checkAndDelete with correct value
1932    delete = new Delete(row1, now + 3);
1933    delete.addColumn(fam1, qf1);
1934    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
1935      new BigDecimalComparator(bd1), delete);
1936    assertEquals("Second delete", true, res);
1937  }
1938
1939  @Test
1940  @Deprecated
1941  public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException {
1942    byte[] row1 = Bytes.toBytes("row1");
1943    byte[] fam1 = Bytes.toBytes("fam1");
1944    byte[] qf1 = Bytes.toBytes("qualifier");
1945    byte[] val1 = Bytes.toBytes("value1");
1946    byte[] val2 = Bytes.toBytes("value2");
1947    byte[] val3 = Bytes.toBytes("value3");
1948    byte[] val4 = Bytes.toBytes("value4");
1949
1950    // Setting up region
1951    this.region = initHRegion(tableName, method, CONF, fam1);
1952    // Putting val3 in key
1953    Put put = new Put(row1);
1954    put.addColumn(fam1, qf1, val3);
1955    region.put(put);
1956
1957    // Test CompareOp.LESS: original = val3, compare with val3, fail
1958    boolean res =
1959      region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS, new BinaryComparator(val3), put);
1960    assertEquals(false, res);
1961
1962    // Test CompareOp.LESS: original = val3, compare with val4, fail
1963    res =
1964      region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS, new BinaryComparator(val4), put);
1965    assertEquals(false, res);
1966
1967    // Test CompareOp.LESS: original = val3, compare with val2,
1968    // succeed (now value = val2)
1969    put = new Put(row1);
1970    put.addColumn(fam1, qf1, val2);
1971    res =
1972      region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS, new BinaryComparator(val2), put);
1973    assertEquals(true, res);
1974
1975    // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val3, fail
1976    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS_OR_EQUAL,
1977      new BinaryComparator(val3), put);
1978    assertEquals(false, res);
1979
1980    // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val2,
1981    // succeed (value still = val2)
1982    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS_OR_EQUAL,
1983      new BinaryComparator(val2), put);
1984    assertEquals(true, res);
1985
1986    // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val1,
1987    // succeed (now value = val3)
1988    put = new Put(row1);
1989    put.addColumn(fam1, qf1, val3);
1990    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS_OR_EQUAL,
1991      new BinaryComparator(val1), put);
1992    assertEquals(true, res);
1993
1994    // Test CompareOp.GREATER: original = val3, compare with val3, fail
1995    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER,
1996      new BinaryComparator(val3), put);
1997    assertEquals(false, res);
1998
1999    // Test CompareOp.GREATER: original = val3, compare with val2, fail
2000    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER,
2001      new BinaryComparator(val2), put);
2002    assertEquals(false, res);
2003
2004    // Test CompareOp.GREATER: original = val3, compare with val4,
2005    // succeed (now value = val2)
2006    put = new Put(row1);
2007    put.addColumn(fam1, qf1, val2);
2008    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER,
2009      new BinaryComparator(val4), put);
2010    assertEquals(true, res);
2011
2012    // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val1, fail
2013    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER_OR_EQUAL,
2014      new BinaryComparator(val1), put);
2015    assertEquals(false, res);
2016
2017    // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val2,
2018    // succeed (value still = val2)
2019    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER_OR_EQUAL,
2020      new BinaryComparator(val2), put);
2021    assertEquals(true, res);
2022
2023    // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val3, succeed
2024    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER_OR_EQUAL,
2025      new BinaryComparator(val3), put);
2026    assertEquals(true, res);
2027  }
2028
2029  @Test
2030  @Deprecated
2031  public void testCheckAndPut_ThatPutWasWritten() throws IOException {
2032    byte[] row1 = Bytes.toBytes("row1");
2033    byte[] fam1 = Bytes.toBytes("fam1");
2034    byte[] fam2 = Bytes.toBytes("fam2");
2035    byte[] qf1 = Bytes.toBytes("qualifier");
2036    byte[] val1 = Bytes.toBytes("value1");
2037    byte[] val2 = Bytes.toBytes("value2");
2038
2039    byte[][] families = { fam1, fam2 };
2040
2041    // Setting up region
2042    this.region = initHRegion(tableName, method, CONF, families);
2043    // Putting data in the key to check
2044    Put put = new Put(row1);
2045    put.addColumn(fam1, qf1, val1);
2046    region.put(put);
2047
2048    // Creating put to add
2049    long ts = EnvironmentEdgeManager.currentTime();
2050    KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
2051    put = new Put(row1);
2052    put.add(kv);
2053
2054    // checkAndPut with wrong value
2055    boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
2056      new BinaryComparator(val1), put);
2057    assertEquals(true, res);
2058
2059    Get get = new Get(row1);
2060    get.addColumn(fam2, qf1);
2061    Cell[] actual = region.get(get).rawCells();
2062
2063    Cell[] expected = { kv };
2064
2065    assertEquals(expected.length, actual.length);
2066    for (int i = 0; i < actual.length; i++) {
2067      assertEquals(expected[i], actual[i]);
2068    }
2069  }
2070
2071  @Test
2072  @Deprecated
2073  public void testCheckAndPut_wrongRowInPut() throws IOException {
2074    this.region = initHRegion(tableName, method, CONF, COLUMNS);
2075    Put put = new Put(row2);
2076    put.addColumn(fam1, qual1, value1);
2077    try {
2078      region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(value2),
2079        put);
2080      fail();
2081    } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
2082      // expected exception.
2083    }
2084  }
2085
2086  @Test
2087  @Deprecated
2088  public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
2089    byte[] row1 = Bytes.toBytes("row1");
2090    byte[] fam1 = Bytes.toBytes("fam1");
2091    byte[] fam2 = Bytes.toBytes("fam2");
2092    byte[] qf1 = Bytes.toBytes("qualifier1");
2093    byte[] qf2 = Bytes.toBytes("qualifier2");
2094    byte[] qf3 = Bytes.toBytes("qualifier3");
2095    byte[] val1 = Bytes.toBytes("value1");
2096    byte[] val2 = Bytes.toBytes("value2");
2097    byte[] val3 = Bytes.toBytes("value3");
2098    byte[] emptyVal = new byte[] {};
2099
2100    byte[][] families = { fam1, fam2 };
2101
2102    // Setting up region
2103    this.region = initHRegion(tableName, method, CONF, families);
2104    // Put content
2105    Put put = new Put(row1);
2106    put.addColumn(fam1, qf1, val1);
2107    region.put(put);
2108    Threads.sleep(2);
2109
2110    put = new Put(row1);
2111    put.addColumn(fam1, qf1, val2);
2112    put.addColumn(fam2, qf1, val3);
2113    put.addColumn(fam2, qf2, val2);
2114    put.addColumn(fam2, qf3, val1);
2115    put.addColumn(fam1, qf3, val1);
2116    region.put(put);
2117
2118    LOG.info("get={}", region.get(new Get(row1).addColumn(fam1, qf1)).toString());
2119
2120    // Multi-column delete
2121    Delete delete = new Delete(row1);
2122    delete.addColumn(fam1, qf1);
2123    delete.addColumn(fam2, qf1);
2124    delete.addColumn(fam1, qf3);
2125    boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
2126      new BinaryComparator(val2), delete);
2127    assertEquals(true, res);
2128
2129    Get get = new Get(row1);
2130    get.addColumn(fam1, qf1);
2131    get.addColumn(fam1, qf3);
2132    get.addColumn(fam2, qf2);
2133    Result r = region.get(get);
2134    assertEquals(2, r.size());
2135    assertArrayEquals(val1, r.getValue(fam1, qf1));
2136    assertArrayEquals(val2, r.getValue(fam2, qf2));
2137
2138    // Family delete
2139    delete = new Delete(row1);
2140    delete.addFamily(fam2);
2141    res = region.checkAndMutate(row1, fam2, qf1, CompareOperator.EQUAL,
2142      new BinaryComparator(emptyVal), delete);
2143    assertEquals(true, res);
2144
2145    get = new Get(row1);
2146    r = region.get(get);
2147    assertEquals(1, r.size());
2148    assertArrayEquals(val1, r.getValue(fam1, qf1));
2149
2150    // Row delete
2151    delete = new Delete(row1);
2152    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val1),
2153      delete);
2154    assertEquals(true, res);
2155    get = new Get(row1);
2156    r = region.get(get);
2157    assertEquals(0, r.size());
2158  }
2159
2160  @Test
2161  @Deprecated
2162  public void testCheckAndMutate_WithFilters() throws Throwable {
2163    final byte[] FAMILY = Bytes.toBytes("fam");
2164
2165    // Setting up region
2166    this.region = initHRegion(tableName, method, CONF, FAMILY);
2167
2168    // Put one row
2169    Put put = new Put(row);
2170    put.addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a"));
2171    put.addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b"));
2172    put.addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c"));
2173    region.put(put);
2174
2175    // Put with success
2176    boolean ok = region.checkAndMutate(row,
2177      new FilterList(
2178        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2179          Bytes.toBytes("a")),
2180        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL,
2181          Bytes.toBytes("b"))),
2182      new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d")));
2183    assertTrue(ok);
2184
2185    Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D")));
2186    assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D"))));
2187
2188    // Put with failure
2189    ok = region.checkAndMutate(row,
2190      new FilterList(
2191        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2192          Bytes.toBytes("a")),
2193        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL,
2194          Bytes.toBytes("c"))),
2195      new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e")));
2196    assertFalse(ok);
2197
2198    assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E"))).isEmpty());
2199
2200    // Delete with success
2201    ok = region.checkAndMutate(row,
2202      new FilterList(
2203        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2204          Bytes.toBytes("a")),
2205        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL,
2206          Bytes.toBytes("b"))),
2207      new Delete(row).addColumns(FAMILY, Bytes.toBytes("D")));
2208    assertTrue(ok);
2209
2210    assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))).isEmpty());
2211
2212    // Mutate with success
2213    ok = region.checkAndRowMutate(row,
2214      new FilterList(
2215        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2216          Bytes.toBytes("a")),
2217        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL,
2218          Bytes.toBytes("b"))),
2219      new RowMutations(row)
2220        .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e")))
2221        .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A"))));
2222    assertTrue(ok);
2223
2224    result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E")));
2225    assertEquals("e", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("E"))));
2226
2227    assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty());
2228  }
2229
2230  @Test
2231  @Deprecated
2232  public void testCheckAndMutate_WithFiltersAndTimeRange() throws Throwable {
2233    final byte[] FAMILY = Bytes.toBytes("fam");
2234
2235    // Setting up region
2236    this.region = initHRegion(tableName, method, CONF, FAMILY);
2237
2238    // Put with specifying the timestamp
2239    region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), 100, Bytes.toBytes("a")));
2240
2241    // Put with success
2242    boolean ok = region.checkAndMutate(row,
2243      new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2244        Bytes.toBytes("a")),
2245      TimeRange.between(0, 101),
2246      new Put(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")));
2247    assertTrue(ok);
2248
2249    Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B")));
2250    assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B"))));
2251
2252    // Put with failure
2253    ok = region.checkAndMutate(row,
2254      new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2255        Bytes.toBytes("a")),
2256      TimeRange.between(0, 100),
2257      new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")));
2258    assertFalse(ok);
2259
2260    assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("C"))).isEmpty());
2261
2262    // Mutate with success
2263    ok = region.checkAndRowMutate(row,
2264      new SingleColumnValueFilter(
2265        FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, Bytes.toBytes("a")),
2266      TimeRange.between(0, 101),
2267      new RowMutations(row)
2268        .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d")))
2269        .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A"))));
2270    assertTrue(ok);
2271
2272    result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D")));
2273    assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D"))));
2274
2275    assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty());
2276  }
2277
2278  @Test
2279  @Deprecated
2280  public void testCheckAndMutate_wrongMutationType() throws Throwable {
2281    // Setting up region
2282    this.region = initHRegion(tableName, method, CONF, fam1);
2283
2284    try {
2285      region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(value1),
2286        new Increment(row).addColumn(fam1, qual1, 1));
2287      fail("should throw DoNotRetryIOException");
2288    } catch (DoNotRetryIOException e) {
2289      assertEquals("Unsupported mutate type: INCREMENT", e.getMessage());
2290    }
2291
2292    try {
2293      region.checkAndMutate(row,
2294        new SingleColumnValueFilter(fam1, qual1, CompareOperator.EQUAL, value1),
2295        new Increment(row).addColumn(fam1, qual1, 1));
2296      fail("should throw DoNotRetryIOException");
2297    } catch (DoNotRetryIOException e) {
2298      assertEquals("Unsupported mutate type: INCREMENT", e.getMessage());
2299    }
2300  }
2301
2302  @Test
2303  @Deprecated
2304  public void testCheckAndMutate_wrongRow() throws Throwable {
2305    final byte[] wrongRow = Bytes.toBytes("wrongRow");
2306
2307    // Setting up region
2308    this.region = initHRegion(tableName, method, CONF, fam1);
2309
2310    try {
2311      region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(value1),
2312        new Put(wrongRow).addColumn(fam1, qual1, value1));
2313      fail("should throw DoNotRetryIOException");
2314    } catch (DoNotRetryIOException e) {
2315      assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>",
2316        e.getMessage());
2317    }
2318
2319    try {
2320      region.checkAndMutate(row,
2321        new SingleColumnValueFilter(fam1, qual1, CompareOperator.EQUAL, value1),
2322        new Put(wrongRow).addColumn(fam1, qual1, value1));
2323      fail("should throw DoNotRetryIOException");
2324    } catch (DoNotRetryIOException e) {
2325      assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>",
2326        e.getMessage());
2327    }
2328
2329    try {
2330      region.checkAndRowMutate(row, fam1, qual1, CompareOperator.EQUAL,
2331        new BinaryComparator(value1),
2332        new RowMutations(wrongRow).add((Mutation) new Put(wrongRow).addColumn(fam1, qual1, value1))
2333          .add((Mutation) new Delete(wrongRow).addColumns(fam1, qual2)));
2334      fail("should throw DoNotRetryIOException");
2335    } catch (DoNotRetryIOException e) {
2336      assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>",
2337        e.getMessage());
2338    }
2339
2340    try {
2341      region.checkAndRowMutate(row,
2342        new SingleColumnValueFilter(fam1, qual1, CompareOperator.EQUAL, value1),
2343        new RowMutations(wrongRow).add((Mutation) new Put(wrongRow).addColumn(fam1, qual1, value1))
2344          .add((Mutation) new Delete(wrongRow).addColumns(fam1, qual2)));
2345      fail("should throw DoNotRetryIOException");
2346    } catch (DoNotRetryIOException e) {
2347      assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>",
2348        e.getMessage());
2349    }
2350  }
2351
2352  @Test
2353  public void testCheckAndMutateWithEmptyRowValue() throws IOException {
2354    byte[] row1 = Bytes.toBytes("row1");
2355    byte[] fam1 = Bytes.toBytes("fam1");
2356    byte[] qf1 = Bytes.toBytes("qualifier");
2357    byte[] emptyVal = new byte[] {};
2358    byte[] val1 = Bytes.toBytes("value1");
2359    byte[] val2 = Bytes.toBytes("value2");
2360
2361    // Setting up region
2362    this.region = initHRegion(tableName, method, CONF, fam1);
2363    // Putting empty data in key
2364    Put put = new Put(row1);
2365    put.addColumn(fam1, qf1, emptyVal);
2366
2367    // checkAndPut with empty value
2368    CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2369      .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(put));
2370    assertTrue(res.isSuccess());
2371    assertNull(res.getResult());
2372
2373    // Putting data in key
2374    put = new Put(row1);
2375    put.addColumn(fam1, qf1, val1);
2376
2377    // checkAndPut with correct value
2378    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2379      .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(put));
2380    assertTrue(res.isSuccess());
2381    assertNull(res.getResult());
2382
2383    // not empty anymore
2384    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2385      .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(put));
2386    assertFalse(res.isSuccess());
2387    assertNull(res.getResult());
2388
2389    Delete delete = new Delete(row1);
2390    delete.addColumn(fam1, qf1);
2391    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2392      .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(delete));
2393    assertFalse(res.isSuccess());
2394    assertNull(res.getResult());
2395
2396    put = new Put(row1);
2397    put.addColumn(fam1, qf1, val2);
2398    // checkAndPut with correct value
2399    res = region.checkAndMutate(
2400      CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(put));
2401    assertTrue(res.isSuccess());
2402    assertNull(res.getResult());
2403
2404    // checkAndDelete with correct value
2405    delete = new Delete(row1);
2406    delete.addColumn(fam1, qf1);
2407    delete.addColumn(fam1, qf1);
2408    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2409      .ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(delete));
2410    assertTrue(res.isSuccess());
2411    assertNull(res.getResult());
2412
2413    delete = new Delete(row1);
2414    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2415      .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(delete));
2416    assertTrue(res.isSuccess());
2417    assertNull(res.getResult());
2418
2419    // checkAndPut looking for a null value
2420    put = new Put(row1);
2421    put.addColumn(fam1, qf1, val1);
2422
2423    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1).ifNotExists(fam1, qf1).build(put));
2424    assertTrue(res.isSuccess());
2425    assertNull(res.getResult());
2426  }
2427
2428  @Test
2429  public void testCheckAndMutateWithWrongValue() throws IOException {
2430    byte[] row1 = Bytes.toBytes("row1");
2431    byte[] fam1 = Bytes.toBytes("fam1");
2432    byte[] qf1 = Bytes.toBytes("qualifier");
2433    byte[] val1 = Bytes.toBytes("value1");
2434    byte[] val2 = Bytes.toBytes("value2");
2435    BigDecimal bd1 = new BigDecimal(Double.MAX_VALUE);
2436    BigDecimal bd2 = new BigDecimal(Double.MIN_VALUE);
2437
2438    // Setting up region
2439    this.region = initHRegion(tableName, method, CONF, fam1);
2440    // Putting data in key
2441    Put put = new Put(row1);
2442    put.addColumn(fam1, qf1, val1);
2443    region.put(put);
2444
2445    // checkAndPut with wrong value
2446    CheckAndMutateResult res = region.checkAndMutate(
2447      CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(put));
2448    assertFalse(res.isSuccess());
2449    assertNull(res.getResult());
2450
2451    // checkAndDelete with wrong value
2452    Delete delete = new Delete(row1);
2453    delete.addFamily(fam1);
2454    res = region.checkAndMutate(
2455      CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(put));
2456    assertFalse(res.isSuccess());
2457    assertNull(res.getResult());
2458
2459    // Putting data in key
2460    put = new Put(row1);
2461    put.addColumn(fam1, qf1, Bytes.toBytes(bd1));
2462    region.put(put);
2463
2464    // checkAndPut with wrong value
2465    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2466      .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd2)).build(put));
2467    assertFalse(res.isSuccess());
2468    assertNull(res.getResult());
2469
2470    // checkAndDelete with wrong value
2471    delete = new Delete(row1);
2472    delete.addFamily(fam1);
2473    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2474      .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd2)).build(delete));
2475    assertFalse(res.isSuccess());
2476    assertNull(res.getResult());
2477  }
2478
2479  @Test
2480  public void testCheckAndMutateWithCorrectValue() throws IOException {
2481    byte[] row1 = Bytes.toBytes("row1");
2482    byte[] fam1 = Bytes.toBytes("fam1");
2483    byte[] qf1 = Bytes.toBytes("qualifier");
2484    byte[] val1 = Bytes.toBytes("value1");
2485    BigDecimal bd1 = new BigDecimal(Double.MIN_VALUE);
2486
2487    // Setting up region
2488    this.region = initHRegion(tableName, method, CONF, fam1);
2489    // Putting data in key
2490    long now = EnvironmentEdgeManager.currentTime();
2491    Put put = new Put(row1);
2492    put.addColumn(fam1, qf1, now, val1);
2493    region.put(put);
2494
2495    // checkAndPut with correct value
2496    CheckAndMutateResult res = region.checkAndMutate(
2497      CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(put));
2498    assertTrue("First", res.isSuccess());
2499
2500    // checkAndDelete with correct value
2501    Delete delete = new Delete(row1, now + 1);
2502    delete.addColumn(fam1, qf1);
2503    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2504      .ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(delete));
2505    assertTrue("Delete", res.isSuccess());
2506    assertNull(res.getResult());
2507
2508    // Putting data in key
2509    put = new Put(row1);
2510    put.addColumn(fam1, qf1, now + 2, Bytes.toBytes(bd1));
2511    region.put(put);
2512
2513    // checkAndPut with correct value
2514    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2515      .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd1)).build(put));
2516    assertTrue("Second put", res.isSuccess());
2517    assertNull(res.getResult());
2518
2519    // checkAndDelete with correct value
2520    delete = new Delete(row1, now + 3);
2521    delete.addColumn(fam1, qf1);
2522    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2523      .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd1)).build(delete));
2524    assertTrue("Second delete", res.isSuccess());
2525    assertNull(res.getResult());
2526  }
2527
2528  @Test
2529  public void testCheckAndMutateWithNonEqualCompareOp() throws IOException {
2530    byte[] row1 = Bytes.toBytes("row1");
2531    byte[] fam1 = Bytes.toBytes("fam1");
2532    byte[] qf1 = Bytes.toBytes("qualifier");
2533    byte[] val1 = Bytes.toBytes("value1");
2534    byte[] val2 = Bytes.toBytes("value2");
2535    byte[] val3 = Bytes.toBytes("value3");
2536    byte[] val4 = Bytes.toBytes("value4");
2537
2538    // Setting up region
2539    this.region = initHRegion(tableName, method, CONF, fam1);
2540    // Putting val3 in key
2541    Put put = new Put(row1);
2542    put.addColumn(fam1, qf1, val3);
2543    region.put(put);
2544
2545    // Test CompareOp.LESS: original = val3, compare with val3, fail
2546    CheckAndMutateResult res = region.checkAndMutate(
2547      CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.LESS, val3).build(put));
2548    assertFalse(res.isSuccess());
2549    assertNull(res.getResult());
2550
2551    // Test CompareOp.LESS: original = val3, compare with val4, fail
2552    res = region.checkAndMutate(
2553      CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.LESS, val4).build(put));
2554    assertFalse(res.isSuccess());
2555    assertNull(res.getResult());
2556
2557    // Test CompareOp.LESS: original = val3, compare with val2,
2558    // succeed (now value = val2)
2559    put = new Put(row1);
2560    put.addColumn(fam1, qf1, val2);
2561    res = region.checkAndMutate(
2562      CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.LESS, val2).build(put));
2563    assertTrue(res.isSuccess());
2564    assertNull(res.getResult());
2565
2566    // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val3, fail
2567    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2568      .ifMatches(fam1, qf1, CompareOperator.LESS_OR_EQUAL, val3).build(put));
2569    assertFalse(res.isSuccess());
2570    assertNull(res.getResult());
2571
2572    // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val2,
2573    // succeed (value still = val2)
2574    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2575      .ifMatches(fam1, qf1, CompareOperator.LESS_OR_EQUAL, val2).build(put));
2576    assertTrue(res.isSuccess());
2577    assertNull(res.getResult());
2578
2579    // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val1,
2580    // succeed (now value = val3)
2581    put = new Put(row1);
2582    put.addColumn(fam1, qf1, val3);
2583    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2584      .ifMatches(fam1, qf1, CompareOperator.LESS_OR_EQUAL, val1).build(put));
2585    assertTrue(res.isSuccess());
2586    assertNull(res.getResult());
2587
2588    // Test CompareOp.GREATER: original = val3, compare with val3, fail
2589    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2590      .ifMatches(fam1, qf1, CompareOperator.GREATER, val3).build(put));
2591    assertFalse(res.isSuccess());
2592    assertNull(res.getResult());
2593
2594    // Test CompareOp.GREATER: original = val3, compare with val2, fail
2595    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2596      .ifMatches(fam1, qf1, CompareOperator.GREATER, val2).build(put));
2597    assertFalse(res.isSuccess());
2598    assertNull(res.getResult());
2599
2600    // Test CompareOp.GREATER: original = val3, compare with val4,
2601    // succeed (now value = val2)
2602    put = new Put(row1);
2603    put.addColumn(fam1, qf1, val2);
2604    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2605      .ifMatches(fam1, qf1, CompareOperator.GREATER, val4).build(put));
2606    assertTrue(res.isSuccess());
2607    assertNull(res.getResult());
2608
2609    // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val1, fail
2610    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2611      .ifMatches(fam1, qf1, CompareOperator.GREATER_OR_EQUAL, val1).build(put));
2612    assertFalse(res.isSuccess());
2613    assertNull(res.getResult());
2614
2615    // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val2,
2616    // succeed (value still = val2)
2617    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2618      .ifMatches(fam1, qf1, CompareOperator.GREATER_OR_EQUAL, val2).build(put));
2619    assertTrue(res.isSuccess());
2620    assertNull(res.getResult());
2621
2622    // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val3, succeed
2623    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2624      .ifMatches(fam1, qf1, CompareOperator.GREATER_OR_EQUAL, val3).build(put));
2625    assertTrue(res.isSuccess());
2626    assertNull(res.getResult());
2627  }
2628
2629  @Test
2630  public void testCheckAndPutThatPutWasWritten() throws IOException {
2631    byte[] row1 = Bytes.toBytes("row1");
2632    byte[] fam1 = Bytes.toBytes("fam1");
2633    byte[] fam2 = Bytes.toBytes("fam2");
2634    byte[] qf1 = Bytes.toBytes("qualifier");
2635    byte[] val1 = Bytes.toBytes("value1");
2636    byte[] val2 = Bytes.toBytes("value2");
2637
2638    byte[][] families = { fam1, fam2 };
2639
2640    // Setting up region
2641    this.region = initHRegion(tableName, method, CONF, families);
2642    // Putting data in the key to check
2643    Put put = new Put(row1);
2644    put.addColumn(fam1, qf1, val1);
2645    region.put(put);
2646
2647    // Creating put to add
2648    long ts = EnvironmentEdgeManager.currentTime();
2649    KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
2650    put = new Put(row1);
2651    put.add(kv);
2652
2653    // checkAndPut with wrong value
2654    CheckAndMutateResult res = region.checkAndMutate(
2655      CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(put));
2656    assertTrue(res.isSuccess());
2657    assertNull(res.getResult());
2658
2659    Get get = new Get(row1);
2660    get.addColumn(fam2, qf1);
2661    Cell[] actual = region.get(get).rawCells();
2662
2663    Cell[] expected = { kv };
2664
2665    assertEquals(expected.length, actual.length);
2666    for (int i = 0; i < actual.length; i++) {
2667      assertEquals(expected[i], actual[i]);
2668    }
2669  }
2670
2671  @Test
2672  public void testCheckAndDeleteThatDeleteWasWritten() throws IOException {
2673    byte[] row1 = Bytes.toBytes("row1");
2674    byte[] fam1 = Bytes.toBytes("fam1");
2675    byte[] fam2 = Bytes.toBytes("fam2");
2676    byte[] qf1 = Bytes.toBytes("qualifier1");
2677    byte[] qf2 = Bytes.toBytes("qualifier2");
2678    byte[] qf3 = Bytes.toBytes("qualifier3");
2679    byte[] val1 = Bytes.toBytes("value1");
2680    byte[] val2 = Bytes.toBytes("value2");
2681    byte[] val3 = Bytes.toBytes("value3");
2682    byte[] emptyVal = new byte[] {};
2683
2684    byte[][] families = { fam1, fam2 };
2685
2686    // Setting up region
2687    this.region = initHRegion(tableName, method, CONF, families);
2688    // Put content
2689    Put put = new Put(row1);
2690    put.addColumn(fam1, qf1, val1);
2691    region.put(put);
2692    Threads.sleep(2);
2693
2694    put = new Put(row1);
2695    put.addColumn(fam1, qf1, val2);
2696    put.addColumn(fam2, qf1, val3);
2697    put.addColumn(fam2, qf2, val2);
2698    put.addColumn(fam2, qf3, val1);
2699    put.addColumn(fam1, qf3, val1);
2700    region.put(put);
2701
2702    LOG.info("get={}", region.get(new Get(row1).addColumn(fam1, qf1)).toString());
2703
2704    // Multi-column delete
2705    Delete delete = new Delete(row1);
2706    delete.addColumn(fam1, qf1);
2707    delete.addColumn(fam2, qf1);
2708    delete.addColumn(fam1, qf3);
2709    CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2710      .ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(delete));
2711    assertTrue(res.isSuccess());
2712    assertNull(res.getResult());
2713
2714    Get get = new Get(row1);
2715    get.addColumn(fam1, qf1);
2716    get.addColumn(fam1, qf3);
2717    get.addColumn(fam2, qf2);
2718    Result r = region.get(get);
2719    assertEquals(2, r.size());
2720    assertArrayEquals(val1, r.getValue(fam1, qf1));
2721    assertArrayEquals(val2, r.getValue(fam2, qf2));
2722
2723    // Family delete
2724    delete = new Delete(row1);
2725    delete.addFamily(fam2);
2726    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2727      .ifMatches(fam2, qf1, CompareOperator.EQUAL, emptyVal).build(delete));
2728    assertTrue(res.isSuccess());
2729    assertNull(res.getResult());
2730
2731    get = new Get(row1);
2732    r = region.get(get);
2733    assertEquals(1, r.size());
2734    assertArrayEquals(val1, r.getValue(fam1, qf1));
2735
2736    // Row delete
2737    delete = new Delete(row1);
2738    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2739      .ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(delete));
2740    assertTrue(res.isSuccess());
2741    assertNull(res.getResult());
2742
2743    get = new Get(row1);
2744    r = region.get(get);
2745    assertEquals(0, r.size());
2746  }
2747
2748  @Test
2749  public void testCheckAndMutateWithFilters() throws Throwable {
2750    final byte[] FAMILY = Bytes.toBytes("fam");
2751
2752    // Setting up region
2753    this.region = initHRegion(tableName, method, CONF, FAMILY);
2754
2755    // Put one row
2756    Put put = new Put(row);
2757    put.addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a"));
2758    put.addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b"));
2759    put.addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c"));
2760    region.put(put);
2761
2762    // Put with success
2763    CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row)
2764      .ifMatches(new FilterList(
2765        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2766          Bytes.toBytes("a")),
2767        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL,
2768          Bytes.toBytes("b"))))
2769      .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d"))));
2770    assertTrue(res.isSuccess());
2771    assertNull(res.getResult());
2772
2773    Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D")));
2774    assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D"))));
2775
2776    // Put with failure
2777    res = region.checkAndMutate(CheckAndMutate.newBuilder(row)
2778      .ifMatches(new FilterList(
2779        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2780          Bytes.toBytes("a")),
2781        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL,
2782          Bytes.toBytes("c"))))
2783      .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e"))));
2784    assertFalse(res.isSuccess());
2785    assertNull(res.getResult());
2786
2787    assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E"))).isEmpty());
2788
2789    // Delete with success
2790    res = region.checkAndMutate(CheckAndMutate.newBuilder(row)
2791      .ifMatches(new FilterList(
2792        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2793          Bytes.toBytes("a")),
2794        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL,
2795          Bytes.toBytes("b"))))
2796      .build(new Delete(row).addColumns(FAMILY, Bytes.toBytes("D"))));
2797    assertTrue(res.isSuccess());
2798    assertNull(res.getResult());
2799
2800    assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))).isEmpty());
2801
2802    // Mutate with success
2803    res = region.checkAndMutate(CheckAndMutate.newBuilder(row)
2804      .ifMatches(new FilterList(
2805        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2806          Bytes.toBytes("a")),
2807        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL,
2808          Bytes.toBytes("b"))))
2809      .build(new RowMutations(row)
2810        .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e")))
2811        .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A")))));
2812    assertTrue(res.isSuccess());
2813    assertNull(res.getResult());
2814
2815    result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E")));
2816    assertEquals("e", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("E"))));
2817
2818    assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty());
2819  }
2820
2821  @Test
2822  public void testCheckAndMutateWithFiltersAndTimeRange() throws Throwable {
2823    final byte[] FAMILY = Bytes.toBytes("fam");
2824
2825    // Setting up region
2826    this.region = initHRegion(tableName, method, CONF, FAMILY);
2827
2828    // Put with specifying the timestamp
2829    region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), 100, Bytes.toBytes("a")));
2830
2831    // Put with success
2832    CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row)
2833      .ifMatches(new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2834        Bytes.toBytes("a")))
2835      .timeRange(TimeRange.between(0, 101))
2836      .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b"))));
2837    assertTrue(res.isSuccess());
2838    assertNull(res.getResult());
2839
2840    Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B")));
2841    assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B"))));
2842
2843    // Put with failure
2844    res = region.checkAndMutate(CheckAndMutate.newBuilder(row)
2845      .ifMatches(new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2846        Bytes.toBytes("a")))
2847      .timeRange(TimeRange.between(0, 100))
2848      .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c"))));
2849    assertFalse(res.isSuccess());
2850    assertNull(res.getResult());
2851
2852    assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("C"))).isEmpty());
2853
2854    // RowMutations with success
2855    res = region.checkAndMutate(CheckAndMutate.newBuilder(row)
2856      .ifMatches(new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2857        Bytes.toBytes("a")))
2858      .timeRange(TimeRange.between(0, 101))
2859      .build(new RowMutations(row)
2860        .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d")))
2861        .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A")))));
2862    assertTrue(res.isSuccess());
2863    assertNull(res.getResult());
2864
2865    result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D")));
2866    assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D"))));
2867
2868    assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty());
2869  }
2870
2871  @Test
2872  public void testCheckAndIncrement() throws Throwable {
2873    final byte[] FAMILY = Bytes.toBytes("fam");
2874
2875    // Setting up region
2876    this.region = initHRegion(tableName, method, CONF, FAMILY);
2877
2878    region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")));
2879
2880    // CheckAndIncrement with correct value
2881    CheckAndMutateResult res = region.checkAndMutate(
2882      CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a"))
2883        .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 1)));
2884    assertTrue(res.isSuccess());
2885    assertEquals(1, Bytes.toLong(res.getResult().getValue(FAMILY, Bytes.toBytes("B"))));
2886
2887    Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B")));
2888    assertEquals(1, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B"))));
2889
2890    // CheckAndIncrement with wrong value
2891    res = region.checkAndMutate(
2892      CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("b"))
2893        .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 1)));
2894    assertFalse(res.isSuccess());
2895    assertNull(res.getResult());
2896
2897    result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B")));
2898    assertEquals(1, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B"))));
2899
2900    region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")));
2901
2902    // CheckAndIncrement with a filter and correct value
2903    res = region.checkAndMutate(CheckAndMutate.newBuilder(row)
2904      .ifMatches(new FilterList(
2905        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2906          Bytes.toBytes("a")),
2907        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL,
2908          Bytes.toBytes("c"))))
2909      .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 2)));
2910    assertTrue(res.isSuccess());
2911    assertEquals(3, Bytes.toLong(res.getResult().getValue(FAMILY, Bytes.toBytes("B"))));
2912
2913    result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B")));
2914    assertEquals(3, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B"))));
2915
2916    // CheckAndIncrement with a filter and correct value
2917    res = region.checkAndMutate(CheckAndMutate.newBuilder(row)
2918      .ifMatches(new FilterList(
2919        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2920          Bytes.toBytes("b")),
2921        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL,
2922          Bytes.toBytes("d"))))
2923      .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 2)));
2924    assertFalse(res.isSuccess());
2925    assertNull(res.getResult());
2926
2927    result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B")));
2928    assertEquals(3, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B"))));
2929  }
2930
2931  @Test
2932  public void testCheckAndAppend() throws Throwable {
2933    final byte[] FAMILY = Bytes.toBytes("fam");
2934
2935    // Setting up region
2936    this.region = initHRegion(tableName, method, CONF, FAMILY);
2937
2938    region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")));
2939
2940    // CheckAndAppend with correct value
2941    CheckAndMutateResult res = region.checkAndMutate(
2942      CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a"))
2943        .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b"))));
2944    assertTrue(res.isSuccess());
2945    assertEquals("b", Bytes.toString(res.getResult().getValue(FAMILY, Bytes.toBytes("B"))));
2946
2947    Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B")));
2948    assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B"))));
2949
2950    // CheckAndAppend with wrong value
2951    res = region.checkAndMutate(
2952      CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("b"))
2953        .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b"))));
2954    assertFalse(res.isSuccess());
2955    assertNull(res.getResult());
2956
2957    result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B")));
2958    assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B"))));
2959
2960    region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")));
2961
2962    // CheckAndAppend with a filter and correct value
2963    res = region.checkAndMutate(CheckAndMutate.newBuilder(row)
2964      .ifMatches(new FilterList(
2965        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2966          Bytes.toBytes("a")),
2967        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL,
2968          Bytes.toBytes("c"))))
2969      .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("bb"))));
2970    assertTrue(res.isSuccess());
2971    assertEquals("bbb", Bytes.toString(res.getResult().getValue(FAMILY, Bytes.toBytes("B"))));
2972
2973    result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B")));
2974    assertEquals("bbb", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B"))));
2975
2976    // CheckAndAppend with a filter and wrong value
2977    res = region.checkAndMutate(CheckAndMutate.newBuilder(row)
2978      .ifMatches(new FilterList(
2979        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2980          Bytes.toBytes("b")),
2981        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL,
2982          Bytes.toBytes("d"))))
2983      .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("bb"))));
2984    assertFalse(res.isSuccess());
2985    assertNull(res.getResult());
2986
2987    result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B")));
2988    assertEquals("bbb", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B"))));
2989  }
2990
2991  @Test
2992  public void testCheckAndIncrementAndAppend() throws Throwable {
2993    // Setting up region
2994    this.region = initHRegion(tableName, method, CONF, fam1);
2995
2996    // CheckAndMutate with Increment and Append
2997    CheckAndMutate checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, qual)
2998      .build(new RowMutations(row).add((Mutation) new Increment(row).addColumn(fam1, qual1, 1L))
2999        .add((Mutation) new Append(row).addColumn(fam1, qual2, Bytes.toBytes("a"))));
3000
3001    CheckAndMutateResult result = region.checkAndMutate(checkAndMutate);
3002    assertTrue(result.isSuccess());
3003    assertEquals(1L, Bytes.toLong(result.getResult().getValue(fam1, qual1)));
3004    assertEquals("a", Bytes.toString(result.getResult().getValue(fam1, qual2)));
3005
3006    Result r = region.get(new Get(row));
3007    assertEquals(1L, Bytes.toLong(r.getValue(fam1, qual1)));
3008    assertEquals("a", Bytes.toString(r.getValue(fam1, qual2)));
3009
3010    // Set return results to false
3011    checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, qual)
3012      .build(new RowMutations(row)
3013        .add((Mutation) new Increment(row).addColumn(fam1, qual1, 1L).setReturnResults(false))
3014        .add((Mutation) new Append(row).addColumn(fam1, qual2, Bytes.toBytes("a"))
3015          .setReturnResults(false)));
3016
3017    result = region.checkAndMutate(checkAndMutate);
3018    assertTrue(result.isSuccess());
3019    assertNull(result.getResult().getValue(fam1, qual1));
3020    assertNull(result.getResult().getValue(fam1, qual2));
3021
3022    r = region.get(new Get(row));
3023    assertEquals(2L, Bytes.toLong(r.getValue(fam1, qual1)));
3024    assertEquals("aa", Bytes.toString(r.getValue(fam1, qual2)));
3025
3026    checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, qual)
3027      .build(new RowMutations(row).add((Mutation) new Increment(row).addColumn(fam1, qual1, 1L))
3028        .add((Mutation) new Append(row).addColumn(fam1, qual2, Bytes.toBytes("a"))
3029          .setReturnResults(false)));
3030
3031    result = region.checkAndMutate(checkAndMutate);
3032    assertTrue(result.isSuccess());
3033    assertEquals(3L, Bytes.toLong(result.getResult().getValue(fam1, qual1)));
3034    assertNull(result.getResult().getValue(fam1, qual2));
3035
3036    r = region.get(new Get(row));
3037    assertEquals(3L, Bytes.toLong(r.getValue(fam1, qual1)));
3038    assertEquals("aaa", Bytes.toString(r.getValue(fam1, qual2)));
3039  }
3040
3041  @Test
3042  public void testCheckAndRowMutations() throws Throwable {
3043    final byte[] row = Bytes.toBytes("row");
3044    final byte[] q1 = Bytes.toBytes("q1");
3045    final byte[] q2 = Bytes.toBytes("q2");
3046    final byte[] q3 = Bytes.toBytes("q3");
3047    final byte[] q4 = Bytes.toBytes("q4");
3048    final String v1 = "v1";
3049
3050    region = initHRegion(tableName, method, CONF, fam1);
3051
3052    // Initial values
3053    region
3054      .batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q2, Bytes.toBytes("toBeDeleted")),
3055        new Put(row).addColumn(fam1, q3, Bytes.toBytes(5L)),
3056        new Put(row).addColumn(fam1, q4, Bytes.toBytes("a")), });
3057
3058    // Do CheckAndRowMutations
3059    CheckAndMutate checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, q1).build(
3060      new RowMutations(row).add(Arrays.asList(new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1)),
3061        new Delete(row).addColumns(fam1, q2), new Increment(row).addColumn(fam1, q3, 1),
3062        new Append(row).addColumn(fam1, q4, Bytes.toBytes("b")))));
3063
3064    CheckAndMutateResult result = region.checkAndMutate(checkAndMutate);
3065    assertTrue(result.isSuccess());
3066    assertEquals(6L, Bytes.toLong(result.getResult().getValue(fam1, q3)));
3067    assertEquals("ab", Bytes.toString(result.getResult().getValue(fam1, q4)));
3068
3069    // Verify the value
3070    Result r = region.get(new Get(row));
3071    assertEquals(v1, Bytes.toString(r.getValue(fam1, q1)));
3072    assertNull(r.getValue(fam1, q2));
3073    assertEquals(6L, Bytes.toLong(r.getValue(fam1, q3)));
3074    assertEquals("ab", Bytes.toString(r.getValue(fam1, q4)));
3075
3076    // Do CheckAndRowMutations again
3077    checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, q1)
3078      .build(new RowMutations(row).add(Arrays.asList(new Delete(row).addColumns(fam1, q1),
3079        new Put(row).addColumn(fam1, q2, Bytes.toBytes(v1)),
3080        new Increment(row).addColumn(fam1, q3, 1),
3081        new Append(row).addColumn(fam1, q4, Bytes.toBytes("b")))));
3082
3083    result = region.checkAndMutate(checkAndMutate);
3084    assertFalse(result.isSuccess());
3085    assertNull(result.getResult());
3086
3087    // Verify the value
3088    r = region.get(new Get(row));
3089    assertEquals(v1, Bytes.toString(r.getValue(fam1, q1)));
3090    assertNull(r.getValue(fam1, q2));
3091    assertEquals(6L, Bytes.toLong(r.getValue(fam1, q3)));
3092    assertEquals("ab", Bytes.toString(r.getValue(fam1, q4)));
3093  }
3094
3095  // ////////////////////////////////////////////////////////////////////////////
3096  // Delete tests
3097  // ////////////////////////////////////////////////////////////////////////////
3098  @Test
3099  public void testDelete_multiDeleteColumn() throws IOException {
3100    byte[] row1 = Bytes.toBytes("row1");
3101    byte[] fam1 = Bytes.toBytes("fam1");
3102    byte[] qual = Bytes.toBytes("qualifier");
3103    byte[] value = Bytes.toBytes("value");
3104
3105    Put put = new Put(row1);
3106    put.addColumn(fam1, qual, 1, value);
3107    put.addColumn(fam1, qual, 2, value);
3108
3109    this.region = initHRegion(tableName, method, CONF, fam1);
3110    region.put(put);
3111
3112    // We do support deleting more than 1 'latest' version
3113    Delete delete = new Delete(row1);
3114    delete.addColumn(fam1, qual);
3115    delete.addColumn(fam1, qual);
3116    region.delete(delete);
3117
3118    Get get = new Get(row1);
3119    get.addFamily(fam1);
3120    Result r = region.get(get);
3121    assertEquals(0, r.size());
3122  }
3123
3124  @Test
3125  public void testDelete_CheckFamily() throws IOException {
3126    byte[] row1 = Bytes.toBytes("row1");
3127    byte[] fam1 = Bytes.toBytes("fam1");
3128    byte[] fam2 = Bytes.toBytes("fam2");
3129    byte[] fam3 = Bytes.toBytes("fam3");
3130    byte[] fam4 = Bytes.toBytes("fam4");
3131
3132    // Setting up region
3133    this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3);
3134    List<Cell> kvs = new ArrayList<>();
3135    kvs.add(new KeyValue(row1, fam4, null, null));
3136
3137    byte[] forUnitTestsOnly = Bytes.toBytes("ForUnitTestsOnly");
3138
3139    // testing existing family
3140    NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
3141    deleteMap.put(fam2, kvs);
3142    region.delete(new Delete(forUnitTestsOnly, HConstants.LATEST_TIMESTAMP, deleteMap));
3143
3144    // testing non existing family
3145    NavigableMap<byte[], List<Cell>> deleteMap2 = new TreeMap<>(Bytes.BYTES_COMPARATOR);
3146    deleteMap2.put(fam4, kvs);
3147    assertThrows("Family " + Bytes.toString(fam4) + " does exist",
3148      NoSuchColumnFamilyException.class,
3149      () -> region.delete(new Delete(forUnitTestsOnly, HConstants.LATEST_TIMESTAMP, deleteMap2)));
3150  }
3151
3152  @Test
3153  public void testDelete_mixed() throws IOException, InterruptedException {
3154    byte[] fam = Bytes.toBytes("info");
3155    byte[][] families = { fam };
3156    this.region = initHRegion(tableName, method, CONF, families);
3157    EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
3158
3159    byte[] row = Bytes.toBytes("table_name");
3160    // column names
3161    byte[] serverinfo = Bytes.toBytes("serverinfo");
3162    byte[] splitA = Bytes.toBytes("splitA");
3163    byte[] splitB = Bytes.toBytes("splitB");
3164
3165    // add some data:
3166    Put put = new Put(row);
3167    put.addColumn(fam, splitA, Bytes.toBytes("reference_A"));
3168    region.put(put);
3169
3170    put = new Put(row);
3171    put.addColumn(fam, splitB, Bytes.toBytes("reference_B"));
3172    region.put(put);
3173
3174    put = new Put(row);
3175    put.addColumn(fam, serverinfo, Bytes.toBytes("ip_address"));
3176    region.put(put);
3177
3178    // ok now delete a split:
3179    Delete delete = new Delete(row);
3180    delete.addColumns(fam, splitA);
3181    region.delete(delete);
3182
3183    // assert some things:
3184    Get get = new Get(row).addColumn(fam, serverinfo);
3185    Result result = region.get(get);
3186    assertEquals(1, result.size());
3187
3188    get = new Get(row).addColumn(fam, splitA);
3189    result = region.get(get);
3190    assertEquals(0, result.size());
3191
3192    get = new Get(row).addColumn(fam, splitB);
3193    result = region.get(get);
3194    assertEquals(1, result.size());
3195
3196    // Assert that after a delete, I can put.
3197    put = new Put(row);
3198    put.addColumn(fam, splitA, Bytes.toBytes("reference_A"));
3199    region.put(put);
3200    get = new Get(row);
3201    result = region.get(get);
3202    assertEquals(3, result.size());
3203
3204    // Now delete all... then test I can add stuff back
3205    delete = new Delete(row);
3206    region.delete(delete);
3207    assertEquals(0, region.get(get).size());
3208
3209    region.put(new Put(row).addColumn(fam, splitA, Bytes.toBytes("reference_A")));
3210    result = region.get(get);
3211    assertEquals(1, result.size());
3212  }
3213
3214  @Test
3215  public void testDeleteRowWithFutureTs() throws IOException {
3216    byte[] fam = Bytes.toBytes("info");
3217    byte[][] families = { fam };
3218    this.region = initHRegion(tableName, method, CONF, families);
3219    byte[] row = Bytes.toBytes("table_name");
3220    // column names
3221    byte[] serverinfo = Bytes.toBytes("serverinfo");
3222
3223    // add data in the far future
3224    Put put = new Put(row);
3225    put.addColumn(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
3226    region.put(put);
3227
3228    // now delete something in the present
3229    Delete delete = new Delete(row);
3230    region.delete(delete);
3231
3232    // make sure we still see our data
3233    Get get = new Get(row).addColumn(fam, serverinfo);
3234    Result result = region.get(get);
3235    assertEquals(1, result.size());
3236
3237    // delete the future row
3238    delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
3239    region.delete(delete);
3240
3241    // make sure it is gone
3242    get = new Get(row).addColumn(fam, serverinfo);
3243    result = region.get(get);
3244    assertEquals(0, result.size());
3245  }
3246
3247  /**
3248   * Tests that the special LATEST_TIMESTAMP option for puts gets replaced by the actual timestamp
3249   */
3250  @Test
3251  public void testPutWithLatestTS() throws IOException {
3252    byte[] fam = Bytes.toBytes("info");
3253    byte[][] families = { fam };
3254    this.region = initHRegion(tableName, method, CONF, families);
3255    byte[] row = Bytes.toBytes("row1");
3256    // column names
3257    byte[] qual = Bytes.toBytes("qual");
3258
3259    // add data with LATEST_TIMESTAMP, put without WAL
3260    Put put = new Put(row);
3261    put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
3262    region.put(put);
3263
3264    // Make sure it shows up with an actual timestamp
3265    Get get = new Get(row).addColumn(fam, qual);
3266    Result result = region.get(get);
3267    assertEquals(1, result.size());
3268    Cell kv = result.rawCells()[0];
3269    LOG.info("Got: " + kv);
3270    assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
3271      kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
3272
3273    // Check same with WAL enabled (historically these took different
3274    // code paths, so check both)
3275    row = Bytes.toBytes("row2");
3276    put = new Put(row);
3277    put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
3278    region.put(put);
3279
3280    // Make sure it shows up with an actual timestamp
3281    get = new Get(row).addColumn(fam, qual);
3282    result = region.get(get);
3283    assertEquals(1, result.size());
3284    kv = result.rawCells()[0];
3285    LOG.info("Got: " + kv);
3286    assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
3287      kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
3288  }
3289
3290  /**
3291   * Tests that there is server-side filtering for invalid timestamp upper bound. Note that the
3292   * timestamp lower bound is automatically handled for us by the TTL field.
3293   */
3294  @Test
3295  public void testPutWithTsSlop() throws IOException {
3296    byte[] fam = Bytes.toBytes("info");
3297    byte[][] families = { fam };
3298
3299    // add data with a timestamp that is too recent for range. Ensure assert
3300    CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
3301    this.region = initHRegion(tableName, method, CONF, families);
3302    boolean caughtExcep = false;
3303    try {
3304      // no TS specified == use latest. should not error
3305      region.put(new Put(row).addColumn(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
3306      // TS out of range. should error
3307      region.put(new Put(row).addColumn(fam, Bytes.toBytes("qual"),
3308        EnvironmentEdgeManager.currentTime() + 2000, Bytes.toBytes("value")));
3309      fail("Expected IOE for TS out of configured timerange");
3310    } catch (FailedSanityCheckException ioe) {
3311      LOG.debug("Received expected exception", ioe);
3312      caughtExcep = true;
3313    }
3314    assertTrue("Should catch FailedSanityCheckException", caughtExcep);
3315  }
3316
3317  @Test
3318  public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
3319    byte[] fam1 = Bytes.toBytes("columnA");
3320    byte[] fam2 = Bytes.toBytes("columnB");
3321    this.region = initHRegion(tableName, method, CONF, fam1, fam2);
3322    byte[] rowA = Bytes.toBytes("rowA");
3323    byte[] rowB = Bytes.toBytes("rowB");
3324
3325    byte[] value = Bytes.toBytes("value");
3326
3327    Delete delete = new Delete(rowA);
3328    delete.addFamily(fam1);
3329
3330    region.delete(delete);
3331
3332    // now create data.
3333    Put put = new Put(rowA);
3334    put.addColumn(fam2, null, value);
3335    region.put(put);
3336
3337    put = new Put(rowB);
3338    put.addColumn(fam1, null, value);
3339    put.addColumn(fam2, null, value);
3340    region.put(put);
3341
3342    Scan scan = new Scan();
3343    scan.addFamily(fam1).addFamily(fam2);
3344    try (InternalScanner s = region.getScanner(scan)) {
3345      List<Cell> results = new ArrayList<>();
3346      s.next(results);
3347      assertTrue(CellUtil.matchingRows(results.get(0), rowA));
3348
3349      results.clear();
3350      s.next(results);
3351      assertTrue(CellUtil.matchingRows(results.get(0), rowB));
3352    }
3353  }
3354
3355  @Test
3356  public void testDataInMemoryWithoutWAL() throws IOException {
3357    FileSystem fs = FileSystem.get(CONF);
3358    Path rootDir = new Path(dir + "testDataInMemoryWithoutWAL");
3359    FSHLog hLog = new FSHLog(fs, rootDir, "testDataInMemoryWithoutWAL", CONF);
3360    hLog.init();
3361    // This chunk creation is done throughout the code base. Do we want to move it into core?
3362    // It is missing from this test. W/o it we NPE.
3363    region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, hLog,
3364      COLUMN_FAMILY_BYTES);
3365
3366    Cell originalCell = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row)
3367      .setFamily(COLUMN_FAMILY_BYTES).setQualifier(qual1)
3368      .setTimestamp(EnvironmentEdgeManager.currentTime()).setType(KeyValue.Type.Put.getCode())
3369      .setValue(value1).build();
3370    final long originalSize = originalCell.getSerializedSize();
3371
3372    Cell addCell = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row)
3373      .setFamily(COLUMN_FAMILY_BYTES).setQualifier(qual1)
3374      .setTimestamp(EnvironmentEdgeManager.currentTime()).setType(KeyValue.Type.Put.getCode())
3375      .setValue(Bytes.toBytes("xxxxxxxxxx")).build();
3376    final long addSize = addCell.getSerializedSize();
3377
3378    LOG.info("originalSize:" + originalSize + ", addSize:" + addSize);
3379    // start test. We expect that the addPut's durability will be replaced
3380    // by originalPut's durability.
3381
3382    // case 1:
3383    testDataInMemoryWithoutWAL(region,
3384      new Put(row).add(originalCell).setDurability(Durability.SKIP_WAL),
3385      new Put(row).add(addCell).setDurability(Durability.SKIP_WAL), originalSize + addSize);
3386
3387    // case 2:
3388    testDataInMemoryWithoutWAL(region,
3389      new Put(row).add(originalCell).setDurability(Durability.SKIP_WAL),
3390      new Put(row).add(addCell).setDurability(Durability.SYNC_WAL), originalSize + addSize);
3391
3392    // case 3:
3393    testDataInMemoryWithoutWAL(region,
3394      new Put(row).add(originalCell).setDurability(Durability.SYNC_WAL),
3395      new Put(row).add(addCell).setDurability(Durability.SKIP_WAL), 0);
3396
3397    // case 4:
3398    testDataInMemoryWithoutWAL(region,
3399      new Put(row).add(originalCell).setDurability(Durability.SYNC_WAL),
3400      new Put(row).add(addCell).setDurability(Durability.SYNC_WAL), 0);
3401  }
3402
3403  private static void testDataInMemoryWithoutWAL(HRegion region, Put originalPut, final Put addPut,
3404    long delta) throws IOException {
3405    final long initSize = region.getDataInMemoryWithoutWAL();
3406    // save normalCPHost and replaced by mockedCPHost
3407    RegionCoprocessorHost normalCPHost = region.getCoprocessorHost();
3408    RegionCoprocessorHost mockedCPHost = mock(RegionCoprocessorHost.class);
3409    // Because the preBatchMutate returns void, we can't do usual Mockito when...then form. Must
3410    // do below format (from Mockito doc).
3411    doAnswer(new Answer<Void>() {
3412      @Override
3413      public Void answer(InvocationOnMock invocation) throws Throwable {
3414        MiniBatchOperationInProgress<Mutation> mb = invocation.getArgument(0);
3415        mb.addOperationsFromCP(0, new Mutation[] { addPut });
3416        return null;
3417      }
3418    }).when(mockedCPHost).preBatchMutate(isA(MiniBatchOperationInProgress.class));
3419    ColumnFamilyDescriptorBuilder builder =
3420      ColumnFamilyDescriptorBuilder.newBuilder(COLUMN_FAMILY_BYTES);
3421    ScanInfo info = new ScanInfo(CONF, builder.build(), Long.MAX_VALUE, Long.MAX_VALUE,
3422      region.getCellComparator());
3423    when(mockedCPHost.preFlushScannerOpen(any(HStore.class), any())).thenReturn(info);
3424
3425    when(mockedCPHost.preFlush(any(), any(StoreScanner.class), any()))
3426      .thenAnswer(i -> i.getArgument(1));
3427    region.setCoprocessorHost(mockedCPHost);
3428
3429    region.put(originalPut);
3430    region.setCoprocessorHost(normalCPHost);
3431    final long finalSize = region.getDataInMemoryWithoutWAL();
3432    assertEquals("finalSize:" + finalSize + ", initSize:" + initSize + ", delta:" + delta,
3433      finalSize, initSize + delta);
3434  }
3435
3436  @Test
3437  public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
3438    Delete delete = new Delete(row);
3439    delete.addColumns(fam1, qual1);
3440    doTestDelete_AndPostInsert(delete);
3441  }
3442
3443  @Test
3444  public void testaddFamily_PostInsert() throws IOException, InterruptedException {
3445    Delete delete = new Delete(row);
3446    delete.addFamily(fam1);
3447    doTestDelete_AndPostInsert(delete);
3448  }
3449
3450  public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
3451    this.region = initHRegion(tableName, method, CONF, fam1);
3452    EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
3453    Put put = new Put(row);
3454    put.addColumn(fam1, qual1, value1);
3455    region.put(put);
3456
3457    // now delete the value:
3458    region.delete(delete);
3459
3460    // ok put data:
3461    put = new Put(row);
3462    put.addColumn(fam1, qual1, value2);
3463    region.put(put);
3464
3465    // ok get:
3466    Get get = new Get(row);
3467    get.addColumn(fam1, qual1);
3468
3469    Result r = region.get(get);
3470    assertEquals(1, r.size());
3471    assertArrayEquals(value2, r.getValue(fam1, qual1));
3472
3473    // next:
3474    Scan scan = new Scan().withStartRow(row);
3475    scan.addColumn(fam1, qual1);
3476    try (InternalScanner s = region.getScanner(scan)) {
3477      List<Cell> results = new ArrayList<>();
3478      assertEquals(false, s.next(results));
3479      assertEquals(1, results.size());
3480      Cell kv = results.get(0);
3481
3482      assertArrayEquals(value2, CellUtil.cloneValue(kv));
3483      assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
3484      assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
3485      assertArrayEquals(row, CellUtil.cloneRow(kv));
3486    }
3487  }
3488
3489  @Test
3490  public void testDelete_CheckTimestampUpdated() throws IOException {
3491    byte[] row1 = Bytes.toBytes("row1");
3492    byte[] col1 = Bytes.toBytes("col1");
3493    byte[] col2 = Bytes.toBytes("col2");
3494    byte[] col3 = Bytes.toBytes("col3");
3495
3496    byte[] forUnitTestsOnly = Bytes.toBytes("ForUnitTestsOnly");
3497
3498    // Setting up region
3499    this.region = initHRegion(tableName, method, CONF, fam1);
3500    // Building checkerList
3501    List<Cell> kvs = new ArrayList<>();
3502    kvs.add(new KeyValue(row1, fam1, col1, null));
3503    kvs.add(new KeyValue(row1, fam1, col2, null));
3504    kvs.add(new KeyValue(row1, fam1, col3, null));
3505
3506    NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
3507    deleteMap.put(fam1, kvs);
3508    region.delete(new Delete(forUnitTestsOnly, HConstants.LATEST_TIMESTAMP, deleteMap));
3509
3510    // extract the key values out the memstore:
3511    // This is kinda hacky, but better than nothing...
3512    long now = EnvironmentEdgeManager.currentTime();
3513    AbstractMemStore memstore = (AbstractMemStore) region.getStore(fam1).memstore;
3514    Cell firstCell = memstore.getActive().first();
3515    assertTrue(firstCell.getTimestamp() <= now);
3516    now = firstCell.getTimestamp();
3517    for (Cell cell : memstore.getActive().getCellSet()) {
3518      assertTrue(cell.getTimestamp() <= now);
3519      now = cell.getTimestamp();
3520    }
3521  }
3522
3523  // ////////////////////////////////////////////////////////////////////////////
3524  // Get tests
3525  // ////////////////////////////////////////////////////////////////////////////
3526  @Test
3527  public void testGet_FamilyChecker() throws IOException {
3528    byte[] row1 = Bytes.toBytes("row1");
3529    byte[] fam1 = Bytes.toBytes("fam1");
3530    byte[] fam2 = Bytes.toBytes("False");
3531    byte[] col1 = Bytes.toBytes("col1");
3532
3533    // Setting up region
3534    this.region = initHRegion(tableName, method, CONF, fam1);
3535    Get get = new Get(row1);
3536    get.addColumn(fam2, col1);
3537
3538    // Test
3539    try {
3540      region.get(get);
3541      fail("Expecting DoNotRetryIOException in get but did not get any");
3542    } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
3543      LOG.info("Got expected DoNotRetryIOException successfully");
3544    }
3545  }
3546
3547  @Test
3548  public void testGet_Basic() throws IOException {
3549    byte[] row1 = Bytes.toBytes("row1");
3550    byte[] fam1 = Bytes.toBytes("fam1");
3551    byte[] col1 = Bytes.toBytes("col1");
3552    byte[] col2 = Bytes.toBytes("col2");
3553    byte[] col3 = Bytes.toBytes("col3");
3554    byte[] col4 = Bytes.toBytes("col4");
3555    byte[] col5 = Bytes.toBytes("col5");
3556
3557    // Setting up region
3558    this.region = initHRegion(tableName, method, CONF, fam1);
3559    // Add to memstore
3560    Put put = new Put(row1);
3561    put.addColumn(fam1, col1, null);
3562    put.addColumn(fam1, col2, null);
3563    put.addColumn(fam1, col3, null);
3564    put.addColumn(fam1, col4, null);
3565    put.addColumn(fam1, col5, null);
3566    region.put(put);
3567
3568    Get get = new Get(row1);
3569    get.addColumn(fam1, col2);
3570    get.addColumn(fam1, col4);
3571    // Expected result
3572    KeyValue kv1 = new KeyValue(row1, fam1, col2);
3573    KeyValue kv2 = new KeyValue(row1, fam1, col4);
3574    KeyValue[] expected = { kv1, kv2 };
3575
3576    // Test
3577    Result res = region.get(get);
3578    assertEquals(expected.length, res.size());
3579    for (int i = 0; i < res.size(); i++) {
3580      assertTrue(CellUtil.matchingRows(expected[i], res.rawCells()[i]));
3581      assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
3582      assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
3583    }
3584
3585    // Test using a filter on a Get
3586    Get g = new Get(row1);
3587    final int count = 2;
3588    g.setFilter(new ColumnCountGetFilter(count));
3589    res = region.get(g);
3590    assertEquals(count, res.size());
3591  }
3592
3593  @Test
3594  public void testGet_Empty() throws IOException {
3595    byte[] row = Bytes.toBytes("row");
3596    byte[] fam = Bytes.toBytes("fam");
3597
3598    this.region = initHRegion(tableName, method, CONF, fam);
3599    Get get = new Get(row);
3600    get.addFamily(fam);
3601    Result r = region.get(get);
3602
3603    assertTrue(r.isEmpty());
3604  }
3605
3606  @Test
3607  public void testGetWithFilter() throws IOException, InterruptedException {
3608    byte[] row1 = Bytes.toBytes("row1");
3609    byte[] fam1 = Bytes.toBytes("fam1");
3610    byte[] col1 = Bytes.toBytes("col1");
3611    byte[] value1 = Bytes.toBytes("value1");
3612    byte[] value2 = Bytes.toBytes("value2");
3613
3614    final int maxVersions = 3;
3615    TableDescriptor tableDescriptor =
3616      TableDescriptorBuilder.newBuilder(TableName.valueOf("testFilterAndColumnTracker"))
3617        .setColumnFamily(
3618          ColumnFamilyDescriptorBuilder.newBuilder(fam1).setMaxVersions(maxVersions).build())
3619        .build();
3620    ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null,
3621      MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);
3622    RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
3623    Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + ".log");
3624    final WAL wal = HBaseTestingUtil.createWal(TEST_UTIL.getConfiguration(), logDir, info);
3625    this.region = TEST_UTIL.createLocalHRegion(info, CONF, tableDescriptor, wal);
3626
3627    // Put 4 version to memstore
3628    long ts = 0;
3629    Put put = new Put(row1, ts);
3630    put.addColumn(fam1, col1, value1);
3631    region.put(put);
3632    put = new Put(row1, ts + 1);
3633    put.addColumn(fam1, col1, Bytes.toBytes("filter1"));
3634    region.put(put);
3635    put = new Put(row1, ts + 2);
3636    put.addColumn(fam1, col1, Bytes.toBytes("filter2"));
3637    region.put(put);
3638    put = new Put(row1, ts + 3);
3639    put.addColumn(fam1, col1, value2);
3640    region.put(put);
3641
3642    Get get = new Get(row1);
3643    get.readAllVersions();
3644    Result res = region.get(get);
3645    // Get 3 versions, the oldest version has gone from user view
3646    assertEquals(maxVersions, res.size());
3647
3648    get.setFilter(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("value")));
3649    res = region.get(get);
3650    // When use value filter, the oldest version should still gone from user view and it
3651    // should only return one key vaule
3652    assertEquals(1, res.size());
3653    assertTrue(CellUtil.matchingValue(new KeyValue(row1, fam1, col1, value2), res.rawCells()[0]));
3654    assertEquals(ts + 3, res.rawCells()[0].getTimestamp());
3655
3656    region.flush(true);
3657    region.compact(true);
3658    Thread.sleep(1000);
3659    res = region.get(get);
3660    // After flush and compact, the result should be consistent with previous result
3661    assertEquals(1, res.size());
3662    assertTrue(CellUtil.matchingValue(new KeyValue(row1, fam1, col1, value2), res.rawCells()[0]));
3663  }
3664
3665  // ////////////////////////////////////////////////////////////////////////////
3666  // Scanner tests
3667  // ////////////////////////////////////////////////////////////////////////////
3668  @Test
3669  public void testGetScanner_WithOkFamilies() throws IOException {
3670    byte[] fam1 = Bytes.toBytes("fam1");
3671    byte[] fam2 = Bytes.toBytes("fam2");
3672
3673    byte[][] families = { fam1, fam2 };
3674
3675    // Setting up region
3676    this.region = initHRegion(tableName, method, CONF, families);
3677    Scan scan = new Scan();
3678    scan.addFamily(fam1);
3679    scan.addFamily(fam2);
3680    region.getScanner(scan).close();
3681  }
3682
3683  @Test
3684  public void testGetScanner_WithNotOkFamilies() throws IOException {
3685    byte[] fam1 = Bytes.toBytes("fam1");
3686    byte[] fam2 = Bytes.toBytes("fam2");
3687
3688    byte[][] families = { fam1 };
3689
3690    // Setting up region
3691    this.region = initHRegion(tableName, method, CONF, families);
3692    Scan scan = new Scan();
3693    scan.addFamily(fam2);
3694    assertThrows(NoSuchColumnFamilyException.class, () -> region.getScanner(scan));
3695  }
3696
3697  @Test
3698  public void testGetScanner_WithNoFamilies() throws IOException {
3699    byte[] row1 = Bytes.toBytes("row1");
3700    byte[] fam1 = Bytes.toBytes("fam1");
3701    byte[] fam2 = Bytes.toBytes("fam2");
3702    byte[] fam3 = Bytes.toBytes("fam3");
3703    byte[] fam4 = Bytes.toBytes("fam4");
3704
3705    byte[][] families = { fam1, fam2, fam3, fam4 };
3706
3707    // Setting up region
3708    this.region = initHRegion(tableName, method, CONF, families);
3709    // Putting data in Region
3710    Put put = new Put(row1);
3711    put.addColumn(fam1, null, null);
3712    put.addColumn(fam2, null, null);
3713    put.addColumn(fam3, null, null);
3714    put.addColumn(fam4, null, null);
3715    region.put(put);
3716
3717    Scan scan = null;
3718
3719    // Testing to see how many scanners that is produced by getScanner, starting with known number,
3720    // 2 - current = 1
3721    scan = new Scan();
3722    scan.addFamily(fam2);
3723    scan.addFamily(fam4);
3724    try (RegionScannerImpl is = region.getScanner(scan)) {
3725      assertEquals(1, is.storeHeap.getHeap().size());
3726    }
3727
3728    scan = new Scan();
3729    try (RegionScannerImpl is = region.getScanner(scan)) {
3730      assertEquals(families.length - 1, is.storeHeap.getHeap().size());
3731    }
3732  }
3733
3734  /**
3735   * This method tests https://issues.apache.org/jira/browse/HBASE-2516.
3736   */
3737  @Test
3738  public void testGetScanner_WithRegionClosed() throws IOException {
3739    byte[] fam1 = Bytes.toBytes("fam1");
3740    byte[] fam2 = Bytes.toBytes("fam2");
3741
3742    byte[][] families = { fam1, fam2 };
3743
3744    // Setting up region
3745    region = initHRegion(tableName, method, CONF, families);
3746    region.closed.set(true);
3747    try {
3748      assertThrows(NotServingRegionException.class, () -> region.getScanner(null));
3749    } finally {
3750      // so we can close the region in tearDown
3751      region.closed.set(false);
3752    }
3753  }
3754
3755  @Test
3756  public void testRegionScanner_Next() throws IOException {
3757    byte[] row1 = Bytes.toBytes("row1");
3758    byte[] row2 = Bytes.toBytes("row2");
3759    byte[] fam1 = Bytes.toBytes("fam1");
3760    byte[] fam2 = Bytes.toBytes("fam2");
3761    byte[] fam3 = Bytes.toBytes("fam3");
3762    byte[] fam4 = Bytes.toBytes("fam4");
3763
3764    byte[][] families = { fam1, fam2, fam3, fam4 };
3765    long ts = EnvironmentEdgeManager.currentTime();
3766
3767    // Setting up region
3768    this.region = initHRegion(tableName, method, CONF, families);
3769    // Putting data in Region
3770    Put put = null;
3771    put = new Put(row1);
3772    put.addColumn(fam1, (byte[]) null, ts, null);
3773    put.addColumn(fam2, (byte[]) null, ts, null);
3774    put.addColumn(fam3, (byte[]) null, ts, null);
3775    put.addColumn(fam4, (byte[]) null, ts, null);
3776    region.put(put);
3777
3778    put = new Put(row2);
3779    put.addColumn(fam1, (byte[]) null, ts, null);
3780    put.addColumn(fam2, (byte[]) null, ts, null);
3781    put.addColumn(fam3, (byte[]) null, ts, null);
3782    put.addColumn(fam4, (byte[]) null, ts, null);
3783    region.put(put);
3784
3785    Scan scan = new Scan();
3786    scan.addFamily(fam2);
3787    scan.addFamily(fam4);
3788    try (InternalScanner is = region.getScanner(scan)) {
3789      List<ExtendedCell> res = null;
3790
3791      // Result 1
3792      List<ExtendedCell> expected1 = new ArrayList<>();
3793      expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
3794      expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
3795
3796      res = new ArrayList<>();
3797      is.next(res);
3798      for (int i = 0; i < res.size(); i++) {
3799        assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
3800      }
3801
3802      // Result 2
3803      List<ExtendedCell> expected2 = new ArrayList<>();
3804      expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
3805      expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
3806
3807      res = new ArrayList<>();
3808      is.next(res);
3809      for (int i = 0; i < res.size(); i++) {
3810        assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
3811      }
3812    }
3813  }
3814
3815  @Test
3816  public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
3817    byte[] row1 = Bytes.toBytes("row1");
3818    byte[] qf1 = Bytes.toBytes("qualifier1");
3819    byte[] qf2 = Bytes.toBytes("qualifier2");
3820    byte[] fam1 = Bytes.toBytes("fam1");
3821    byte[][] families = { fam1 };
3822
3823    long ts1 = EnvironmentEdgeManager.currentTime();
3824    long ts2 = ts1 + 1;
3825    long ts3 = ts1 + 2;
3826
3827    // Setting up region
3828    this.region = initHRegion(tableName, method, CONF, families);
3829    // Putting data in Region
3830    Put put = null;
3831    KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
3832    KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
3833    KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
3834
3835    KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
3836    KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
3837    KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
3838
3839    put = new Put(row1);
3840    put.add(kv13);
3841    put.add(kv12);
3842    put.add(kv11);
3843    put.add(kv23);
3844    put.add(kv22);
3845    put.add(kv21);
3846    region.put(put);
3847
3848    // Expected
3849    List<Cell> expected = new ArrayList<>();
3850    expected.add(kv13);
3851    expected.add(kv12);
3852
3853    Scan scan = new Scan().withStartRow(row1);
3854    scan.addColumn(fam1, qf1);
3855    scan.readVersions(MAX_VERSIONS);
3856    List<Cell> actual = new ArrayList<>();
3857    try (InternalScanner scanner = region.getScanner(scan)) {
3858      boolean hasNext = scanner.next(actual);
3859      assertEquals(false, hasNext);
3860
3861      // Verify result
3862      for (int i = 0; i < expected.size(); i++) {
3863        assertEquals(expected.get(i), actual.get(i));
3864      }
3865    }
3866  }
3867
3868  @Test
3869  public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
3870    byte[] row1 = Bytes.toBytes("row1");
3871    byte[] qf1 = Bytes.toBytes("qualifier1");
3872    byte[] qf2 = Bytes.toBytes("qualifier2");
3873    byte[] fam1 = Bytes.toBytes("fam1");
3874    byte[][] families = { fam1 };
3875
3876    long ts1 = 1;
3877    long ts2 = ts1 + 1;
3878    long ts3 = ts1 + 2;
3879
3880    // Setting up region
3881    this.region = initHRegion(tableName, method, CONF, families);
3882    // Putting data in Region
3883    Put put = null;
3884    KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
3885    KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
3886    KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
3887
3888    KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
3889    KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
3890    KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
3891
3892    put = new Put(row1);
3893    put.add(kv13);
3894    put.add(kv12);
3895    put.add(kv11);
3896    put.add(kv23);
3897    put.add(kv22);
3898    put.add(kv21);
3899    region.put(put);
3900    region.flush(true);
3901
3902    // Expected
3903    List<ExtendedCell> expected = new ArrayList<>();
3904    expected.add(kv13);
3905    expected.add(kv12);
3906    expected.add(kv23);
3907    expected.add(kv22);
3908
3909    Scan scan = new Scan().withStartRow(row1);
3910    scan.addColumn(fam1, qf1);
3911    scan.addColumn(fam1, qf2);
3912    scan.readVersions(MAX_VERSIONS);
3913    List<ExtendedCell> actual = new ArrayList<>();
3914    try (InternalScanner scanner = region.getScanner(scan)) {
3915      boolean hasNext = scanner.next(actual);
3916      assertEquals(false, hasNext);
3917
3918      // Verify result
3919      for (int i = 0; i < expected.size(); i++) {
3920        assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
3921      }
3922    }
3923  }
3924
3925  @Test
3926  public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions()
3927    throws IOException {
3928    byte[] row1 = Bytes.toBytes("row1");
3929    byte[] fam1 = Bytes.toBytes("fam1");
3930    byte[][] families = { fam1 };
3931    byte[] qf1 = Bytes.toBytes("qualifier1");
3932    byte[] qf2 = Bytes.toBytes("qualifier2");
3933
3934    long ts1 = 1;
3935    long ts2 = ts1 + 1;
3936    long ts3 = ts1 + 2;
3937    long ts4 = ts1 + 3;
3938
3939    // Setting up region
3940    this.region = initHRegion(tableName, method, CONF, families);
3941    // Putting data in Region
3942    KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
3943    KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
3944    KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
3945    KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
3946
3947    KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
3948    KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
3949    KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
3950    KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
3951
3952    Put put = null;
3953    put = new Put(row1);
3954    put.add(kv14);
3955    put.add(kv24);
3956    region.put(put);
3957    region.flush(true);
3958
3959    put = new Put(row1);
3960    put.add(kv23);
3961    put.add(kv13);
3962    region.put(put);
3963    region.flush(true);
3964
3965    put = new Put(row1);
3966    put.add(kv22);
3967    put.add(kv12);
3968    region.put(put);
3969    region.flush(true);
3970
3971    put = new Put(row1);
3972    put.add(kv21);
3973    put.add(kv11);
3974    region.put(put);
3975
3976    // Expected
3977    List<ExtendedCell> expected = new ArrayList<>();
3978    expected.add(kv14);
3979    expected.add(kv13);
3980    expected.add(kv12);
3981    expected.add(kv24);
3982    expected.add(kv23);
3983    expected.add(kv22);
3984
3985    Scan scan = new Scan().withStartRow(row1);
3986    scan.addColumn(fam1, qf1);
3987    scan.addColumn(fam1, qf2);
3988    int versions = 3;
3989    scan.readVersions(versions);
3990    List<ExtendedCell> actual = new ArrayList<>();
3991    try (InternalScanner scanner = region.getScanner(scan)) {
3992      boolean hasNext = scanner.next(actual);
3993      assertEquals(false, hasNext);
3994
3995      // Verify result
3996      for (int i = 0; i < expected.size(); i++) {
3997        assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
3998      }
3999    }
4000  }
4001
4002  @Test
4003  public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
4004    byte[] row1 = Bytes.toBytes("row1");
4005    byte[] qf1 = Bytes.toBytes("qualifier1");
4006    byte[] qf2 = Bytes.toBytes("qualifier2");
4007    byte[] fam1 = Bytes.toBytes("fam1");
4008    byte[][] families = { fam1 };
4009
4010    long ts1 = EnvironmentEdgeManager.currentTime();
4011    long ts2 = ts1 + 1;
4012    long ts3 = ts1 + 2;
4013
4014    // Setting up region
4015    this.region = initHRegion(tableName, method, CONF, families);
4016    // Putting data in Region
4017    Put put = null;
4018    KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
4019    KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
4020    KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
4021
4022    KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
4023    KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
4024    KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
4025
4026    put = new Put(row1);
4027    put.add(kv13);
4028    put.add(kv12);
4029    put.add(kv11);
4030    put.add(kv23);
4031    put.add(kv22);
4032    put.add(kv21);
4033    region.put(put);
4034
4035    // Expected
4036    List<Cell> expected = new ArrayList<>();
4037    expected.add(kv13);
4038    expected.add(kv12);
4039    expected.add(kv23);
4040    expected.add(kv22);
4041
4042    Scan scan = new Scan().withStartRow(row1);
4043    scan.addFamily(fam1);
4044    scan.readVersions(MAX_VERSIONS);
4045    List<Cell> actual = new ArrayList<>();
4046    try (InternalScanner scanner = region.getScanner(scan)) {
4047      boolean hasNext = scanner.next(actual);
4048      assertEquals(false, hasNext);
4049
4050      // Verify result
4051      for (int i = 0; i < expected.size(); i++) {
4052        assertEquals(expected.get(i), actual.get(i));
4053      }
4054    }
4055  }
4056
4057  @Test
4058  public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
4059    byte[] row1 = Bytes.toBytes("row1");
4060    byte[] qf1 = Bytes.toBytes("qualifier1");
4061    byte[] qf2 = Bytes.toBytes("qualifier2");
4062    byte[] fam1 = Bytes.toBytes("fam1");
4063
4064    long ts1 = 1;
4065    long ts2 = ts1 + 1;
4066    long ts3 = ts1 + 2;
4067
4068    // Setting up region
4069    this.region = initHRegion(tableName, method, CONF, fam1);
4070    // Putting data in Region
4071    Put put = null;
4072    KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
4073    KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
4074    KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
4075
4076    KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
4077    KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
4078    KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
4079
4080    put = new Put(row1);
4081    put.add(kv13);
4082    put.add(kv12);
4083    put.add(kv11);
4084    put.add(kv23);
4085    put.add(kv22);
4086    put.add(kv21);
4087    region.put(put);
4088    region.flush(true);
4089
4090    // Expected
4091    List<ExtendedCell> expected = new ArrayList<>();
4092    expected.add(kv13);
4093    expected.add(kv12);
4094    expected.add(kv23);
4095    expected.add(kv22);
4096
4097    Scan scan = new Scan().withStartRow(row1);
4098    scan.addFamily(fam1);
4099    scan.readVersions(MAX_VERSIONS);
4100    List<ExtendedCell> actual = new ArrayList<>();
4101    try (InternalScanner scanner = region.getScanner(scan)) {
4102      boolean hasNext = scanner.next(actual);
4103      assertEquals(false, hasNext);
4104
4105      // Verify result
4106      for (int i = 0; i < expected.size(); i++) {
4107        assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
4108      }
4109    }
4110  }
4111
4112  @Test
4113  public void testScanner_StopRow1542() throws IOException {
4114    byte[] family = Bytes.toBytes("testFamily");
4115    this.region = initHRegion(tableName, method, CONF, family);
4116    byte[] row1 = Bytes.toBytes("row111");
4117    byte[] row2 = Bytes.toBytes("row222");
4118    byte[] row3 = Bytes.toBytes("row333");
4119    byte[] row4 = Bytes.toBytes("row444");
4120    byte[] row5 = Bytes.toBytes("row555");
4121
4122    byte[] col1 = Bytes.toBytes("Pub111");
4123    byte[] col2 = Bytes.toBytes("Pub222");
4124
4125    Put put = new Put(row1);
4126    put.addColumn(family, col1, Bytes.toBytes(10L));
4127    region.put(put);
4128
4129    put = new Put(row2);
4130    put.addColumn(family, col1, Bytes.toBytes(15L));
4131    region.put(put);
4132
4133    put = new Put(row3);
4134    put.addColumn(family, col2, Bytes.toBytes(20L));
4135    region.put(put);
4136
4137    put = new Put(row4);
4138    put.addColumn(family, col2, Bytes.toBytes(30L));
4139    region.put(put);
4140
4141    put = new Put(row5);
4142    put.addColumn(family, col1, Bytes.toBytes(40L));
4143    region.put(put);
4144
4145    Scan scan = new Scan().withStartRow(row3).withStopRow(row4);
4146    scan.readAllVersions();
4147    scan.addColumn(family, col1);
4148    try (InternalScanner s = region.getScanner(scan)) {
4149      List<Cell> results = new ArrayList<>();
4150      assertEquals(false, s.next(results));
4151      assertEquals(0, results.size());
4152    }
4153  }
4154
4155  @Test
4156  public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
4157    byte[] row1 = Bytes.toBytes("row1");
4158    byte[] fam1 = Bytes.toBytes("fam1");
4159    byte[] qf1 = Bytes.toBytes("qualifier1");
4160    byte[] qf2 = Bytes.toBytes("quateslifier2");
4161
4162    long ts1 = 1;
4163    long ts2 = ts1 + 1;
4164    long ts3 = ts1 + 2;
4165    long ts4 = ts1 + 3;
4166
4167    // Setting up region
4168    this.region = initHRegion(tableName, method, CONF, fam1);
4169    // Putting data in Region
4170    KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
4171    KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
4172    KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
4173    KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
4174
4175    KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
4176    KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
4177    KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
4178    KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
4179
4180    Put put = null;
4181    put = new Put(row1);
4182    put.add(kv14);
4183    put.add(kv24);
4184    region.put(put);
4185    region.flush(true);
4186
4187    put = new Put(row1);
4188    put.add(kv23);
4189    put.add(kv13);
4190    region.put(put);
4191    region.flush(true);
4192
4193    put = new Put(row1);
4194    put.add(kv22);
4195    put.add(kv12);
4196    region.put(put);
4197    region.flush(true);
4198
4199    put = new Put(row1);
4200    put.add(kv21);
4201    put.add(kv11);
4202    region.put(put);
4203
4204    // Expected
4205    List<KeyValue> expected = new ArrayList<>();
4206    expected.add(kv14);
4207    expected.add(kv13);
4208    expected.add(kv12);
4209    expected.add(kv24);
4210    expected.add(kv23);
4211    expected.add(kv22);
4212
4213    Scan scan = new Scan().withStartRow(row1);
4214    int versions = 3;
4215    scan.readVersions(versions);
4216    List<ExtendedCell> actual = new ArrayList<>();
4217    try (InternalScanner scanner = region.getScanner(scan)) {
4218      boolean hasNext = scanner.next(actual);
4219      assertEquals(false, hasNext);
4220
4221      // Verify result
4222      for (int i = 0; i < expected.size(); i++) {
4223        assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
4224      }
4225    }
4226  }
4227
4228  /**
4229   * Added for HBASE-5416 Here we test scan optimization when only subset of CFs are used in filter
4230   * conditions.
4231   */
4232  @Test
4233  public void testScanner_JoinedScanners() throws IOException {
4234    byte[] cf_essential = Bytes.toBytes("essential");
4235    byte[] cf_joined = Bytes.toBytes("joined");
4236    byte[] cf_alpha = Bytes.toBytes("alpha");
4237    this.region = initHRegion(tableName, method, CONF, cf_essential, cf_joined, cf_alpha);
4238    byte[] row1 = Bytes.toBytes("row1");
4239    byte[] row2 = Bytes.toBytes("row2");
4240    byte[] row3 = Bytes.toBytes("row3");
4241
4242    byte[] col_normal = Bytes.toBytes("d");
4243    byte[] col_alpha = Bytes.toBytes("a");
4244
4245    byte[] filtered_val = Bytes.toBytes(3);
4246
4247    Put put = new Put(row1);
4248    put.addColumn(cf_essential, col_normal, Bytes.toBytes(1));
4249    put.addColumn(cf_joined, col_alpha, Bytes.toBytes(1));
4250    region.put(put);
4251
4252    put = new Put(row2);
4253    put.addColumn(cf_essential, col_alpha, Bytes.toBytes(2));
4254    put.addColumn(cf_joined, col_normal, Bytes.toBytes(2));
4255    put.addColumn(cf_alpha, col_alpha, Bytes.toBytes(2));
4256    region.put(put);
4257
4258    put = new Put(row3);
4259    put.addColumn(cf_essential, col_normal, filtered_val);
4260    put.addColumn(cf_joined, col_normal, filtered_val);
4261    region.put(put);
4262
4263    // Check two things:
4264    // 1. result list contains expected values
4265    // 2. result list is sorted properly
4266
4267    Scan scan = new Scan();
4268    Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
4269      CompareOperator.NOT_EQUAL, filtered_val);
4270    scan.setFilter(filter);
4271    scan.setLoadColumnFamiliesOnDemand(true);
4272    try (InternalScanner s = region.getScanner(scan)) {
4273      List<Cell> results = new ArrayList<>();
4274      assertTrue(s.next(results));
4275      assertEquals(1, results.size());
4276      results.clear();
4277
4278      assertTrue(s.next(results));
4279      assertEquals(3, results.size());
4280      assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
4281      assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
4282      assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
4283      results.clear();
4284
4285      assertFalse(s.next(results));
4286      assertEquals(0, results.size());
4287    }
4288  }
4289
4290  /**
4291   * HBASE-5416 Test case when scan limits amount of KVs returned on each next() call.
4292   */
4293  @Test
4294  public void testScanner_JoinedScannersWithLimits() throws IOException {
4295    final byte[] cf_first = Bytes.toBytes("first");
4296    final byte[] cf_second = Bytes.toBytes("second");
4297
4298    this.region = initHRegion(tableName, method, CONF, cf_first, cf_second);
4299    final byte[] col_a = Bytes.toBytes("a");
4300    final byte[] col_b = Bytes.toBytes("b");
4301
4302    Put put;
4303
4304    for (int i = 0; i < 10; i++) {
4305      put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
4306      put.addColumn(cf_first, col_a, Bytes.toBytes(i));
4307      if (i < 5) {
4308        put.addColumn(cf_first, col_b, Bytes.toBytes(i));
4309        put.addColumn(cf_second, col_a, Bytes.toBytes(i));
4310        put.addColumn(cf_second, col_b, Bytes.toBytes(i));
4311      }
4312      region.put(put);
4313    }
4314
4315    Scan scan = new Scan();
4316    scan.setLoadColumnFamiliesOnDemand(true);
4317    Filter bogusFilter = new FilterBase() {
4318      @Override
4319      public ReturnCode filterCell(final Cell ignored) throws IOException {
4320        return ReturnCode.INCLUDE;
4321      }
4322
4323      @Override
4324      public boolean isFamilyEssential(byte[] name) {
4325        return Bytes.equals(name, cf_first);
4326      }
4327    };
4328
4329    scan.setFilter(bogusFilter);
4330    try (InternalScanner s = region.getScanner(scan)) {
4331      // Our data looks like this:
4332      // r0: first:a, first:b, second:a, second:b
4333      // r1: first:a, first:b, second:a, second:b
4334      // r2: first:a, first:b, second:a, second:b
4335      // r3: first:a, first:b, second:a, second:b
4336      // r4: first:a, first:b, second:a, second:b
4337      // r5: first:a
4338      // r6: first:a
4339      // r7: first:a
4340      // r8: first:a
4341      // r9: first:a
4342
4343      // But due to next's limit set to 3, we should get this:
4344      // r0: first:a, first:b, second:a
4345      // r0: second:b
4346      // r1: first:a, first:b, second:a
4347      // r1: second:b
4348      // r2: first:a, first:b, second:a
4349      // r2: second:b
4350      // r3: first:a, first:b, second:a
4351      // r3: second:b
4352      // r4: first:a, first:b, second:a
4353      // r4: second:b
4354      // r5: first:a
4355      // r6: first:a
4356      // r7: first:a
4357      // r8: first:a
4358      // r9: first:a
4359
4360      List<Cell> results = new ArrayList<>();
4361      int index = 0;
4362      ScannerContext scannerContext = ScannerContext.newBuilder().setBatchLimit(3).build();
4363      while (true) {
4364        boolean more = s.next(results, scannerContext);
4365        if ((index >> 1) < 5) {
4366          if (index % 2 == 0) {
4367            assertEquals(3, results.size());
4368          } else {
4369            assertEquals(1, results.size());
4370          }
4371        } else {
4372          assertEquals(1, results.size());
4373        }
4374        results.clear();
4375        index++;
4376        if (!more) {
4377          break;
4378        }
4379      }
4380    }
4381  }
4382
4383  @Test
4384  public void testScannerOperationId() throws IOException {
4385    region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES);
4386    Scan scan = new Scan();
4387    try (RegionScanner scanner = region.getScanner(scan)) {
4388      assertNull(scanner.getOperationId());
4389    }
4390
4391    String operationId = "test_operation_id_0101";
4392    scan = new Scan().setId(operationId);
4393    try (RegionScanner scanner = region.getScanner(scan)) {
4394      assertEquals(operationId, scanner.getOperationId());
4395    }
4396
4397    HBaseTestingUtil.closeRegionAndWAL(this.region);
4398  }
4399
4400  /**
4401   * Write an HFile block full with Cells whose qualifier that are identical between 0 and
4402   * Short.MAX_VALUE. See HBASE-13329.
4403   */
4404  @Test
4405  public void testLongQualifier() throws Exception {
4406    byte[] family = Bytes.toBytes("family");
4407    this.region = initHRegion(tableName, method, CONF, family);
4408    byte[] q = new byte[Short.MAX_VALUE + 2];
4409    Arrays.fill(q, 0, q.length - 1, (byte) 42);
4410    for (byte i = 0; i < 10; i++) {
4411      Put p = new Put(Bytes.toBytes("row"));
4412      // qualifiers that differ past Short.MAX_VALUE
4413      q[q.length - 1] = i;
4414      p.addColumn(family, q, q);
4415      region.put(p);
4416    }
4417    region.flush(false);
4418  }
4419
4420  /**
4421   * Flushes the cache in a thread while scanning. The tests verify that the scan is coherent - e.g.
4422   * the returned results are always of the same or later update as the previous results. scan /
4423   * compact thread join
4424   */
4425  @Test
4426  public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
4427    byte[] family = Bytes.toBytes("family");
4428    int numRows = 1000;
4429    int flushAndScanInterval = 10;
4430    int compactInterval = 10 * flushAndScanInterval;
4431
4432    this.region = initHRegion(tableName, method, CONF, family);
4433    FlushThread flushThread = new FlushThread();
4434    try {
4435      flushThread.start();
4436
4437      Scan scan = new Scan();
4438      scan.addFamily(family);
4439      scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOperator.EQUAL,
4440        new BinaryComparator(Bytes.toBytes(5L))));
4441
4442      int expectedCount = 0;
4443      List<Cell> res = new ArrayList<>();
4444
4445      boolean toggle = true;
4446      for (long i = 0; i < numRows; i++) {
4447        Put put = new Put(Bytes.toBytes(i));
4448        put.setDurability(Durability.SKIP_WAL);
4449        put.addColumn(family, qual1, Bytes.toBytes(i % 10));
4450        region.put(put);
4451
4452        if (i != 0 && i % compactInterval == 0) {
4453          LOG.debug("iteration = " + i + " ts=" + EnvironmentEdgeManager.currentTime());
4454          region.compact(true);
4455        }
4456
4457        if (i % 10 == 5L) {
4458          expectedCount++;
4459        }
4460
4461        if (i != 0 && i % flushAndScanInterval == 0) {
4462          res.clear();
4463          try (InternalScanner scanner = region.getScanner(scan)) {
4464            if (toggle) {
4465              flushThread.flush();
4466            }
4467            while (scanner.next(res)) {
4468              // ignore
4469            }
4470          }
4471          if (!toggle) {
4472            flushThread.flush();
4473          }
4474          assertEquals(
4475            "toggle=" + toggle + "i=" + i + " ts=" + EnvironmentEdgeManager.currentTime(),
4476            expectedCount, res.size());
4477          toggle = !toggle;
4478        }
4479      }
4480
4481    } finally {
4482      try {
4483        flushThread.done();
4484        flushThread.join();
4485        flushThread.checkNoError();
4486      } catch (InterruptedException ie) {
4487        LOG.warn("Caught exception when joining with flushThread", ie);
4488      }
4489      HBaseTestingUtil.closeRegionAndWAL(this.region);
4490      this.region = null;
4491    }
4492  }
4493
4494  protected class FlushThread extends Thread {
4495    private volatile boolean done;
4496    private Throwable error = null;
4497
4498    FlushThread() {
4499      super("FlushThread");
4500    }
4501
4502    public void done() {
4503      done = true;
4504      synchronized (this) {
4505        interrupt();
4506      }
4507    }
4508
4509    public void checkNoError() {
4510      if (error != null) {
4511        assertNull(error);
4512      }
4513    }
4514
4515    @Override
4516    public void run() {
4517      done = false;
4518      while (!done) {
4519        synchronized (this) {
4520          try {
4521            wait();
4522          } catch (InterruptedException ignored) {
4523            if (done) {
4524              break;
4525            }
4526          }
4527        }
4528        try {
4529          region.flush(true);
4530        } catch (IOException e) {
4531          if (!done) {
4532            LOG.error("Error while flushing cache", e);
4533            error = e;
4534          }
4535          break;
4536        } catch (Throwable t) {
4537          LOG.error("Uncaught exception", t);
4538          throw t;
4539        }
4540      }
4541    }
4542
4543    public void flush() {
4544      synchronized (this) {
4545        notify();
4546      }
4547    }
4548  }
4549
4550  /**
4551   * So can be overridden in subclasses.
4552   */
4553  protected int getNumQualifiersForTestWritesWhileScanning() {
4554    return 100;
4555  }
4556
4557  /**
4558   * So can be overridden in subclasses.
4559   */
4560  protected int getTestCountForTestWritesWhileScanning() {
4561    return 100;
4562  }
4563
4564  /**
4565   * Writes very wide records and scans for the latest every time.. Flushes and compacts the region
4566   * every now and then to keep things realistic. by flush / scan / compaction when joining threads
4567   */
4568  @Test
4569  public void testWritesWhileScanning() throws IOException, InterruptedException {
4570    int testCount = getTestCountForTestWritesWhileScanning();
4571    int numRows = 1;
4572    int numFamilies = 10;
4573    int numQualifiers = getNumQualifiersForTestWritesWhileScanning();
4574    int flushInterval = 7;
4575    int compactInterval = 5 * flushInterval;
4576    byte[][] families = new byte[numFamilies][];
4577    for (int i = 0; i < numFamilies; i++) {
4578      families[i] = Bytes.toBytes("family" + i);
4579    }
4580    byte[][] qualifiers = new byte[numQualifiers][];
4581    for (int i = 0; i < numQualifiers; i++) {
4582      qualifiers[i] = Bytes.toBytes("qual" + i);
4583    }
4584
4585    this.region = initHRegion(tableName, method, CONF, families);
4586    FlushThread flushThread = new FlushThread();
4587    PutThread putThread = new PutThread(numRows, families, qualifiers);
4588    try {
4589      putThread.start();
4590      putThread.waitForFirstPut();
4591
4592      flushThread.start();
4593
4594      Scan scan = new Scan().withStartRow(Bytes.toBytes("row0")).withStopRow(Bytes.toBytes("row1"));
4595
4596      int expectedCount = numFamilies * numQualifiers;
4597      List<Cell> res = new ArrayList<>();
4598
4599      long prevTimestamp = 0L;
4600      for (int i = 0; i < testCount; i++) {
4601
4602        if (i != 0 && i % compactInterval == 0) {
4603          region.compact(true);
4604          for (HStore store : region.getStores()) {
4605            store.closeAndArchiveCompactedFiles();
4606          }
4607        }
4608
4609        if (i != 0 && i % flushInterval == 0) {
4610          flushThread.flush();
4611        }
4612
4613        boolean previousEmpty = res.isEmpty();
4614        res.clear();
4615        try (InternalScanner scanner = region.getScanner(scan)) {
4616          boolean moreRows;
4617          do {
4618            moreRows = scanner.next(res);
4619          } while (moreRows);
4620        }
4621        if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
4622          assertEquals("i=" + i, expectedCount, res.size());
4623          long timestamp = res.get(0).getTimestamp();
4624          assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
4625            timestamp >= prevTimestamp);
4626          prevTimestamp = timestamp;
4627        }
4628      }
4629
4630      putThread.done();
4631
4632      region.flush(true);
4633
4634    } finally {
4635      try {
4636        flushThread.done();
4637        flushThread.join();
4638        flushThread.checkNoError();
4639
4640        putThread.join();
4641        putThread.checkNoError();
4642      } catch (InterruptedException ie) {
4643        LOG.warn("Caught exception when joining with flushThread", ie);
4644      }
4645
4646      try {
4647        HBaseTestingUtil.closeRegionAndWAL(this.region);
4648      } catch (DroppedSnapshotException dse) {
4649        // We could get this on way out because we interrupt the background flusher and it could
4650        // fail anywhere causing a DSE over in the background flusher... only it is not properly
4651        // dealt with so could still be memory hanging out when we get to here -- memory we can't
4652        // flush because the accounting is 'off' since original DSE.
4653      }
4654      this.region = null;
4655    }
4656  }
4657
4658  @Test
4659  public void testCloseAndArchiveCompactedFiles() throws IOException {
4660    byte[] CF1 = Bytes.toBytes("CF1");
4661    byte[] CF2 = Bytes.toBytes("CF2");
4662    this.region = initHRegion(tableName, method, CONF, CF1, CF2);
4663    for (int i = 0; i < 2; i++) {
4664      int index = i;
4665      Put put =
4666        new Put(Bytes.toBytes(index)).addColumn(CF1, Bytes.toBytes("q"), Bytes.toBytes(index));
4667      region.put(put);
4668      region.flush(true);
4669    }
4670
4671    region.compact(true);
4672
4673    HStore store1 = region.getStore(CF1);
4674    HStore store2 = region.getStore(CF2);
4675    store1.closeAndArchiveCompactedFiles();
4676    store2.closeAndArchiveCompactedFiles();
4677
4678    int storefilesCount = region.getStores().stream().mapToInt(Store::getStorefilesCount).sum();
4679    assertTrue(storefilesCount == 1);
4680
4681    FileSystem fs = region.getRegionFileSystem().getFileSystem();
4682    Configuration conf = region.getReadOnlyConfiguration();
4683    RegionInfo regionInfo = region.getRegionInfo();
4684    Path store1ArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, CF1);
4685    assertTrue(fs.exists(store1ArchiveDir));
4686    // The archived dir of CF2 does not exist because this column family has no data at all
4687    Path store2ArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, CF2);
4688    assertFalse(fs.exists(store2ArchiveDir));
4689  }
4690
4691  protected class PutThread extends Thread {
4692    private volatile boolean done;
4693    private volatile int numPutsFinished = 0;
4694
4695    private Throwable error = null;
4696    private int numRows;
4697    private byte[][] families;
4698    private byte[][] qualifiers;
4699
4700    private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
4701      super("PutThread");
4702      this.numRows = numRows;
4703      this.families = families;
4704      this.qualifiers = qualifiers;
4705    }
4706
4707    /**
4708     * Block calling thread until this instance of PutThread has put at least one row.
4709     */
4710    public void waitForFirstPut() throws InterruptedException {
4711      // wait until put thread actually puts some data
4712      while (isAlive() && numPutsFinished == 0) {
4713        checkNoError();
4714        Thread.sleep(50);
4715      }
4716    }
4717
4718    public void done() {
4719      done = true;
4720      synchronized (this) {
4721        interrupt();
4722      }
4723    }
4724
4725    public void checkNoError() {
4726      if (error != null) {
4727        assertNull(error);
4728      }
4729    }
4730
4731    @Override
4732    public void run() {
4733      done = false;
4734      while (!done) {
4735        try {
4736          for (int r = 0; r < numRows; r++) {
4737            byte[] row = Bytes.toBytes("row" + r);
4738            Put put = new Put(row);
4739            put.setDurability(Durability.SKIP_WAL);
4740            byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
4741            for (byte[] family : families) {
4742              for (byte[] qualifier : qualifiers) {
4743                put.addColumn(family, qualifier, numPutsFinished, value);
4744              }
4745            }
4746            region.put(put);
4747            numPutsFinished++;
4748            if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
4749              LOG.debug("put iteration = {}", numPutsFinished);
4750              Delete delete = new Delete(row, (long) numPutsFinished - 30);
4751              region.delete(delete);
4752            }
4753            numPutsFinished++;
4754          }
4755        } catch (InterruptedIOException e) {
4756          // This is fine. It means we are done, or didn't get the lock on time
4757          LOG.info("Interrupted", e);
4758        } catch (IOException e) {
4759          LOG.error("Error while putting records", e);
4760          error = e;
4761          break;
4762        }
4763      }
4764
4765    }
4766
4767  }
4768
4769  /**
4770   * Writes very wide records and gets the latest row every time.. Flushes and compacts the region
4771   * aggressivly to catch issues. by flush / scan / compaction when joining threads
4772   */
4773  @Test
4774  public void testWritesWhileGetting() throws Exception {
4775    int testCount = 50;
4776    int numRows = 1;
4777    int numFamilies = 10;
4778    int numQualifiers = 100;
4779    int compactInterval = 100;
4780    byte[][] families = new byte[numFamilies][];
4781    for (int i = 0; i < numFamilies; i++) {
4782      families[i] = Bytes.toBytes("family" + i);
4783    }
4784    byte[][] qualifiers = new byte[numQualifiers][];
4785    for (int i = 0; i < numQualifiers; i++) {
4786      qualifiers[i] = Bytes.toBytes("qual" + i);
4787    }
4788
4789    // This test flushes constantly and can cause many files to be created,
4790    // possibly
4791    // extending over the ulimit. Make sure compactions are aggressive in
4792    // reducing
4793    // the number of HFiles created.
4794    Configuration conf = HBaseConfiguration.create(CONF);
4795    conf.setInt("hbase.hstore.compaction.min", 1);
4796    conf.setInt("hbase.hstore.compaction.max", 1000);
4797    this.region = initHRegion(tableName, method, conf, families);
4798    PutThread putThread = null;
4799    MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
4800    try {
4801      putThread = new PutThread(numRows, families, qualifiers);
4802      putThread.start();
4803      putThread.waitForFirstPut();
4804
4805      // Add a thread that flushes as fast as possible
4806      ctx.addThread(new RepeatingTestThread(ctx) {
4807
4808        @Override
4809        public void doAnAction() throws Exception {
4810          region.flush(true);
4811          // Compact regularly to avoid creating too many files and exceeding
4812          // the ulimit.
4813          region.compact(false);
4814          for (HStore store : region.getStores()) {
4815            store.closeAndArchiveCompactedFiles();
4816          }
4817        }
4818      });
4819      ctx.startThreads();
4820
4821      Get get = new Get(Bytes.toBytes("row0"));
4822      Result result = null;
4823
4824      int expectedCount = numFamilies * numQualifiers;
4825
4826      long prevTimestamp = 0L;
4827      for (int i = 0; i < testCount; i++) {
4828        LOG.info("testWritesWhileGetting verify turn " + i);
4829        boolean previousEmpty = result == null || result.isEmpty();
4830        result = region.get(get);
4831        if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
4832          assertEquals("i=" + i, expectedCount, result.size());
4833          // TODO this was removed, now what dangit?!
4834          // search looking for the qualifier in question?
4835          long timestamp = 0;
4836          for (Cell kv : result.rawCells()) {
4837            if (
4838              CellUtil.matchingFamily(kv, families[0])
4839                && CellUtil.matchingQualifier(kv, qualifiers[0])
4840            ) {
4841              timestamp = kv.getTimestamp();
4842            }
4843          }
4844          assertTrue(timestamp >= prevTimestamp);
4845          prevTimestamp = timestamp;
4846          ExtendedCell previousKV = null;
4847
4848          for (ExtendedCell kv : ClientInternalHelper.getExtendedRawCells(result)) {
4849            byte[] thisValue = CellUtil.cloneValue(kv);
4850            if (previousKV != null) {
4851              if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
4852                LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
4853                  + "(memStoreTS:" + previousKV.getSequenceId() + ")" + ", New KV: " + kv
4854                  + "(memStoreTS:" + kv.getSequenceId() + ")");
4855                assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
4856              }
4857            }
4858            previousKV = kv;
4859          }
4860        }
4861      }
4862    } finally {
4863      if (putThread != null) putThread.done();
4864
4865      region.flush(true);
4866
4867      if (putThread != null) {
4868        putThread.join();
4869        putThread.checkNoError();
4870      }
4871
4872      ctx.stop();
4873      HBaseTestingUtil.closeRegionAndWAL(this.region);
4874      this.region = null;
4875    }
4876  }
4877
4878  @Test
4879  public void testHolesInMeta() throws Exception {
4880    byte[] family = Bytes.toBytes("family");
4881    this.region =
4882      initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF, false, family);
4883    byte[] rowNotServed = Bytes.toBytes("a");
4884    Get g = new Get(rowNotServed);
4885    try {
4886      region.get(g);
4887      fail();
4888    } catch (WrongRegionException x) {
4889      // OK
4890    }
4891    byte[] row = Bytes.toBytes("y");
4892    g = new Get(row);
4893    region.get(g);
4894  }
4895
4896  @Test
4897  public void testIndexesScanWithOneDeletedRow() throws IOException {
4898    byte[] family = Bytes.toBytes("family");
4899
4900    // Setting up region
4901    this.region = initHRegion(tableName, method, CONF, family);
4902    Put put = new Put(Bytes.toBytes(1L));
4903    put.addColumn(family, qual1, 1L, Bytes.toBytes(1L));
4904    region.put(put);
4905
4906    region.flush(true);
4907
4908    Delete delete = new Delete(Bytes.toBytes(1L), 1L);
4909    region.delete(delete);
4910
4911    put = new Put(Bytes.toBytes(2L));
4912    put.addColumn(family, qual1, 2L, Bytes.toBytes(2L));
4913    region.put(put);
4914
4915    Scan idxScan = new Scan();
4916    idxScan.addFamily(family);
4917    idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,
4918      Arrays.<Filter> asList(
4919        new SingleColumnValueFilter(family, qual1, CompareOperator.GREATER_OR_EQUAL,
4920          new BinaryComparator(Bytes.toBytes(0L))),
4921        new SingleColumnValueFilter(family, qual1, CompareOperator.LESS_OR_EQUAL,
4922          new BinaryComparator(Bytes.toBytes(3L))))));
4923    try (InternalScanner scanner = region.getScanner(idxScan)) {
4924      List<Cell> res = new ArrayList<>();
4925
4926      while (scanner.next(res)) {
4927        // Ignore res value.
4928      }
4929      assertEquals(1L, res.size());
4930    }
4931  }
4932
4933  // ////////////////////////////////////////////////////////////////////////////
4934  // Bloom filter test
4935  // ////////////////////////////////////////////////////////////////////////////
4936  @Test
4937  public void testBloomFilterSize() throws IOException {
4938    byte[] fam1 = Bytes.toBytes("fam1");
4939    byte[] qf1 = Bytes.toBytes("col");
4940    byte[] val1 = Bytes.toBytes("value1");
4941    // Create Table
4942    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
4943      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam1)
4944        .setMaxVersions(Integer.MAX_VALUE).setBloomFilterType(BloomType.ROWCOL).build())
4945      .build();
4946    RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
4947    this.region = TEST_UTIL.createLocalHRegion(info, tableDescriptor);
4948    int num_unique_rows = 10;
4949    int duplicate_multiplier = 2;
4950    int num_storefiles = 4;
4951
4952    int version = 0;
4953    for (int f = 0; f < num_storefiles; f++) {
4954      for (int i = 0; i < duplicate_multiplier; i++) {
4955        for (int j = 0; j < num_unique_rows; j++) {
4956          Put put = new Put(Bytes.toBytes("row" + j));
4957          put.setDurability(Durability.SKIP_WAL);
4958          long ts = version++;
4959          put.addColumn(fam1, qf1, ts, val1);
4960          region.put(put);
4961        }
4962      }
4963      region.flush(true);
4964    }
4965    // before compaction
4966    HStore store = region.getStore(fam1);
4967    Collection<HStoreFile> storeFiles = store.getStorefiles();
4968    for (HStoreFile storefile : storeFiles) {
4969      StoreFileReader reader = storefile.getReader();
4970      reader.loadFileInfo();
4971      reader.loadBloomfilter();
4972      assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
4973      assertEquals(num_unique_rows, reader.getFilterEntries());
4974    }
4975
4976    region.compact(true);
4977
4978    // after compaction
4979    storeFiles = store.getStorefiles();
4980    for (HStoreFile storefile : storeFiles) {
4981      StoreFileReader reader = storefile.getReader();
4982      reader.loadFileInfo();
4983      reader.loadBloomfilter();
4984      assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
4985      assertEquals(num_unique_rows, reader.getFilterEntries());
4986    }
4987  }
4988
4989  @Test
4990  public void testAllColumnsWithBloomFilter() throws IOException {
4991    byte[] TABLE = Bytes.toBytes(name.getMethodName());
4992    byte[] FAMILY = Bytes.toBytes("family");
4993
4994    // Create table
4995    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(TableName.valueOf(TABLE))
4996      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY)
4997        .setMaxVersions(Integer.MAX_VALUE).setBloomFilterType(BloomType.ROWCOL).build())
4998      .build();
4999    RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
5000    this.region = TEST_UTIL.createLocalHRegion(info, tableDescriptor);
5001    // For row:0, col:0: insert versions 1 through 5.
5002    byte[] row = Bytes.toBytes("row:" + 0);
5003    byte[] column = Bytes.toBytes("column:" + 0);
5004    Put put = new Put(row);
5005    put.setDurability(Durability.SKIP_WAL);
5006    for (long idx = 1; idx <= 4; idx++) {
5007      put.addColumn(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
5008    }
5009    region.put(put);
5010
5011    // Flush
5012    region.flush(true);
5013
5014    // Get rows
5015    Get get = new Get(row);
5016    get.readAllVersions();
5017    Cell[] kvs = region.get(get).rawCells();
5018
5019    // Check if rows are correct
5020    assertEquals(4, kvs.length);
5021    checkOneCell(kvs[0], FAMILY, 0, 0, 4);
5022    checkOneCell(kvs[1], FAMILY, 0, 0, 3);
5023    checkOneCell(kvs[2], FAMILY, 0, 0, 2);
5024    checkOneCell(kvs[3], FAMILY, 0, 0, 1);
5025  }
5026
5027  /**
5028   * Testcase to cover bug-fix for HBASE-2823 Ensures correct delete when issuing delete row on
5029   * columns with bloom filter set to row+col (BloomType.ROWCOL)
5030   */
5031  @Test
5032  public void testDeleteRowWithBloomFilter() throws IOException {
5033    byte[] familyName = Bytes.toBytes("familyName");
5034
5035    // Create Table
5036    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
5037      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(familyName)
5038        .setMaxVersions(Integer.MAX_VALUE).setBloomFilterType(BloomType.ROWCOL).build())
5039      .build();
5040    RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
5041    this.region = TEST_UTIL.createLocalHRegion(info, tableDescriptor);
5042    // Insert some data
5043    byte[] row = Bytes.toBytes("row1");
5044    byte[] col = Bytes.toBytes("col1");
5045
5046    Put put = new Put(row);
5047    put.addColumn(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
5048    region.put(put);
5049    region.flush(true);
5050
5051    Delete del = new Delete(row);
5052    region.delete(del);
5053    region.flush(true);
5054
5055    // Get remaining rows (should have none)
5056    Get get = new Get(row);
5057    get.addColumn(familyName, col);
5058
5059    Cell[] keyValues = region.get(get).rawCells();
5060    assertEquals(0, keyValues.length);
5061  }
5062
5063  @Test
5064  public void testgetHDFSBlocksDistribution() throws Exception {
5065    HBaseTestingUtil htu = new HBaseTestingUtil();
5066    // Why do we set the block size in this test? If we set it smaller than the kvs, then we'll
5067    // break up the file in to more pieces that can be distributed across the three nodes and we
5068    // won't be able to have the condition this test asserts; that at least one node has
5069    // a copy of all replicas -- if small block size, then blocks are spread evenly across the
5070    // the three nodes. hfilev3 with tags seems to put us over the block size. St.Ack.
5071    // final int DEFAULT_BLOCK_SIZE = 1024;
5072    // htu.getConfiguration().setLong("dfs.blocksize", DEFAULT_BLOCK_SIZE);
5073    htu.getConfiguration().setInt("dfs.replication", 2);
5074
5075    // set up a cluster with 3 nodes
5076    SingleProcessHBaseCluster cluster = null;
5077    String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
5078    int regionServersCount = 3;
5079
5080    try {
5081      StartTestingClusterOption option = StartTestingClusterOption.builder()
5082        .numRegionServers(regionServersCount).dataNodeHosts(dataNodeHosts).build();
5083      cluster = htu.startMiniCluster(option);
5084      byte[][] families = { fam1, fam2 };
5085      Table ht = htu.createTable(tableName, families);
5086
5087      // Setting up region
5088      byte row[] = Bytes.toBytes("row1");
5089      byte col[] = Bytes.toBytes("col1");
5090
5091      Put put = new Put(row);
5092      put.addColumn(fam1, col, 1, Bytes.toBytes("test1"));
5093      put.addColumn(fam2, col, 1, Bytes.toBytes("test2"));
5094      ht.put(put);
5095
5096      HRegion firstRegion = htu.getHBaseCluster().getRegions(tableName).get(0);
5097      firstRegion.flush(true);
5098      HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();
5099
5100      // Given the default replication factor is 2 and we have 2 HFiles,
5101      // we will have total of 4 replica of blocks on 3 datanodes; thus there
5102      // must be at least one host that have replica for 2 HFiles. That host's
5103      // weight will be equal to the unique block weight.
5104      long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
5105      StringBuilder sb = new StringBuilder();
5106      for (String host : blocksDistribution1.getTopHosts()) {
5107        if (sb.length() > 0) sb.append(", ");
5108        sb.append(host);
5109        sb.append("=");
5110        sb.append(blocksDistribution1.getWeight(host));
5111      }
5112
5113      String topHost = blocksDistribution1.getTopHosts().get(0);
5114      long topHostWeight = blocksDistribution1.getWeight(topHost);
5115      String msg = "uniqueBlocksWeight=" + uniqueBlocksWeight1 + ", topHostWeight=" + topHostWeight
5116        + ", topHost=" + topHost + "; " + sb.toString();
5117      LOG.info(msg);
5118      assertTrue(msg, uniqueBlocksWeight1 == topHostWeight);
5119
5120      // use the static method to compute the value, it should be the same.
5121      // static method is used by load balancer or other components
5122      HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
5123        htu.getConfiguration(), firstRegion.getTableDescriptor(), firstRegion.getRegionInfo());
5124      long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();
5125
5126      assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
5127
5128      ht.close();
5129    } finally {
5130      if (cluster != null) {
5131        htu.shutdownMiniCluster();
5132      }
5133    }
5134  }
5135
5136  /**
5137   * Testcase to check state of region initialization task set to ABORTED or not if any exceptions
5138   * during initialization
5139   */
5140  @Test
5141  public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
5142    RegionInfo info;
5143    try {
5144      FileSystem fs = mock(FileSystem.class);
5145      when(fs.exists(any())).thenThrow(new IOException());
5146      TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(tableName);
5147      ColumnFamilyDescriptor columnFamilyDescriptor =
5148        ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf")).build();
5149      tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
5150      info = RegionInfoBuilder.newBuilder(tableName).build();
5151      Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
5152      region = HRegion.newHRegion(path, null, fs, CONF, info, tableDescriptorBuilder.build(), null);
5153      // region initialization throws IOException and set task state to ABORTED.
5154      region.initialize();
5155      fail("Region initialization should fail due to IOException");
5156    } catch (IOException io) {
5157      List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
5158      for (MonitoredTask monitoredTask : tasks) {
5159        if (
5160          !(monitoredTask instanceof MonitoredRPCHandler)
5161            && monitoredTask.getDescription().contains(region.toString())
5162        ) {
5163          assertTrue("Region state should be ABORTED.",
5164            monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
5165          break;
5166        }
5167      }
5168    }
5169  }
5170
5171  /**
5172   * Verifies that the .regioninfo file is written on region creation and that is recreated if
5173   * missing during region opening.
5174   */
5175  @Test
5176  public void testRegionInfoFileCreation() throws IOException {
5177    Path rootDir = new Path(dir + "testRegionInfoFileCreation");
5178
5179    TableDescriptorBuilder tableDescriptorBuilder =
5180      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()));
5181    ColumnFamilyDescriptor columnFamilyDescriptor =
5182      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf")).build();
5183    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
5184    TableDescriptor tableDescriptor = tableDescriptorBuilder.build();
5185
5186    RegionInfo hri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
5187
5188    // Create a region and skip the initialization (like CreateTableHandler)
5189    region = HBaseTestingUtil.createRegionAndWAL(hri, rootDir, CONF, tableDescriptor, false);
5190    Path regionDir = region.getRegionFileSystem().getRegionDir();
5191    FileSystem fs = region.getRegionFileSystem().getFileSystem();
5192    HBaseTestingUtil.closeRegionAndWAL(region);
5193
5194    Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
5195
5196    // Verify that the .regioninfo file is present
5197    assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
5198      fs.exists(regionInfoFile));
5199
5200    // Try to open the region
5201    region = HRegion.openHRegion(rootDir, hri, tableDescriptor, null, CONF);
5202    assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
5203    HBaseTestingUtil.closeRegionAndWAL(region);
5204
5205    // Verify that the .regioninfo file is still there
5206    assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
5207      fs.exists(regionInfoFile));
5208
5209    // Remove the .regioninfo file and verify is recreated on region open
5210    fs.delete(regionInfoFile, true);
5211    assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
5212      fs.exists(regionInfoFile));
5213
5214    region = HRegion.openHRegion(rootDir, hri, tableDescriptor, null, CONF);
5215    // region = TEST_UTIL.openHRegion(hri, htd);
5216    assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
5217    HBaseTestingUtil.closeRegionAndWAL(region);
5218
5219    // Verify that the .regioninfo file is still there
5220    assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
5221      fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
5222
5223    region = null;
5224  }
5225
5226  /**
5227   * TestCase for increment
5228   */
5229  private static class Incrementer implements Runnable {
5230    private HRegion region;
5231    private final static byte[] incRow = Bytes.toBytes("incRow");
5232    private final static byte[] family = Bytes.toBytes("family");
5233    private final static byte[] qualifier = Bytes.toBytes("qualifier");
5234    private final static long ONE = 1L;
5235    private int incCounter;
5236
5237    public Incrementer(HRegion region, int incCounter) {
5238      this.region = region;
5239      this.incCounter = incCounter;
5240    }
5241
5242    @Override
5243    public void run() {
5244      int count = 0;
5245      while (count < incCounter) {
5246        Increment inc = new Increment(incRow);
5247        inc.addColumn(family, qualifier, ONE);
5248        count++;
5249        try {
5250          region.increment(inc);
5251        } catch (IOException e) {
5252          LOG.info("Count=" + count + ", " + e);
5253          break;
5254        }
5255      }
5256    }
5257  }
5258
5259  /**
5260   * Test case to check increment function with memstore flushing
5261   */
5262  @Test
5263  public void testParallelIncrementWithMemStoreFlush() throws Exception {
5264    byte[] family = Incrementer.family;
5265    this.region = initHRegion(tableName, method, CONF, family);
5266    final HRegion region = this.region;
5267    final AtomicBoolean incrementDone = new AtomicBoolean(false);
5268    Runnable flusher = new Runnable() {
5269      @Override
5270      public void run() {
5271        while (!incrementDone.get()) {
5272          try {
5273            region.flush(true);
5274          } catch (Exception e) {
5275            e.printStackTrace();
5276          }
5277        }
5278      }
5279    };
5280
5281    // after all increment finished, the row will increment to 20*100 = 2000
5282    int threadNum = 20;
5283    int incCounter = 100;
5284    long expected = (long) threadNum * incCounter;
5285    Thread[] incrementers = new Thread[threadNum];
5286    Thread flushThread = new Thread(flusher);
5287    for (int i = 0; i < threadNum; i++) {
5288      incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
5289      incrementers[i].start();
5290    }
5291    flushThread.start();
5292    for (int i = 0; i < threadNum; i++) {
5293      incrementers[i].join();
5294    }
5295
5296    incrementDone.set(true);
5297    flushThread.join();
5298
5299    Get get = new Get(Incrementer.incRow);
5300    get.addColumn(Incrementer.family, Incrementer.qualifier);
5301    get.readVersions(1);
5302    Result res = this.region.get(get);
5303    List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
5304
5305    // we just got the latest version
5306    assertEquals(1, kvs.size());
5307    Cell kv = kvs.get(0);
5308    assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
5309  }
5310
5311  /**
5312   * TestCase for append
5313   */
5314  private static class Appender implements Runnable {
5315    private HRegion region;
5316    private final static byte[] appendRow = Bytes.toBytes("appendRow");
5317    private final static byte[] family = Bytes.toBytes("family");
5318    private final static byte[] qualifier = Bytes.toBytes("qualifier");
5319    private final static byte[] CHAR = Bytes.toBytes("a");
5320    private int appendCounter;
5321
5322    public Appender(HRegion region, int appendCounter) {
5323      this.region = region;
5324      this.appendCounter = appendCounter;
5325    }
5326
5327    @Override
5328    public void run() {
5329      int count = 0;
5330      while (count < appendCounter) {
5331        Append app = new Append(appendRow);
5332        app.addColumn(family, qualifier, CHAR);
5333        count++;
5334        try {
5335          region.append(app);
5336        } catch (IOException e) {
5337          LOG.info("Count=" + count + ", max=" + appendCounter + ", " + e);
5338          break;
5339        }
5340      }
5341    }
5342  }
5343
5344  /**
5345   * Test case to check append function with memstore flushing
5346   */
5347  @Test
5348  public void testParallelAppendWithMemStoreFlush() throws Exception {
5349    byte[] family = Appender.family;
5350    this.region = initHRegion(tableName, method, CONF, family);
5351    final HRegion region = this.region;
5352    final AtomicBoolean appendDone = new AtomicBoolean(false);
5353    Runnable flusher = new Runnable() {
5354      @Override
5355      public void run() {
5356        while (!appendDone.get()) {
5357          try {
5358            region.flush(true);
5359          } catch (Exception e) {
5360            e.printStackTrace();
5361          }
5362        }
5363      }
5364    };
5365
5366    // After all append finished, the value will append to threadNum *
5367    // appendCounter Appender.CHAR
5368    int threadNum = 20;
5369    int appendCounter = 100;
5370    byte[] expected = new byte[threadNum * appendCounter];
5371    for (int i = 0; i < threadNum * appendCounter; i++) {
5372      System.arraycopy(Appender.CHAR, 0, expected, i, 1);
5373    }
5374    Thread[] appenders = new Thread[threadNum];
5375    Thread flushThread = new Thread(flusher);
5376    for (int i = 0; i < threadNum; i++) {
5377      appenders[i] = new Thread(new Appender(this.region, appendCounter));
5378      appenders[i].start();
5379    }
5380    flushThread.start();
5381    for (int i = 0; i < threadNum; i++) {
5382      appenders[i].join();
5383    }
5384
5385    appendDone.set(true);
5386    flushThread.join();
5387
5388    Get get = new Get(Appender.appendRow);
5389    get.addColumn(Appender.family, Appender.qualifier);
5390    get.readVersions(1);
5391    Result res = this.region.get(get);
5392    List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
5393
5394    // we just got the latest version
5395    assertEquals(1, kvs.size());
5396    Cell kv = kvs.get(0);
5397    byte[] appendResult = new byte[kv.getValueLength()];
5398    System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
5399    assertArrayEquals(expected, appendResult);
5400  }
5401
5402  /**
5403   * Test case to check put function with memstore flushing for same row, same ts
5404   */
5405  @Test
5406  public void testPutWithMemStoreFlush() throws Exception {
5407    byte[] family = Bytes.toBytes("family");
5408    byte[] qualifier = Bytes.toBytes("qualifier");
5409    byte[] row = Bytes.toBytes("putRow");
5410    byte[] value = null;
5411    this.region = initHRegion(tableName, method, CONF, family);
5412    Put put = null;
5413    Get get = null;
5414    List<Cell> kvs = null;
5415    Result res = null;
5416
5417    put = new Put(row);
5418    value = Bytes.toBytes("value0");
5419    put.addColumn(family, qualifier, 1234567L, value);
5420    region.put(put);
5421    get = new Get(row);
5422    get.addColumn(family, qualifier);
5423    get.readAllVersions();
5424    res = this.region.get(get);
5425    kvs = res.getColumnCells(family, qualifier);
5426    assertEquals(1, kvs.size());
5427    assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
5428
5429    region.flush(true);
5430    get = new Get(row);
5431    get.addColumn(family, qualifier);
5432    get.readAllVersions();
5433    res = this.region.get(get);
5434    kvs = res.getColumnCells(family, qualifier);
5435    assertEquals(1, kvs.size());
5436    assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
5437
5438    put = new Put(row);
5439    value = Bytes.toBytes("value1");
5440    put.addColumn(family, qualifier, 1234567L, value);
5441    region.put(put);
5442    get = new Get(row);
5443    get.addColumn(family, qualifier);
5444    get.readAllVersions();
5445    res = this.region.get(get);
5446    kvs = res.getColumnCells(family, qualifier);
5447    assertEquals(1, kvs.size());
5448    assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
5449
5450    region.flush(true);
5451    get = new Get(row);
5452    get.addColumn(family, qualifier);
5453    get.readAllVersions();
5454    res = this.region.get(get);
5455    kvs = res.getColumnCells(family, qualifier);
5456    assertEquals(1, kvs.size());
5457    assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
5458  }
5459
5460  /**
5461   * For this test,the spied {@link AsyncFSWAL} can not work properly because of a Mockito defect
5462   * that can not deal with classes which have a field of an inner class. See discussions in
5463   * HBASE-15536.When we reuse the code of {@link AsyncFSWAL} for {@link FSHLog}, this test could
5464   * not work for {@link FSHLog} also.
5465   */
5466  @Test
5467  @Ignore
5468  public void testDurability() throws Exception {
5469    // there are 5 x 5 cases:
5470    // table durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT) x mutation
5471    // durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT)
5472
5473    // expected cases for append and sync wal
5474    durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
5475    durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
5476    durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
5477
5478    durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
5479    durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
5480    durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
5481
5482    durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
5483    durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
5484
5485    durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
5486    durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
5487
5488    durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
5489    durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
5490    durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
5491
5492    // expected cases for async wal
5493    durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
5494    durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
5495    durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
5496    durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
5497    durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
5498    durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
5499
5500    durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
5501    durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
5502    durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
5503    durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
5504    durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
5505    durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
5506
5507    // expect skip wal cases
5508    durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
5509    durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
5510    durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
5511    durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false);
5512    durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false);
5513    durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false);
5514
5515  }
5516
5517  private void durabilityTest(String method, Durability tableDurability,
5518    Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync,
5519    final boolean expectSyncFromLogSyncer) throws Exception {
5520    Configuration conf = HBaseConfiguration.create(CONF);
5521    conf.setLong(AbstractFSWAL.WAL_SHUTDOWN_WAIT_TIMEOUT_MS, 60 * 60 * 1000);
5522    method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
5523    byte[] family = Bytes.toBytes("family");
5524    Path logDir = new Path(new Path(dir + method), "log");
5525    final Configuration walConf = new Configuration(conf);
5526    CommonFSUtils.setRootDir(walConf, logDir);
5527    // XXX: The spied AsyncFSWAL can not work properly because of a Mockito defect that can not
5528    // deal with classes which have a field of an inner class. See discussions in HBASE-15536.
5529    walConf.set(WALFactory.WAL_PROVIDER, "filesystem");
5530    final WALFactory wals = new WALFactory(walConf, HBaseTestingUtil.getRandomUUID().toString());
5531    final WAL wal = spy(wals.getWAL(RegionInfoBuilder.newBuilder(tableName).build()));
5532    this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF,
5533      false, tableDurability, wal, new byte[][] { family });
5534
5535    Put put = new Put(Bytes.toBytes("r1"));
5536    put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
5537    put.setDurability(mutationDurability);
5538    region.put(put);
5539
5540    // verify append called or not
5541    verify(wal, expectAppend ? times(1) : never()).appendData((RegionInfo) any(),
5542      (WALKeyImpl) any(), (WALEdit) any());
5543
5544    // verify sync called or not
5545    if (expectSync || expectSyncFromLogSyncer) {
5546      TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
5547        @Override
5548        public boolean evaluate() throws Exception {
5549          try {
5550            if (expectSync) {
5551              verify(wal, times(1)).sync(anyLong()); // Hregion calls this one
5552            } else if (expectSyncFromLogSyncer) {
5553              verify(wal, times(1)).sync(); // wal syncer calls this one
5554            }
5555          } catch (Throwable ignore) {
5556          }
5557          return true;
5558        }
5559      });
5560    } else {
5561      // verify(wal, never()).sync(anyLong());
5562      verify(wal, never()).sync();
5563    }
5564
5565    HBaseTestingUtil.closeRegionAndWAL(this.region);
5566    wals.close();
5567    this.region = null;
5568  }
5569
5570  @Test
5571  public void testRegionReplicaSecondary() throws IOException {
5572    // create a primary region, load some data and flush
5573    // create a secondary region, and do a get against that
5574    Path rootDir = new Path(dir + name.getMethodName());
5575    CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
5576
5577    byte[][] families =
5578      new byte[][] { Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3") };
5579    byte[] cq = Bytes.toBytes("cq");
5580    TableDescriptorBuilder builder =
5581      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()));
5582    for (byte[] family : families) {
5583      builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family));
5584    }
5585    TableDescriptor tableDescriptor = builder.build();
5586    long time = EnvironmentEdgeManager.currentTime();
5587    RegionInfo primaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName())
5588      .setRegionId(time).setReplicaId(0).build();
5589    RegionInfo secondaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName())
5590      .setRegionId(time).setReplicaId(1).build();
5591
5592    HRegion primaryRegion = null, secondaryRegion = null;
5593
5594    try {
5595      primaryRegion = HBaseTestingUtil.createRegionAndWAL(primaryHri, rootDir,
5596        TEST_UTIL.getConfiguration(), tableDescriptor);
5597
5598      // load some data
5599      putData(primaryRegion, 0, 1000, cq, families);
5600
5601      // flush region
5602      primaryRegion.flush(true);
5603
5604      // open secondary region
5605      secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, tableDescriptor, null, CONF);
5606
5607      verifyData(secondaryRegion, 0, 1000, cq, families);
5608    } finally {
5609      if (primaryRegion != null) {
5610        HBaseTestingUtil.closeRegionAndWAL(primaryRegion);
5611      }
5612      if (secondaryRegion != null) {
5613        HBaseTestingUtil.closeRegionAndWAL(secondaryRegion);
5614      }
5615    }
5616  }
5617
5618  @Test
5619  public void testRegionReplicaSecondaryIsReadOnly() throws IOException {
5620    // create a primary region, load some data and flush
5621    // create a secondary region, and do a put against that
5622    Path rootDir = new Path(dir + name.getMethodName());
5623    CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
5624
5625    byte[][] families =
5626      new byte[][] { Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3") };
5627    byte[] cq = Bytes.toBytes("cq");
5628    TableDescriptorBuilder builder =
5629      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()));
5630    for (byte[] family : families) {
5631      builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family));
5632    }
5633    TableDescriptor tableDescriptor = builder.build();
5634    long time = EnvironmentEdgeManager.currentTime();
5635    RegionInfo primaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName())
5636      .setRegionId(time).setReplicaId(0).build();
5637    RegionInfo secondaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName())
5638      .setRegionId(time).setReplicaId(1).build();
5639
5640    HRegion primaryRegion = null, secondaryRegion = null;
5641
5642    try {
5643      primaryRegion = HBaseTestingUtil.createRegionAndWAL(primaryHri, rootDir,
5644        TEST_UTIL.getConfiguration(), tableDescriptor);
5645
5646      // load some data
5647      putData(primaryRegion, 0, 1000, cq, families);
5648
5649      // flush region
5650      primaryRegion.flush(true);
5651
5652      // open secondary region
5653      secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, tableDescriptor, null, CONF);
5654
5655      try {
5656        putData(secondaryRegion, 0, 1000, cq, families);
5657        fail("Should have thrown exception");
5658      } catch (IOException ex) {
5659        // expected
5660      }
5661    } finally {
5662      if (primaryRegion != null) {
5663        HBaseTestingUtil.closeRegionAndWAL(primaryRegion);
5664      }
5665      if (secondaryRegion != null) {
5666        HBaseTestingUtil.closeRegionAndWAL(secondaryRegion);
5667      }
5668    }
5669  }
5670
5671  static WALFactory createWALFactory(Configuration conf, Path rootDir) throws IOException {
5672    Configuration confForWAL = new Configuration(conf);
5673    confForWAL.set(HConstants.HBASE_DIR, rootDir.toString());
5674    return new WALFactory(confForWAL, "hregion-" + RandomStringUtils.randomNumeric(8));
5675  }
5676
5677  @Test
5678  public void testCompactionFromPrimary() throws IOException {
5679    Path rootDir = new Path(dir + name.getMethodName());
5680    CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
5681
5682    byte[][] families =
5683      new byte[][] { Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3") };
5684    byte[] cq = Bytes.toBytes("cq");
5685    TableDescriptorBuilder builder =
5686      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()));
5687    for (byte[] family : families) {
5688      builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family));
5689    }
5690    TableDescriptor tableDescriptor = builder.build();
5691    long time = EnvironmentEdgeManager.currentTime();
5692    RegionInfo primaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName())
5693      .setRegionId(time).setReplicaId(0).build();
5694    RegionInfo secondaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName())
5695      .setRegionId(time).setReplicaId(1).build();
5696
5697    HRegion primaryRegion = null, secondaryRegion = null;
5698
5699    try {
5700      primaryRegion = HBaseTestingUtil.createRegionAndWAL(primaryHri, rootDir,
5701        TEST_UTIL.getConfiguration(), tableDescriptor);
5702
5703      // load some data
5704      putData(primaryRegion, 0, 1000, cq, families);
5705
5706      // flush region
5707      primaryRegion.flush(true);
5708
5709      // open secondary region
5710      secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, tableDescriptor, null, CONF);
5711
5712      // move the file of the primary region to the archive, simulating a compaction
5713      Collection<HStoreFile> storeFiles = primaryRegion.getStore(families[0]).getStorefiles();
5714      HRegionFileSystem regionFs = primaryRegion.getRegionFileSystem();
5715      StoreFileTracker sft = StoreFileTrackerFactory.create(primaryRegion.getBaseConf(), false,
5716        StoreContext.getBuilder()
5717          .withColumnFamilyDescriptor(ColumnFamilyDescriptorBuilder.newBuilder(families[0]).build())
5718          .withFamilyStoreDirectoryPath(
5719            new Path(regionFs.getRegionDir(), Bytes.toString(families[0])))
5720          .withRegionFileSystem(regionFs).build());
5721      sft.removeStoreFiles(storeFiles.stream().collect(Collectors.toList()));
5722      Collection<StoreFileInfo> storeFileInfos = sft.load();
5723      Assert.assertTrue(storeFileInfos == null || storeFileInfos.isEmpty());
5724
5725      verifyData(secondaryRegion, 0, 1000, cq, families);
5726    } finally {
5727      if (primaryRegion != null) {
5728        HBaseTestingUtil.closeRegionAndWAL(primaryRegion);
5729      }
5730      if (secondaryRegion != null) {
5731        HBaseTestingUtil.closeRegionAndWAL(secondaryRegion);
5732      }
5733    }
5734  }
5735
5736  private void putData(int startRow, int numRows, byte[] qf, byte[]... families)
5737    throws IOException {
5738    putData(this.region, startRow, numRows, qf, families);
5739  }
5740
5741  private void putData(HRegion region, int startRow, int numRows, byte[] qf, byte[]... families)
5742    throws IOException {
5743    putData(region, Durability.SKIP_WAL, startRow, numRows, qf, families);
5744  }
5745
5746  static void putData(HRegion region, Durability durability, int startRow, int numRows, byte[] qf,
5747    byte[]... families) throws IOException {
5748    for (int i = startRow; i < startRow + numRows; i++) {
5749      Put put = new Put(Bytes.toBytes("" + i));
5750      put.setDurability(durability);
5751      for (byte[] family : families) {
5752        put.addColumn(family, qf, null);
5753      }
5754      region.put(put);
5755      LOG.info(put.toString());
5756    }
5757  }
5758
5759  static void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
5760    throws IOException {
5761    for (int i = startRow; i < startRow + numRows; i++) {
5762      byte[] row = Bytes.toBytes("" + i);
5763      Get get = new Get(row);
5764      for (byte[] family : families) {
5765        get.addColumn(family, qf);
5766      }
5767      Result result = newReg.get(get);
5768      Cell[] raw = result.rawCells();
5769      assertEquals(families.length, result.size());
5770      for (int j = 0; j < families.length; j++) {
5771        assertTrue(CellUtil.matchingRows(raw[j], row));
5772        assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
5773        assertTrue(CellUtil.matchingQualifier(raw[j], qf));
5774      }
5775    }
5776  }
5777
5778  static void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
5779    // Now I have k, get values out and assert they are as expected.
5780    Get get = new Get(k).addFamily(family).readAllVersions();
5781    Cell[] results = r.get(get).rawCells();
5782    for (int j = 0; j < results.length; j++) {
5783      byte[] tmp = CellUtil.cloneValue(results[j]);
5784      // Row should be equal to value every time.
5785      assertTrue(Bytes.equals(k, tmp));
5786    }
5787  }
5788
5789  /*
5790   * Assert first value in the passed region is <code>firstValue</code>.
5791   */
5792  protected void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue)
5793    throws IOException {
5794    byte[][] families = { fs };
5795    Scan scan = new Scan();
5796    for (int i = 0; i < families.length; i++)
5797      scan.addFamily(families[i]);
5798    try (InternalScanner s = r.getScanner(scan)) {
5799      List<Cell> curVals = new ArrayList<>();
5800      boolean first = true;
5801      OUTER_LOOP: while (s.next(curVals)) {
5802        for (Cell kv : curVals) {
5803          byte[] val = CellUtil.cloneValue(kv);
5804          byte[] curval = val;
5805          if (first) {
5806            first = false;
5807            assertTrue(Bytes.compareTo(curval, firstValue) == 0);
5808          } else {
5809            // Not asserting anything. Might as well break.
5810            break OUTER_LOOP;
5811          }
5812        }
5813      }
5814    }
5815  }
5816
5817  /**
5818   * Test that we get the expected flush results back
5819   */
5820  @Test
5821  public void testFlushResult() throws IOException {
5822    byte[] family = Bytes.toBytes("family");
5823
5824    this.region = initHRegion(tableName, method, family);
5825
5826    // empty memstore, flush doesn't run
5827    HRegion.FlushResult fr = region.flush(true);
5828    assertFalse(fr.isFlushSucceeded());
5829    assertFalse(fr.isCompactionNeeded());
5830
5831    // Flush enough files to get up to the threshold, doesn't need compactions
5832    for (int i = 0; i < 2; i++) {
5833      Put put = new Put(tableName.toBytes()).addColumn(family, family, tableName.toBytes());
5834      region.put(put);
5835      fr = region.flush(true);
5836      assertTrue(fr.isFlushSucceeded());
5837      assertFalse(fr.isCompactionNeeded());
5838    }
5839
5840    // Two flushes after the threshold, compactions are needed
5841    for (int i = 0; i < 2; i++) {
5842      Put put = new Put(tableName.toBytes()).addColumn(family, family, tableName.toBytes());
5843      region.put(put);
5844      fr = region.flush(true);
5845      assertTrue(fr.isFlushSucceeded());
5846      assertTrue(fr.isCompactionNeeded());
5847    }
5848  }
5849
5850  protected Configuration initSplit() {
5851    // Always compact if there is more than one store file.
5852    CONF.setInt("hbase.hstore.compactionThreshold", 2);
5853
5854    CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
5855
5856    // Increase the amount of time between client retries
5857    CONF.setLong("hbase.client.pause", 15 * 1000);
5858
5859    // This size should make it so we always split using the addContent
5860    // below. After adding all data, the first region is 1.3M
5861    CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
5862    return CONF;
5863  }
5864
5865  /**
5866   * @return A region on which you must call {@link HBaseTestingUtil#closeRegionAndWAL(HRegion)}
5867   *         when done.
5868   */
5869  protected HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
5870    byte[]... families) throws IOException {
5871    return initHRegion(tableName, callingMethod, conf, false, families);
5872  }
5873
5874  /**
5875   * @return A region on which you must call {@link HBaseTestingUtil#closeRegionAndWAL(HRegion)}
5876   *         when done.
5877   */
5878  private HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
5879    boolean isReadOnly, byte[]... families) throws IOException {
5880    return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
5881  }
5882
5883  private HRegion initHRegion(TableName tableName, byte[] startKey, byte[] stopKey,
5884    String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
5885    throws IOException {
5886    Path logDir = TEST_UTIL.getDataTestDirOnTestFS(callingMethod + ".log");
5887    RegionInfo hri =
5888      RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey).setEndKey(stopKey).build();
5889    final WAL wal = HBaseTestingUtil.createWal(conf, logDir, hri);
5890    return initHRegion(tableName, startKey, stopKey, conf, isReadOnly, Durability.SYNC_WAL, wal,
5891      families);
5892  }
5893
5894  /**
5895   * @return A region on which you must call {@link HBaseTestingUtil#closeRegionAndWAL(HRegion)}
5896   *         when done.
5897   */
5898  protected HRegion initHRegion(TableName tableName, byte[] startKey, byte[] stopKey,
5899    Configuration conf, boolean isReadOnly, Durability durability, WAL wal, byte[]... families)
5900    throws IOException {
5901    ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null,
5902      MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);
5903    return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, conf, isReadOnly, durability,
5904      wal, families);
5905  }
5906
5907  /**
5908   * Assert that the passed in Cell has expected contents for the specified row, column & timestamp.
5909   */
5910  private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
5911    String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
5912    assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
5913      Bytes.toString(CellUtil.cloneRow(kv)));
5914    assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
5915      Bytes.toString(CellUtil.cloneFamily(kv)));
5916    assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
5917      Bytes.toString(CellUtil.cloneQualifier(kv)));
5918    assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
5919    assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
5920      Bytes.toString(CellUtil.cloneValue(kv)));
5921  }
5922
5923  @Test
5924  public void testReverseScanner_FromMemStore_SingleCF_Normal() throws IOException {
5925    byte[] rowC = Bytes.toBytes("rowC");
5926    byte[] rowA = Bytes.toBytes("rowA");
5927    byte[] rowB = Bytes.toBytes("rowB");
5928    byte[] cf = Bytes.toBytes("CF");
5929    byte[][] families = { cf };
5930    byte[] col = Bytes.toBytes("C");
5931    long ts = 1;
5932    this.region = initHRegion(tableName, method, families);
5933    KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
5934    KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, null);
5935    KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
5936    KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
5937    Put put = null;
5938    put = new Put(rowC);
5939    put.add(kv1);
5940    put.add(kv11);
5941    region.put(put);
5942    put = new Put(rowA);
5943    put.add(kv2);
5944    region.put(put);
5945    put = new Put(rowB);
5946    put.add(kv3);
5947    region.put(put);
5948
5949    Scan scan = new Scan().withStartRow(rowC);
5950    scan.readVersions(5);
5951    scan.setReversed(true);
5952    try (InternalScanner scanner = region.getScanner(scan)) {
5953      List<Cell> currRow = new ArrayList<>();
5954      boolean hasNext = scanner.next(currRow);
5955      assertEquals(2, currRow.size());
5956      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
5957        currRow.get(0).getRowLength(), rowC, 0, rowC.length));
5958      assertTrue(hasNext);
5959      currRow.clear();
5960      hasNext = scanner.next(currRow);
5961      assertEquals(1, currRow.size());
5962      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
5963        currRow.get(0).getRowLength(), rowB, 0, rowB.length));
5964      assertTrue(hasNext);
5965      currRow.clear();
5966      hasNext = scanner.next(currRow);
5967      assertEquals(1, currRow.size());
5968      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
5969        currRow.get(0).getRowLength(), rowA, 0, rowA.length));
5970      assertFalse(hasNext);
5971    }
5972  }
5973
5974  @Test
5975  public void testReverseScanner_FromMemStore_SingleCF_LargerKey() throws IOException {
5976    byte[] rowC = Bytes.toBytes("rowC");
5977    byte[] rowA = Bytes.toBytes("rowA");
5978    byte[] rowB = Bytes.toBytes("rowB");
5979    byte[] rowD = Bytes.toBytes("rowD");
5980    byte[] cf = Bytes.toBytes("CF");
5981    byte[][] families = { cf };
5982    byte[] col = Bytes.toBytes("C");
5983    long ts = 1;
5984    this.region = initHRegion(tableName, method, families);
5985    KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
5986    KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, null);
5987    KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
5988    KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
5989    Put put = null;
5990    put = new Put(rowC);
5991    put.add(kv1);
5992    put.add(kv11);
5993    region.put(put);
5994    put = new Put(rowA);
5995    put.add(kv2);
5996    region.put(put);
5997    put = new Put(rowB);
5998    put.add(kv3);
5999    region.put(put);
6000
6001    Scan scan = new Scan().withStartRow(rowD);
6002    List<Cell> currRow = new ArrayList<>();
6003    scan.setReversed(true);
6004    scan.readVersions(5);
6005    try (InternalScanner scanner = region.getScanner(scan)) {
6006      boolean hasNext = scanner.next(currRow);
6007      assertEquals(2, currRow.size());
6008      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6009        currRow.get(0).getRowLength(), rowC, 0, rowC.length));
6010      assertTrue(hasNext);
6011      currRow.clear();
6012      hasNext = scanner.next(currRow);
6013      assertEquals(1, currRow.size());
6014      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6015        currRow.get(0).getRowLength(), rowB, 0, rowB.length));
6016      assertTrue(hasNext);
6017      currRow.clear();
6018      hasNext = scanner.next(currRow);
6019      assertEquals(1, currRow.size());
6020      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6021        currRow.get(0).getRowLength(), rowA, 0, rowA.length));
6022      assertFalse(hasNext);
6023    }
6024  }
6025
6026  @Test
6027  public void testReverseScanner_FromMemStore_SingleCF_FullScan() throws IOException {
6028    byte[] rowC = Bytes.toBytes("rowC");
6029    byte[] rowA = Bytes.toBytes("rowA");
6030    byte[] rowB = Bytes.toBytes("rowB");
6031    byte[] cf = Bytes.toBytes("CF");
6032    byte[][] families = { cf };
6033    byte[] col = Bytes.toBytes("C");
6034    long ts = 1;
6035    this.region = initHRegion(tableName, method, families);
6036    KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
6037    KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, null);
6038    KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
6039    KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
6040    Put put = null;
6041    put = new Put(rowC);
6042    put.add(kv1);
6043    put.add(kv11);
6044    region.put(put);
6045    put = new Put(rowA);
6046    put.add(kv2);
6047    region.put(put);
6048    put = new Put(rowB);
6049    put.add(kv3);
6050    region.put(put);
6051    Scan scan = new Scan();
6052    List<Cell> currRow = new ArrayList<>();
6053    scan.setReversed(true);
6054    try (InternalScanner scanner = region.getScanner(scan)) {
6055      boolean hasNext = scanner.next(currRow);
6056      assertEquals(1, currRow.size());
6057      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6058        currRow.get(0).getRowLength(), rowC, 0, rowC.length));
6059      assertTrue(hasNext);
6060      currRow.clear();
6061      hasNext = scanner.next(currRow);
6062      assertEquals(1, currRow.size());
6063      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6064        currRow.get(0).getRowLength(), rowB, 0, rowB.length));
6065      assertTrue(hasNext);
6066      currRow.clear();
6067      hasNext = scanner.next(currRow);
6068      assertEquals(1, currRow.size());
6069      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6070        currRow.get(0).getRowLength(), rowA, 0, rowA.length));
6071      assertFalse(hasNext);
6072    }
6073  }
6074
6075  @Test
6076  public void testReverseScanner_moreRowsMayExistAfter() throws IOException {
6077    // case for "INCLUDE_AND_SEEK_NEXT_ROW & SEEK_NEXT_ROW" endless loop
6078    byte[] rowA = Bytes.toBytes("rowA");
6079    byte[] rowB = Bytes.toBytes("rowB");
6080    byte[] rowC = Bytes.toBytes("rowC");
6081    byte[] rowD = Bytes.toBytes("rowD");
6082    byte[] rowE = Bytes.toBytes("rowE");
6083    byte[] cf = Bytes.toBytes("CF");
6084    byte[][] families = { cf };
6085    byte[] col1 = Bytes.toBytes("col1");
6086    byte[] col2 = Bytes.toBytes("col2");
6087    long ts = 1;
6088    this.region = initHRegion(tableName, method, families);
6089    KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
6090    KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
6091    KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
6092    KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
6093    KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
6094    KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
6095    Put put = null;
6096    put = new Put(rowA);
6097    put.add(kv1);
6098    region.put(put);
6099    put = new Put(rowB);
6100    put.add(kv2);
6101    region.put(put);
6102    put = new Put(rowC);
6103    put.add(kv3);
6104    region.put(put);
6105    put = new Put(rowD);
6106    put.add(kv4_1);
6107    region.put(put);
6108    put = new Put(rowD);
6109    put.add(kv4_2);
6110    region.put(put);
6111    put = new Put(rowE);
6112    put.add(kv5);
6113    region.put(put);
6114    region.flush(true);
6115    Scan scan = new Scan().withStartRow(rowD).withStopRow(rowA);
6116    scan.addColumn(families[0], col1);
6117    scan.setReversed(true);
6118    List<Cell> currRow = new ArrayList<>();
6119    try (InternalScanner scanner = region.getScanner(scan)) {
6120      boolean hasNext = scanner.next(currRow);
6121      assertEquals(1, currRow.size());
6122      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6123        currRow.get(0).getRowLength(), rowD, 0, rowD.length));
6124      assertTrue(hasNext);
6125      currRow.clear();
6126      hasNext = scanner.next(currRow);
6127      assertEquals(1, currRow.size());
6128      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6129        currRow.get(0).getRowLength(), rowC, 0, rowC.length));
6130      assertTrue(hasNext);
6131      currRow.clear();
6132      hasNext = scanner.next(currRow);
6133      assertEquals(1, currRow.size());
6134      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6135        currRow.get(0).getRowLength(), rowB, 0, rowB.length));
6136      assertFalse(hasNext);
6137    }
6138
6139    scan = new Scan().withStartRow(rowD).withStopRow(rowA);
6140    scan.addColumn(families[0], col2);
6141    scan.setReversed(true);
6142    currRow.clear();
6143    try (InternalScanner scanner = region.getScanner(scan)) {
6144      boolean hasNext = scanner.next(currRow);
6145      assertEquals(1, currRow.size());
6146      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6147        currRow.get(0).getRowLength(), rowD, 0, rowD.length));
6148      assertTrue(hasNext);
6149    }
6150  }
6151
6152  @Test
6153  public void testReverseScanner_smaller_blocksize() throws IOException {
6154    // case to ensure no conflict with HFile index optimization
6155    byte[] rowA = Bytes.toBytes("rowA");
6156    byte[] rowB = Bytes.toBytes("rowB");
6157    byte[] rowC = Bytes.toBytes("rowC");
6158    byte[] rowD = Bytes.toBytes("rowD");
6159    byte[] rowE = Bytes.toBytes("rowE");
6160    byte[] cf = Bytes.toBytes("CF");
6161    byte[][] families = { cf };
6162    byte[] col1 = Bytes.toBytes("col1");
6163    byte[] col2 = Bytes.toBytes("col2");
6164    long ts = 1;
6165    Configuration conf = new Configuration(CONF);
6166    conf.setInt("test.block.size", 1);
6167    this.region = initHRegion(tableName, method, conf, families);
6168    KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
6169    KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
6170    KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
6171    KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
6172    KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
6173    KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
6174    Put put = null;
6175    put = new Put(rowA);
6176    put.add(kv1);
6177    region.put(put);
6178    put = new Put(rowB);
6179    put.add(kv2);
6180    region.put(put);
6181    put = new Put(rowC);
6182    put.add(kv3);
6183    region.put(put);
6184    put = new Put(rowD);
6185    put.add(kv4_1);
6186    region.put(put);
6187    put = new Put(rowD);
6188    put.add(kv4_2);
6189    region.put(put);
6190    put = new Put(rowE);
6191    put.add(kv5);
6192    region.put(put);
6193    region.flush(true);
6194    Scan scan = new Scan().withStartRow(rowD).withStopRow(rowA);
6195    scan.addColumn(families[0], col1);
6196    scan.setReversed(true);
6197    List<Cell> currRow = new ArrayList<>();
6198    try (InternalScanner scanner = region.getScanner(scan)) {
6199      boolean hasNext = scanner.next(currRow);
6200      assertEquals(1, currRow.size());
6201      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6202        currRow.get(0).getRowLength(), rowD, 0, rowD.length));
6203      assertTrue(hasNext);
6204      currRow.clear();
6205      hasNext = scanner.next(currRow);
6206      assertEquals(1, currRow.size());
6207      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6208        currRow.get(0).getRowLength(), rowC, 0, rowC.length));
6209      assertTrue(hasNext);
6210      currRow.clear();
6211      hasNext = scanner.next(currRow);
6212      assertEquals(1, currRow.size());
6213      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6214        currRow.get(0).getRowLength(), rowB, 0, rowB.length));
6215      assertFalse(hasNext);
6216    }
6217
6218    scan = new Scan().withStartRow(rowD).withStopRow(rowA);
6219    scan.addColumn(families[0], col2);
6220    scan.setReversed(true);
6221    currRow.clear();
6222    try (InternalScanner scanner = region.getScanner(scan)) {
6223      boolean hasNext = scanner.next(currRow);
6224      assertEquals(1, currRow.size());
6225      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6226        currRow.get(0).getRowLength(), rowD, 0, rowD.length));
6227      assertTrue(hasNext);
6228    }
6229  }
6230
6231  @Test
6232  public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1() throws IOException {
6233    byte[] row0 = Bytes.toBytes("row0"); // 1 kv
6234    byte[] row1 = Bytes.toBytes("row1"); // 2 kv
6235    byte[] row2 = Bytes.toBytes("row2"); // 4 kv
6236    byte[] row3 = Bytes.toBytes("row3"); // 2 kv
6237    byte[] row4 = Bytes.toBytes("row4"); // 5 kv
6238    byte[] row5 = Bytes.toBytes("row5"); // 2 kv
6239    byte[] cf1 = Bytes.toBytes("CF1");
6240    byte[] cf2 = Bytes.toBytes("CF2");
6241    byte[] cf3 = Bytes.toBytes("CF3");
6242    byte[][] families = { cf1, cf2, cf3 };
6243    byte[] col = Bytes.toBytes("C");
6244    long ts = 1;
6245    Configuration conf = new Configuration(CONF);
6246    // disable compactions in this test.
6247    conf.setInt("hbase.hstore.compactionThreshold", 10000);
6248    this.region = initHRegion(tableName, method, conf, families);
6249    // kv naming style: kv(row number) totalKvCountInThisRow seq no
6250    KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put, null);
6251    KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put, null);
6252    KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1, KeyValue.Type.Put, null);
6253    KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
6254    KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put, null);
6255    KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put, null);
6256    KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4, KeyValue.Type.Put, null);
6257    KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put, null);
6258    KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4, KeyValue.Type.Put, null);
6259    KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put, null);
6260    KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put, null);
6261    KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5, KeyValue.Type.Put, null);
6262    KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put, null);
6263    KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3, KeyValue.Type.Put, null);
6264    KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put, null);
6265    KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put, null);
6266    // hfiles(cf1/cf2) :"row1"(1 kv) / "row2"(1 kv) / "row4"(2 kv)
6267    Put put = null;
6268    put = new Put(row1);
6269    put.add(kv1_2_1);
6270    region.put(put);
6271    put = new Put(row2);
6272    put.add(kv2_4_1);
6273    region.put(put);
6274    put = new Put(row4);
6275    put.add(kv4_5_4);
6276    put.add(kv4_5_5);
6277    region.put(put);
6278    region.flush(true);
6279    // hfiles(cf1/cf3) : "row1" (1 kvs) / "row2" (1 kv) / "row4" (2 kv)
6280    put = new Put(row4);
6281    put.add(kv4_5_1);
6282    put.add(kv4_5_3);
6283    region.put(put);
6284    put = new Put(row1);
6285    put.add(kv1_2_2);
6286    region.put(put);
6287    put = new Put(row2);
6288    put.add(kv2_4_4);
6289    region.put(put);
6290    region.flush(true);
6291    // hfiles(cf1/cf3) : "row2"(2 kv) / "row3"(1 kvs) / "row4" (1 kv)
6292    put = new Put(row4);
6293    put.add(kv4_5_2);
6294    region.put(put);
6295    put = new Put(row2);
6296    put.add(kv2_4_2);
6297    put.add(kv2_4_3);
6298    region.put(put);
6299    put = new Put(row3);
6300    put.add(kv3_2_2);
6301    region.put(put);
6302    region.flush(true);
6303    // memstore(cf1/cf2/cf3) : "row0" (1 kvs) / "row3" ( 1 kv) / "row5" (max)
6304    // ( 2 kv)
6305    put = new Put(row0);
6306    put.add(kv0_1_1);
6307    region.put(put);
6308    put = new Put(row3);
6309    put.add(kv3_2_1);
6310    region.put(put);
6311    put = new Put(row5);
6312    put.add(kv5_2_1);
6313    put.add(kv5_2_2);
6314    region.put(put);
6315    // scan range = ["row4", min), skip the max "row5"
6316    Scan scan = new Scan().withStartRow(row4);
6317    scan.readVersions(5);
6318    scan.setBatch(3);
6319    scan.setReversed(true);
6320    try (InternalScanner scanner = region.getScanner(scan)) {
6321      List<Cell> currRow = new ArrayList<>();
6322      boolean hasNext = false;
6323      // 1. scan out "row4" (5 kvs), "row5" can't be scanned out since not
6324      // included in scan range
6325      // "row4" takes 2 next() calls since batch=3
6326      hasNext = scanner.next(currRow);
6327      assertEquals(3, currRow.size());
6328      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6329        currRow.get(0).getRowLength(), row4, 0, row4.length));
6330      assertTrue(hasNext);
6331      currRow.clear();
6332      hasNext = scanner.next(currRow);
6333      assertEquals(2, currRow.size());
6334      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6335        currRow.get(0).getRowLength(), row4, 0, row4.length));
6336      assertTrue(hasNext);
6337      // 2. scan out "row3" (2 kv)
6338      currRow.clear();
6339      hasNext = scanner.next(currRow);
6340      assertEquals(2, currRow.size());
6341      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6342        currRow.get(0).getRowLength(), row3, 0, row3.length));
6343      assertTrue(hasNext);
6344      // 3. scan out "row2" (4 kvs)
6345      // "row2" takes 2 next() calls since batch=3
6346      currRow.clear();
6347      hasNext = scanner.next(currRow);
6348      assertEquals(3, currRow.size());
6349      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6350        currRow.get(0).getRowLength(), row2, 0, row2.length));
6351      assertTrue(hasNext);
6352      currRow.clear();
6353      hasNext = scanner.next(currRow);
6354      assertEquals(1, currRow.size());
6355      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6356        currRow.get(0).getRowLength(), row2, 0, row2.length));
6357      assertTrue(hasNext);
6358      // 4. scan out "row1" (2 kv)
6359      currRow.clear();
6360      hasNext = scanner.next(currRow);
6361      assertEquals(2, currRow.size());
6362      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6363        currRow.get(0).getRowLength(), row1, 0, row1.length));
6364      assertTrue(hasNext);
6365      // 5. scan out "row0" (1 kv)
6366      currRow.clear();
6367      hasNext = scanner.next(currRow);
6368      assertEquals(1, currRow.size());
6369      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6370        currRow.get(0).getRowLength(), row0, 0, row0.length));
6371      assertFalse(hasNext);
6372    }
6373  }
6374
6375  @Test
6376  public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2() throws IOException {
6377    byte[] row1 = Bytes.toBytes("row1");
6378    byte[] row2 = Bytes.toBytes("row2");
6379    byte[] row3 = Bytes.toBytes("row3");
6380    byte[] row4 = Bytes.toBytes("row4");
6381    byte[] cf1 = Bytes.toBytes("CF1");
6382    byte[] cf2 = Bytes.toBytes("CF2");
6383    byte[] cf3 = Bytes.toBytes("CF3");
6384    byte[] cf4 = Bytes.toBytes("CF4");
6385    byte[][] families = { cf1, cf2, cf3, cf4 };
6386    byte[] col = Bytes.toBytes("C");
6387    long ts = 1;
6388    Configuration conf = new Configuration(CONF);
6389    // disable compactions in this test.
6390    conf.setInt("hbase.hstore.compactionThreshold", 10000);
6391    this.region = initHRegion(tableName, method, conf, families);
6392    KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null);
6393    KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
6394    KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null);
6395    KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null);
6396    // storefile1
6397    Put put = new Put(row1);
6398    put.add(kv1);
6399    region.put(put);
6400    region.flush(true);
6401    // storefile2
6402    put = new Put(row2);
6403    put.add(kv2);
6404    region.put(put);
6405    region.flush(true);
6406    // storefile3
6407    put = new Put(row3);
6408    put.add(kv3);
6409    region.put(put);
6410    region.flush(true);
6411    // memstore
6412    put = new Put(row4);
6413    put.add(kv4);
6414    region.put(put);
6415    // scan range = ["row4", min)
6416    Scan scan = new Scan().withStartRow(row4);
6417    scan.setReversed(true);
6418    scan.setBatch(10);
6419    try (InternalScanner scanner = region.getScanner(scan)) {
6420      List<Cell> currRow = new ArrayList<>();
6421      boolean hasNext = scanner.next(currRow);
6422      assertEquals(1, currRow.size());
6423      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6424        currRow.get(0).getRowLength(), row4, 0, row4.length));
6425      assertTrue(hasNext);
6426      currRow.clear();
6427      hasNext = scanner.next(currRow);
6428      assertEquals(1, currRow.size());
6429      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6430        currRow.get(0).getRowLength(), row3, 0, row3.length));
6431      assertTrue(hasNext);
6432      currRow.clear();
6433      hasNext = scanner.next(currRow);
6434      assertEquals(1, currRow.size());
6435      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6436        currRow.get(0).getRowLength(), row2, 0, row2.length));
6437      assertTrue(hasNext);
6438      currRow.clear();
6439      hasNext = scanner.next(currRow);
6440      assertEquals(1, currRow.size());
6441      assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6442        currRow.get(0).getRowLength(), row1, 0, row1.length));
6443      assertFalse(hasNext);
6444    }
6445  }
6446
6447  /**
6448   * Test for HBASE-14497: Reverse Scan threw StackOverflow caused by readPt checking
6449   */
6450  @Test
6451  public void testReverseScanner_StackOverflow() throws IOException {
6452    byte[] cf1 = Bytes.toBytes("CF1");
6453    byte[][] families = { cf1 };
6454    byte[] col = Bytes.toBytes("C");
6455    Configuration conf = new Configuration(CONF);
6456    this.region = initHRegion(tableName, method, conf, families);
6457    // setup with one storefile and one memstore, to create scanner and get an earlier readPt
6458    Put put = new Put(Bytes.toBytes("19998"));
6459    put.addColumn(cf1, col, Bytes.toBytes("val"));
6460    region.put(put);
6461    region.flushcache(true, true, FlushLifeCycleTracker.DUMMY);
6462    Put put2 = new Put(Bytes.toBytes("19997"));
6463    put2.addColumn(cf1, col, Bytes.toBytes("val"));
6464    region.put(put2);
6465
6466    Scan scan = new Scan().withStartRow(Bytes.toBytes("19998"));
6467    scan.setReversed(true);
6468    try (InternalScanner scanner = region.getScanner(scan)) {
6469      // create one storefile contains many rows will be skipped
6470      // to check StoreFileScanner.seekToPreviousRow
6471      for (int i = 10000; i < 20000; i++) {
6472        Put p = new Put(Bytes.toBytes("" + i));
6473        p.addColumn(cf1, col, Bytes.toBytes("" + i));
6474        region.put(p);
6475      }
6476      region.flushcache(true, true, FlushLifeCycleTracker.DUMMY);
6477
6478      // create one memstore contains many rows will be skipped
6479      // to check MemStoreScanner.seekToPreviousRow
6480      for (int i = 10000; i < 20000; i++) {
6481        Put p = new Put(Bytes.toBytes("" + i));
6482        p.addColumn(cf1, col, Bytes.toBytes("" + i));
6483        region.put(p);
6484      }
6485
6486      List<Cell> currRow = new ArrayList<>();
6487      boolean hasNext;
6488      do {
6489        hasNext = scanner.next(currRow);
6490      } while (hasNext);
6491      assertEquals(2, currRow.size());
6492      assertEquals("19998", Bytes.toString(currRow.get(0).getRowArray(),
6493        currRow.get(0).getRowOffset(), currRow.get(0).getRowLength()));
6494      assertEquals("19997", Bytes.toString(currRow.get(1).getRowArray(),
6495        currRow.get(1).getRowOffset(), currRow.get(1).getRowLength()));
6496    }
6497  }
6498
6499  @Test
6500  public void testReverseScanShouldNotScanMemstoreIfReadPtLesser() throws Exception {
6501    byte[] cf1 = Bytes.toBytes("CF1");
6502    byte[][] families = { cf1 };
6503    byte[] col = Bytes.toBytes("C");
6504    this.region = initHRegion(tableName, method, CONF, families);
6505    // setup with one storefile and one memstore, to create scanner and get an earlier readPt
6506    Put put = new Put(Bytes.toBytes("19996"));
6507    put.addColumn(cf1, col, Bytes.toBytes("val"));
6508    region.put(put);
6509    Put put2 = new Put(Bytes.toBytes("19995"));
6510    put2.addColumn(cf1, col, Bytes.toBytes("val"));
6511    region.put(put2);
6512    // create a reverse scan
6513    Scan scan = new Scan().withStartRow(Bytes.toBytes("19996"));
6514    scan.setReversed(true);
6515    try (RegionScannerImpl scanner = region.getScanner(scan)) {
6516      // flush the cache. This will reset the store scanner
6517      region.flushcache(true, true, FlushLifeCycleTracker.DUMMY);
6518
6519      // create one memstore contains many rows will be skipped
6520      // to check MemStoreScanner.seekToPreviousRow
6521      for (int i = 10000; i < 20000; i++) {
6522        Put p = new Put(Bytes.toBytes("" + i));
6523        p.addColumn(cf1, col, Bytes.toBytes("" + i));
6524        region.put(p);
6525      }
6526      List<Cell> currRow = new ArrayList<>();
6527      boolean hasNext;
6528      boolean assertDone = false;
6529      do {
6530        hasNext = scanner.next(currRow);
6531        // With HBASE-15871, after the scanner is reset the memstore scanner should not be
6532        // added here
6533        if (!assertDone) {
6534          StoreScanner current = (StoreScanner) (scanner.storeHeap).getCurrentForTesting();
6535          List<KeyValueScanner> scanners = current.getAllScannersForTesting();
6536          assertEquals("There should be only one scanner the store file scanner", 1,
6537            scanners.size());
6538          assertDone = true;
6539        }
6540      } while (hasNext);
6541      assertEquals(2, currRow.size());
6542      assertEquals("19996", Bytes.toString(currRow.get(0).getRowArray(),
6543        currRow.get(0).getRowOffset(), currRow.get(0).getRowLength()));
6544      assertEquals("19995", Bytes.toString(currRow.get(1).getRowArray(),
6545        currRow.get(1).getRowOffset(), currRow.get(1).getRowLength()));
6546    }
6547  }
6548
6549  @Test
6550  public void testReverseScanWhenPutCellsAfterOpenReverseScan() throws Exception {
6551    byte[] cf1 = Bytes.toBytes("CF1");
6552    byte[][] families = { cf1 };
6553    byte[] col = Bytes.toBytes("C");
6554
6555    this.region = initHRegion(tableName, method, CONF, families);
6556
6557    Put put = new Put(Bytes.toBytes("199996"));
6558    put.addColumn(cf1, col, Bytes.toBytes("val"));
6559    region.put(put);
6560    Put put2 = new Put(Bytes.toBytes("199995"));
6561    put2.addColumn(cf1, col, Bytes.toBytes("val"));
6562    region.put(put2);
6563
6564    // Create a reverse scan
6565    Scan scan = new Scan().withStartRow(Bytes.toBytes("199996"));
6566    scan.setReversed(true);
6567    try (RegionScannerImpl scanner = region.getScanner(scan)) {
6568      // Put a lot of cells that have sequenceIDs grater than the readPt of the reverse scan
6569      for (int i = 100000; i < 200000; i++) {
6570        Put p = new Put(Bytes.toBytes("" + i));
6571        p.addColumn(cf1, col, Bytes.toBytes("" + i));
6572        region.put(p);
6573      }
6574      List<Cell> currRow = new ArrayList<>();
6575      boolean hasNext;
6576      do {
6577        hasNext = scanner.next(currRow);
6578      } while (hasNext);
6579
6580      assertEquals(2, currRow.size());
6581      assertEquals("199996", Bytes.toString(currRow.get(0).getRowArray(),
6582        currRow.get(0).getRowOffset(), currRow.get(0).getRowLength()));
6583      assertEquals("199995", Bytes.toString(currRow.get(1).getRowArray(),
6584        currRow.get(1).getRowOffset(), currRow.get(1).getRowLength()));
6585    }
6586  }
6587
6588  @Test
6589  public void testWriteRequestsCounter() throws IOException {
6590    byte[] fam = Bytes.toBytes("info");
6591    byte[][] families = { fam };
6592    this.region = initHRegion(tableName, method, CONF, families);
6593
6594    Assert.assertEquals(0L, region.getWriteRequestsCount());
6595
6596    Put put = new Put(row);
6597    put.addColumn(fam, fam, fam);
6598
6599    Assert.assertEquals(0L, region.getWriteRequestsCount());
6600    region.put(put);
6601    Assert.assertEquals(1L, region.getWriteRequestsCount());
6602    region.put(put);
6603    Assert.assertEquals(2L, region.getWriteRequestsCount());
6604    region.put(put);
6605    Assert.assertEquals(3L, region.getWriteRequestsCount());
6606
6607    region.delete(new Delete(row));
6608    Assert.assertEquals(4L, region.getWriteRequestsCount());
6609  }
6610
6611  @Test
6612  public void testOpenRegionWrittenToWAL() throws Exception {
6613    final ServerName serverName = ServerName.valueOf(name.getMethodName(), 100, 42);
6614    final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName));
6615
6616    TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
6617      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1))
6618      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam2)).build();
6619    RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
6620
6621    // open the region w/o rss and wal and flush some files
6622    region = HBaseTestingUtil.createRegionAndWAL(hri, TEST_UTIL.getDataTestDir(),
6623      TEST_UTIL.getConfiguration(), htd);
6624    assertNotNull(region);
6625
6626    // create a file in fam1 for the region before opening in OpenRegionHandler
6627    region.put(new Put(Bytes.toBytes("a")).addColumn(fam1, fam1, fam1));
6628    region.flush(true);
6629    HBaseTestingUtil.closeRegionAndWAL(region);
6630
6631    ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class);
6632
6633    // capture append() calls
6634    WAL wal = mockWAL();
6635    when(rss.getWAL(any(RegionInfo.class))).thenReturn(wal);
6636
6637    region =
6638      HRegion.openHRegion(hri, htd, rss.getWAL(hri), TEST_UTIL.getConfiguration(), rss, null);
6639
6640    verify(wal, times(1)).appendMarker(any(RegionInfo.class), any(WALKeyImpl.class),
6641      editCaptor.capture());
6642
6643    WALEdit edit = editCaptor.getValue();
6644    assertNotNull(edit);
6645    assertNotNull(edit.getCells());
6646    assertEquals(1, edit.getCells().size());
6647    RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0));
6648    assertNotNull(desc);
6649
6650    LOG.info("RegionEventDescriptor from WAL: " + desc);
6651
6652    assertEquals(RegionEventDescriptor.EventType.REGION_OPEN, desc.getEventType());
6653    assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getTableName().toBytes()));
6654    assertTrue(
6655      Bytes.equals(desc.getEncodedRegionName().toByteArray(), hri.getEncodedNameAsBytes()));
6656    assertTrue(desc.getLogSequenceNumber() > 0);
6657    assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer()));
6658    assertEquals(2, desc.getStoresCount());
6659
6660    StoreDescriptor store = desc.getStores(0);
6661    assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1));
6662    assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1));
6663    assertEquals(1, store.getStoreFileCount()); // 1store file
6664    assertFalse(store.getStoreFile(0).contains("/")); // ensure path is relative
6665
6666    store = desc.getStores(1);
6667    assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2));
6668    assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2));
6669    assertEquals(0, store.getStoreFileCount()); // no store files
6670  }
6671
6672  // Helper for test testOpenRegionWrittenToWALForLogReplay
6673  static class HRegionWithSeqId extends HRegion {
6674    public HRegionWithSeqId(final Path tableDir, final WAL wal, final FileSystem fs,
6675      final Configuration confParam, final RegionInfo regionInfo, final TableDescriptor htd,
6676      final RegionServerServices rsServices) {
6677      super(tableDir, wal, fs, confParam, regionInfo, htd, rsServices);
6678    }
6679
6680    @Override
6681    protected long getNextSequenceId(WAL wal) throws IOException {
6682      return 42;
6683    }
6684  }
6685
6686  @Test
6687  public void testFlushedFileWithNoTags() throws Exception {
6688    final TableName tableName = TableName.valueOf(name.getMethodName());
6689    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
6690      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)).build();
6691    RegionInfo info = RegionInfoBuilder.newBuilder(tableName).build();
6692    Path path = TEST_UTIL.getDataTestDir(getClass().getSimpleName());
6693    region = HBaseTestingUtil.createRegionAndWAL(info, path, TEST_UTIL.getConfiguration(),
6694      tableDescriptor);
6695    Put put = new Put(Bytes.toBytes("a-b-0-0"));
6696    put.addColumn(fam1, qual1, Bytes.toBytes("c1-value"));
6697    region.put(put);
6698    region.flush(true);
6699    HStore store = region.getStore(fam1);
6700    Collection<HStoreFile> storefiles = store.getStorefiles();
6701    for (HStoreFile sf : storefiles) {
6702      assertFalse("Tags should not be present ",
6703        sf.getReader().getHFileReader().getFileContext().isIncludesTags());
6704    }
6705  }
6706
6707  /**
6708   * Utility method to setup a WAL mock.
6709   * <p/>
6710   * Needs to do the bit where we close latch on the WALKeyImpl on append else test hangs.
6711   * @return a mock WAL
6712   */
6713  private WAL mockWAL() throws IOException {
6714    WAL wal = mock(WAL.class);
6715    when(wal.appendData(any(RegionInfo.class), any(WALKeyImpl.class), any(WALEdit.class)))
6716      .thenAnswer(new Answer<Long>() {
6717        @Override
6718        public Long answer(InvocationOnMock invocation) throws Throwable {
6719          WALKeyImpl key = invocation.getArgument(1);
6720          MultiVersionConcurrencyControl.WriteEntry we = key.getMvcc().begin();
6721          key.setWriteEntry(we);
6722          return 1L;
6723        }
6724      });
6725    when(wal.appendMarker(any(RegionInfo.class), any(WALKeyImpl.class), any(WALEdit.class)))
6726      .thenAnswer(new Answer<Long>() {
6727        @Override
6728        public Long answer(InvocationOnMock invocation) throws Throwable {
6729          WALKeyImpl key = invocation.getArgument(1);
6730          MultiVersionConcurrencyControl.WriteEntry we = key.getMvcc().begin();
6731          key.setWriteEntry(we);
6732          return 1L;
6733        }
6734      });
6735    return wal;
6736  }
6737
6738  @Test
6739  public void testCloseRegionWrittenToWAL() throws Exception {
6740    Path rootDir = new Path(dir + name.getMethodName());
6741    CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
6742
6743    final ServerName serverName = ServerName.valueOf("testCloseRegionWrittenToWAL", 100, 42);
6744    final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName));
6745
6746    TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
6747      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1))
6748      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam2)).build();
6749    RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
6750
6751    ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class);
6752
6753    // capture append() calls
6754    WAL wal = mockWAL();
6755    when(rss.getWAL(any(RegionInfo.class))).thenReturn(wal);
6756
6757    // create the region
6758    region = HBaseTestingUtil.createRegionAndWAL(hri, rootDir, CONF, htd);
6759    HBaseTestingUtil.closeRegionAndWAL(region);
6760    region = null;
6761    // open the region first and then close it
6762    HRegion.openHRegion(hri, htd, rss.getWAL(hri), TEST_UTIL.getConfiguration(), rss, null).close();
6763
6764    // 2 times, one for region open, the other close region
6765    verify(wal, times(2)).appendMarker(any(RegionInfo.class), (WALKeyImpl) any(WALKeyImpl.class),
6766      editCaptor.capture());
6767
6768    WALEdit edit = editCaptor.getAllValues().get(1);
6769    assertNotNull(edit);
6770    assertNotNull(edit.getCells());
6771    assertEquals(1, edit.getCells().size());
6772    RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0));
6773    assertNotNull(desc);
6774
6775    LOG.info("RegionEventDescriptor from WAL: " + desc);
6776
6777    assertEquals(RegionEventDescriptor.EventType.REGION_CLOSE, desc.getEventType());
6778    assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getTableName().toBytes()));
6779    assertTrue(
6780      Bytes.equals(desc.getEncodedRegionName().toByteArray(), hri.getEncodedNameAsBytes()));
6781    assertTrue(desc.getLogSequenceNumber() > 0);
6782    assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer()));
6783    assertEquals(2, desc.getStoresCount());
6784
6785    StoreDescriptor store = desc.getStores(0);
6786    assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1));
6787    assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1));
6788    assertEquals(0, store.getStoreFileCount()); // no store files
6789
6790    store = desc.getStores(1);
6791    assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2));
6792    assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2));
6793    assertEquals(0, store.getStoreFileCount()); // no store files
6794  }
6795
6796  /**
6797   * Test RegionTooBusyException thrown when region is busy
6798   */
6799  @Test
6800  public void testRegionTooBusy() throws IOException {
6801    byte[] family = Bytes.toBytes("family");
6802    long defaultBusyWaitDuration =
6803      CONF.getLong("hbase.busy.wait.duration", HRegion.DEFAULT_BUSY_WAIT_DURATION);
6804    CONF.setLong("hbase.busy.wait.duration", 1000);
6805    region = initHRegion(tableName, method, CONF, family);
6806    final AtomicBoolean stopped = new AtomicBoolean(true);
6807    Thread t = new Thread(new Runnable() {
6808      @Override
6809      public void run() {
6810        try {
6811          region.lock.writeLock().lock();
6812          stopped.set(false);
6813          while (!stopped.get()) {
6814            Thread.sleep(100);
6815          }
6816        } catch (InterruptedException ie) {
6817        } finally {
6818          region.lock.writeLock().unlock();
6819        }
6820      }
6821    });
6822    t.start();
6823    Get get = new Get(row);
6824    try {
6825      while (stopped.get()) {
6826        Thread.sleep(100);
6827      }
6828      region.get(get);
6829      fail("Should throw RegionTooBusyException");
6830    } catch (InterruptedException ie) {
6831      fail("test interrupted");
6832    } catch (RegionTooBusyException e) {
6833      // Good, expected
6834    } finally {
6835      stopped.set(true);
6836      try {
6837        t.join();
6838      } catch (Throwable e) {
6839      }
6840
6841      HBaseTestingUtil.closeRegionAndWAL(region);
6842      region = null;
6843      CONF.setLong("hbase.busy.wait.duration", defaultBusyWaitDuration);
6844    }
6845  }
6846
6847  @Test
6848  public void testCellTTLs() throws IOException {
6849    IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
6850    EnvironmentEdgeManager.injectEdge(edge);
6851
6852    final byte[] row = Bytes.toBytes("testRow");
6853    final byte[] q1 = Bytes.toBytes("q1");
6854    final byte[] q2 = Bytes.toBytes("q2");
6855    final byte[] q3 = Bytes.toBytes("q3");
6856    final byte[] q4 = Bytes.toBytes("q4");
6857
6858    // 10 seconds
6859    TableDescriptor tableDescriptor =
6860      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
6861        .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam1).setTimeToLive(10).build())
6862        .build();
6863
6864    Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
6865    conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS);
6866
6867    region = HBaseTestingUtil.createRegionAndWAL(
6868      RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(),
6869      TEST_UTIL.getDataTestDir(), conf, tableDescriptor);
6870    assertNotNull(region);
6871    long now = EnvironmentEdgeManager.currentTime();
6872    // Add a cell that will expire in 5 seconds via cell TTL
6873    region.put(new Put(row).add(new KeyValue(row, fam1, q1, now, HConstants.EMPTY_BYTE_ARRAY,
6874      new ArrayBackedTag[] {
6875        // TTL tags specify ts in milliseconds
6876        new ArrayBackedTag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) })));
6877    // Add a cell that will expire after 10 seconds via family setting
6878    region.put(new Put(row).addColumn(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY));
6879    // Add a cell that will expire in 15 seconds via cell TTL
6880    region.put(new Put(row).add(new KeyValue(row, fam1, q3, now + 10000 - 1,
6881      HConstants.EMPTY_BYTE_ARRAY, new ArrayBackedTag[] {
6882        // TTL tags specify ts in milliseconds
6883        new ArrayBackedTag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) })));
6884    // Add a cell that will expire in 20 seconds via family setting
6885    region.put(new Put(row).addColumn(fam1, q4, now + 10000 - 1, HConstants.EMPTY_BYTE_ARRAY));
6886
6887    // Flush so we are sure store scanning gets this right
6888    region.flush(true);
6889
6890    // A query at time T+0 should return all cells
6891    Result r = region.get(new Get(row));
6892    assertNotNull(r.getValue(fam1, q1));
6893    assertNotNull(r.getValue(fam1, q2));
6894    assertNotNull(r.getValue(fam1, q3));
6895    assertNotNull(r.getValue(fam1, q4));
6896
6897    // Increment time to T+5 seconds
6898    edge.incrementTime(5000);
6899
6900    r = region.get(new Get(row));
6901    assertNull(r.getValue(fam1, q1));
6902    assertNotNull(r.getValue(fam1, q2));
6903    assertNotNull(r.getValue(fam1, q3));
6904    assertNotNull(r.getValue(fam1, q4));
6905
6906    // Increment time to T+10 seconds
6907    edge.incrementTime(5000);
6908
6909    r = region.get(new Get(row));
6910    assertNull(r.getValue(fam1, q1));
6911    assertNull(r.getValue(fam1, q2));
6912    assertNotNull(r.getValue(fam1, q3));
6913    assertNotNull(r.getValue(fam1, q4));
6914
6915    // Increment time to T+15 seconds
6916    edge.incrementTime(5000);
6917
6918    r = region.get(new Get(row));
6919    assertNull(r.getValue(fam1, q1));
6920    assertNull(r.getValue(fam1, q2));
6921    assertNull(r.getValue(fam1, q3));
6922    assertNotNull(r.getValue(fam1, q4));
6923
6924    // Increment time to T+20 seconds
6925    edge.incrementTime(10000);
6926
6927    r = region.get(new Get(row));
6928    assertNull(r.getValue(fam1, q1));
6929    assertNull(r.getValue(fam1, q2));
6930    assertNull(r.getValue(fam1, q3));
6931    assertNull(r.getValue(fam1, q4));
6932
6933    // Fun with disappearing increments
6934
6935    // Start at 1
6936    region.put(new Put(row).addColumn(fam1, q1, Bytes.toBytes(1L)));
6937    r = region.get(new Get(row));
6938    byte[] val = r.getValue(fam1, q1);
6939    assertNotNull(val);
6940    assertEquals(1L, Bytes.toLong(val));
6941
6942    // Increment with a TTL of 5 seconds
6943    Increment incr = new Increment(row).addColumn(fam1, q1, 1L);
6944    incr.setTTL(5000);
6945    region.increment(incr); // 2
6946
6947    // New value should be 2
6948    r = region.get(new Get(row));
6949    val = r.getValue(fam1, q1);
6950    assertNotNull(val);
6951    assertEquals(2L, Bytes.toLong(val));
6952
6953    // Increment time to T+25 seconds
6954    edge.incrementTime(5000);
6955
6956    // Value should be back to 1
6957    r = region.get(new Get(row));
6958    val = r.getValue(fam1, q1);
6959    assertNotNull(val);
6960    assertEquals(1L, Bytes.toLong(val));
6961
6962    // Increment time to T+30 seconds
6963    edge.incrementTime(5000);
6964
6965    // Original value written at T+20 should be gone now via family TTL
6966    r = region.get(new Get(row));
6967    assertNull(r.getValue(fam1, q1));
6968  }
6969
6970  @Test
6971  public void testTTLsUsingSmallHeartBeatCells() throws IOException {
6972    IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
6973    EnvironmentEdgeManager.injectEdge(edge);
6974
6975    final byte[] row = Bytes.toBytes("testRow");
6976    final byte[] q1 = Bytes.toBytes("q1");
6977    final byte[] q2 = Bytes.toBytes("q2");
6978    final byte[] q3 = Bytes.toBytes("q3");
6979    final byte[] q4 = Bytes.toBytes("q4");
6980    final byte[] q5 = Bytes.toBytes("q5");
6981    final byte[] q6 = Bytes.toBytes("q6");
6982    final byte[] q7 = Bytes.toBytes("q7");
6983    final byte[] q8 = Bytes.toBytes("q8");
6984
6985    // 10 seconds
6986    int ttlSecs = 10;
6987    TableDescriptor tableDescriptor =
6988      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(
6989        ColumnFamilyDescriptorBuilder.newBuilder(fam1).setTimeToLive(ttlSecs).build()).build();
6990
6991    Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
6992    conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS);
6993    // using small heart beat cells
6994    conf.setLong(StoreScanner.HBASE_CELLS_SCANNED_PER_HEARTBEAT_CHECK, 2);
6995
6996    region = HBaseTestingUtil.createRegionAndWAL(
6997      RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(),
6998      TEST_UTIL.getDataTestDir(), conf, tableDescriptor);
6999    assertNotNull(region);
7000    long now = EnvironmentEdgeManager.currentTime();
7001    // Add a cell that will expire in 5 seconds via cell TTL
7002    region.put(new Put(row).addColumn(fam1, q1, now, HConstants.EMPTY_BYTE_ARRAY));
7003    region.put(new Put(row).addColumn(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY));
7004    region.put(new Put(row).addColumn(fam1, q3, now, HConstants.EMPTY_BYTE_ARRAY));
7005    // Add a cell that will expire after 10 seconds via family setting
7006    region
7007      .put(new Put(row).addColumn(fam1, q4, now + ttlSecs * 1000 + 1, HConstants.EMPTY_BYTE_ARRAY));
7008    region
7009      .put(new Put(row).addColumn(fam1, q5, now + ttlSecs * 1000 + 1, HConstants.EMPTY_BYTE_ARRAY));
7010
7011    region.put(new Put(row).addColumn(fam1, q6, now, HConstants.EMPTY_BYTE_ARRAY));
7012    region.put(new Put(row).addColumn(fam1, q7, now, HConstants.EMPTY_BYTE_ARRAY));
7013    region
7014      .put(new Put(row).addColumn(fam1, q8, now + ttlSecs * 1000 + 1, HConstants.EMPTY_BYTE_ARRAY));
7015
7016    // Flush so we are sure store scanning gets this right
7017    region.flush(true);
7018
7019    // A query at time T+0 should return all cells
7020    checkScan(8);
7021    region.delete(new Delete(row).addColumn(fam1, q8));
7022
7023    // Increment time to T+ttlSecs seconds
7024    edge.incrementTime(ttlSecs * 1000);
7025    checkScan(2);
7026  }
7027
7028  private void checkScan(int expectCellSize) throws IOException {
7029    Scan s = new Scan().withStartRow(row);
7030    ScannerContext.Builder contextBuilder = ScannerContext.newBuilder(true);
7031    ScannerContext scannerContext = contextBuilder.build();
7032    try (RegionScanner scanner = region.getScanner(s)) {
7033      List<Cell> kvs = new ArrayList<>();
7034      scanner.next(kvs, scannerContext);
7035      assertEquals(expectCellSize, kvs.size());
7036    }
7037  }
7038
7039  @Test
7040  public void testIncrementTimestampsAreMonotonic() throws IOException {
7041    region = initHRegion(tableName, method, CONF, fam1);
7042    ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
7043    EnvironmentEdgeManager.injectEdge(edge);
7044
7045    edge.setValue(10);
7046    Increment inc = new Increment(row);
7047    inc.setDurability(Durability.SKIP_WAL);
7048    inc.addColumn(fam1, qual1, 1L);
7049    region.increment(inc);
7050
7051    Result result = region.get(new Get(row));
7052    Cell c = result.getColumnLatestCell(fam1, qual1);
7053    assertNotNull(c);
7054    assertEquals(10L, c.getTimestamp());
7055
7056    edge.setValue(1); // clock goes back
7057    region.increment(inc);
7058    result = region.get(new Get(row));
7059    c = result.getColumnLatestCell(fam1, qual1);
7060    assertEquals(11L, c.getTimestamp());
7061    assertEquals(2L, Bytes.toLong(c.getValueArray(), c.getValueOffset(), c.getValueLength()));
7062  }
7063
7064  @Test
7065  public void testAppendTimestampsAreMonotonic() throws IOException {
7066    region = initHRegion(tableName, method, CONF, fam1);
7067    ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
7068    EnvironmentEdgeManager.injectEdge(edge);
7069
7070    edge.setValue(10);
7071    Append a = new Append(row);
7072    a.setDurability(Durability.SKIP_WAL);
7073    a.addColumn(fam1, qual1, qual1);
7074    region.append(a);
7075
7076    Result result = region.get(new Get(row));
7077    Cell c = result.getColumnLatestCell(fam1, qual1);
7078    assertNotNull(c);
7079    assertEquals(10L, c.getTimestamp());
7080
7081    edge.setValue(1); // clock goes back
7082    region.append(a);
7083    result = region.get(new Get(row));
7084    c = result.getColumnLatestCell(fam1, qual1);
7085    assertEquals(11L, c.getTimestamp());
7086
7087    byte[] expected = new byte[qual1.length * 2];
7088    System.arraycopy(qual1, 0, expected, 0, qual1.length);
7089    System.arraycopy(qual1, 0, expected, qual1.length, qual1.length);
7090
7091    assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(), expected, 0,
7092      expected.length));
7093  }
7094
7095  @Test
7096  public void testCheckAndMutateTimestampsAreMonotonic() throws IOException {
7097    region = initHRegion(tableName, method, CONF, fam1);
7098    ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
7099    EnvironmentEdgeManager.injectEdge(edge);
7100
7101    edge.setValue(10);
7102    Put p = new Put(row);
7103    p.setDurability(Durability.SKIP_WAL);
7104    p.addColumn(fam1, qual1, qual1);
7105    region.put(p);
7106
7107    Result result = region.get(new Get(row));
7108    Cell c = result.getColumnLatestCell(fam1, qual1);
7109    assertNotNull(c);
7110    assertEquals(10L, c.getTimestamp());
7111
7112    edge.setValue(1); // clock goes back
7113    p = new Put(row);
7114    p.setDurability(Durability.SKIP_WAL);
7115    p.addColumn(fam1, qual1, qual2);
7116    region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(qual1), p);
7117    result = region.get(new Get(row));
7118    c = result.getColumnLatestCell(fam1, qual1);
7119    assertEquals(10L, c.getTimestamp());
7120
7121    assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(), qual2, 0,
7122      qual2.length));
7123  }
7124
7125  @Test
7126  public void testBatchMutateWithWrongRegionException() throws Exception {
7127    final byte[] a = Bytes.toBytes("a");
7128    final byte[] b = Bytes.toBytes("b");
7129    final byte[] c = Bytes.toBytes("c"); // exclusive
7130
7131    int prevLockTimeout = CONF.getInt("hbase.rowlock.wait.duration", 30000);
7132    CONF.setInt("hbase.rowlock.wait.duration", 1000);
7133    region = initHRegion(tableName, a, c, method, CONF, false, fam1);
7134
7135    Mutation[] mutations = new Mutation[] {
7136      new Put(a).add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a)
7137        .setFamily(fam1).setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()),
7138      // this is outside the region boundary
7139      new Put(c).add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(c)
7140        .setFamily(fam1).setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Type.Put).build()),
7141      new Put(b)
7142        .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1)
7143          .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()) };
7144
7145    OperationStatus[] status = region.batchMutate(mutations);
7146    assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode());
7147    assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, status[1].getOperationStatusCode());
7148    assertEquals(OperationStatusCode.SUCCESS, status[2].getOperationStatusCode());
7149
7150    // test with a row lock held for a long time
7151    final CountDownLatch obtainedRowLock = new CountDownLatch(1);
7152    ExecutorService exec = Executors.newFixedThreadPool(2);
7153    Future<Void> f1 = exec.submit(new Callable<Void>() {
7154      @Override
7155      public Void call() throws Exception {
7156        LOG.info("Acquiring row lock");
7157        RowLock rl = region.getRowLock(b);
7158        obtainedRowLock.countDown();
7159        LOG.info("Waiting for 5 seconds before releasing lock");
7160        Threads.sleep(5000);
7161        LOG.info("Releasing row lock");
7162        rl.release();
7163        return null;
7164      }
7165    });
7166    obtainedRowLock.await(30, TimeUnit.SECONDS);
7167
7168    Future<Void> f2 = exec.submit(new Callable<Void>() {
7169      @Override
7170      public Void call() throws Exception {
7171        Mutation[] mutations = new Mutation[] {
7172          new Put(a)
7173            .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a).setFamily(fam1)
7174              .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()),
7175          new Put(b)
7176            .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1)
7177              .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), };
7178
7179        // this will wait for the row lock, and it will eventually succeed
7180        OperationStatus[] status = region.batchMutate(mutations);
7181        assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode());
7182        assertEquals(OperationStatusCode.SUCCESS, status[1].getOperationStatusCode());
7183        return null;
7184      }
7185    });
7186
7187    f1.get();
7188    f2.get();
7189
7190    CONF.setInt("hbase.rowlock.wait.duration", prevLockTimeout);
7191  }
7192
7193  @Test
7194  public void testBatchMutateWithZeroRowLockWait() throws Exception {
7195    final byte[] a = Bytes.toBytes("a");
7196    final byte[] b = Bytes.toBytes("b");
7197    final byte[] c = Bytes.toBytes("c"); // exclusive
7198
7199    Configuration conf = new Configuration(CONF);
7200    conf.setInt("hbase.rowlock.wait.duration", 0);
7201    final RegionInfo hri =
7202      RegionInfoBuilder.newBuilder(tableName).setStartKey(a).setEndKey(c).build();
7203    final TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName)
7204      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)).build();
7205    region = HRegion.createHRegion(hri, TEST_UTIL.getDataTestDir(), conf, htd,
7206      HBaseTestingUtil.createWal(conf, TEST_UTIL.getDataTestDirOnTestFS(method + ".log"), hri));
7207
7208    Mutation[] mutations = new Mutation[] {
7209      new Put(a).add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a)
7210        .setFamily(fam1).setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()),
7211      new Put(b)
7212        .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1)
7213          .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()) };
7214
7215    OperationStatus[] status = region.batchMutate(mutations);
7216    assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode());
7217    assertEquals(OperationStatusCode.SUCCESS, status[1].getOperationStatusCode());
7218
7219    // test with a row lock held for a long time
7220    final CountDownLatch obtainedRowLock = new CountDownLatch(1);
7221    ExecutorService exec = Executors.newFixedThreadPool(2);
7222    Future<Void> f1 = exec.submit(new Callable<Void>() {
7223      @Override
7224      public Void call() throws Exception {
7225        LOG.info("Acquiring row lock");
7226        RowLock rl = region.getRowLock(b);
7227        obtainedRowLock.countDown();
7228        LOG.info("Waiting for 5 seconds before releasing lock");
7229        Threads.sleep(5000);
7230        LOG.info("Releasing row lock");
7231        rl.release();
7232        return null;
7233      }
7234    });
7235    obtainedRowLock.await(30, TimeUnit.SECONDS);
7236
7237    Future<Void> f2 = exec.submit(new Callable<Void>() {
7238      @Override
7239      public Void call() throws Exception {
7240        Mutation[] mutations = new Mutation[] {
7241          new Put(a)
7242            .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a).setFamily(fam1)
7243              .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()),
7244          new Put(b)
7245            .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1)
7246              .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), };
7247        // when handling row b we are going to spin on the failure to get the row lock
7248        // until the lock above is released, but we will still succeed so long as that
7249        // takes less time then the test time out.
7250        OperationStatus[] status = region.batchMutate(mutations);
7251        assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode());
7252        assertEquals(OperationStatusCode.SUCCESS, status[1].getOperationStatusCode());
7253        return null;
7254      }
7255    });
7256
7257    f1.get();
7258    f2.get();
7259  }
7260
7261  @Test
7262  public void testCheckAndRowMutateTimestampsAreMonotonic() throws IOException {
7263    region = initHRegion(tableName, method, CONF, fam1);
7264    ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
7265    EnvironmentEdgeManager.injectEdge(edge);
7266
7267    edge.setValue(10);
7268    Put p = new Put(row);
7269    p.setDurability(Durability.SKIP_WAL);
7270    p.addColumn(fam1, qual1, qual1);
7271    region.put(p);
7272
7273    Result result = region.get(new Get(row));
7274    Cell c = result.getColumnLatestCell(fam1, qual1);
7275    assertNotNull(c);
7276    assertEquals(10L, c.getTimestamp());
7277
7278    edge.setValue(1); // clock goes back
7279    p = new Put(row);
7280    p.setDurability(Durability.SKIP_WAL);
7281    p.addColumn(fam1, qual1, qual2);
7282    RowMutations rm = new RowMutations(row);
7283    rm.add(p);
7284    assertTrue(region.checkAndRowMutate(row, fam1, qual1, CompareOperator.EQUAL,
7285      new BinaryComparator(qual1), rm));
7286    result = region.get(new Get(row));
7287    c = result.getColumnLatestCell(fam1, qual1);
7288    assertEquals(10L, c.getTimestamp());
7289    LOG.info(
7290      "c value " + Bytes.toStringBinary(c.getValueArray(), c.getValueOffset(), c.getValueLength()));
7291
7292    assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(), qual2, 0,
7293      qual2.length));
7294  }
7295
7296  private HRegion initHRegion(TableName tableName, String callingMethod, byte[]... families)
7297    throws IOException {
7298    return initHRegion(tableName, callingMethod, HBaseConfiguration.create(), families);
7299  }
7300
7301  /**
7302   * HBASE-16429 Make sure no stuck if roll writer when ring buffer is filled with appends
7303   * @throws IOException if IO error occurred during test
7304   */
7305  @Test
7306  public void testWritesWhileRollWriter() throws IOException {
7307    int testCount = 10;
7308    int numRows = 1024;
7309    int numFamilies = 2;
7310    int numQualifiers = 2;
7311    final byte[][] families = new byte[numFamilies][];
7312    for (int i = 0; i < numFamilies; i++) {
7313      families[i] = Bytes.toBytes("family" + i);
7314    }
7315    final byte[][] qualifiers = new byte[numQualifiers][];
7316    for (int i = 0; i < numQualifiers; i++) {
7317      qualifiers[i] = Bytes.toBytes("qual" + i);
7318    }
7319
7320    CONF.setInt("hbase.regionserver.wal.disruptor.event.count", 2);
7321    this.region = initHRegion(tableName, method, CONF, families);
7322    try {
7323      List<Thread> threads = new ArrayList<>();
7324      for (int i = 0; i < numRows; i++) {
7325        final int count = i;
7326        Thread t = new Thread(new Runnable() {
7327
7328          @Override
7329          public void run() {
7330            byte[] row = Bytes.toBytes("row" + count);
7331            Put put = new Put(row);
7332            put.setDurability(Durability.SYNC_WAL);
7333            byte[] value = Bytes.toBytes(String.valueOf(count));
7334            for (byte[] family : families) {
7335              for (byte[] qualifier : qualifiers) {
7336                put.addColumn(family, qualifier, count, value);
7337              }
7338            }
7339            try {
7340              region.put(put);
7341            } catch (IOException e) {
7342              throw new RuntimeException(e);
7343            }
7344          }
7345        });
7346        threads.add(t);
7347      }
7348      for (Thread t : threads) {
7349        t.start();
7350      }
7351
7352      for (int i = 0; i < testCount; i++) {
7353        region.getWAL().rollWriter();
7354        Thread.yield();
7355      }
7356    } finally {
7357      try {
7358        HBaseTestingUtil.closeRegionAndWAL(this.region);
7359        CONF.setInt("hbase.regionserver.wal.disruptor.event.count", 16 * 1024);
7360      } catch (DroppedSnapshotException dse) {
7361        // We could get this on way out because we interrupt the background flusher and it could
7362        // fail anywhere causing a DSE over in the background flusher... only it is not properly
7363        // dealt with so could still be memory hanging out when we get to here -- memory we can't
7364        // flush because the accounting is 'off' since original DSE.
7365      }
7366      this.region = null;
7367    }
7368  }
7369
7370  @Test
7371  public void testMutateRow() throws Exception {
7372    final byte[] row = Bytes.toBytes("row");
7373    final byte[] q1 = Bytes.toBytes("q1");
7374    final byte[] q2 = Bytes.toBytes("q2");
7375    final byte[] q3 = Bytes.toBytes("q3");
7376    final byte[] q4 = Bytes.toBytes("q4");
7377    final String v1 = "v1";
7378
7379    region = initHRegion(tableName, method, CONF, fam1);
7380
7381    // Initial values
7382    region
7383      .batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q2, Bytes.toBytes("toBeDeleted")),
7384        new Put(row).addColumn(fam1, q3, Bytes.toBytes(5L)),
7385        new Put(row).addColumn(fam1, q4, Bytes.toBytes("a")), });
7386
7387    // Do mutateRow
7388    Result result = region.mutateRow(
7389      new RowMutations(row).add(Arrays.asList(new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1)),
7390        new Delete(row).addColumns(fam1, q2), new Increment(row).addColumn(fam1, q3, 1),
7391        new Append(row).addColumn(fam1, q4, Bytes.toBytes("b")))));
7392
7393    assertNotNull(result);
7394    assertEquals(6L, Bytes.toLong(result.getValue(fam1, q3)));
7395    assertEquals("ab", Bytes.toString(result.getValue(fam1, q4)));
7396
7397    // Verify the value
7398    result = region.get(new Get(row));
7399    assertEquals(v1, Bytes.toString(result.getValue(fam1, q1)));
7400    assertNull(result.getValue(fam1, q2));
7401    assertEquals(6L, Bytes.toLong(result.getValue(fam1, q3)));
7402    assertEquals("ab", Bytes.toString(result.getValue(fam1, q4)));
7403  }
7404
7405  @Test
7406  public void testMutateRowInParallel() throws Exception {
7407    final int numReaderThreads = 100;
7408    final CountDownLatch latch = new CountDownLatch(numReaderThreads);
7409
7410    final byte[] row = Bytes.toBytes("row");
7411    final byte[] q1 = Bytes.toBytes("q1");
7412    final byte[] q2 = Bytes.toBytes("q2");
7413    final byte[] q3 = Bytes.toBytes("q3");
7414    final byte[] q4 = Bytes.toBytes("q4");
7415    final String v1 = "v1";
7416    final String v2 = "v2";
7417
7418    // We need to ensure the timestamp of the delete operation is more than the previous one
7419    final AtomicLong deleteTimestamp = new AtomicLong();
7420
7421    region = initHRegion(tableName, method, CONF, fam1);
7422
7423    // Initial values
7424    region.batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1))
7425      .addColumn(fam1, q2, deleteTimestamp.getAndIncrement(), Bytes.toBytes(v2))
7426      .addColumn(fam1, q3, Bytes.toBytes(1L)).addColumn(fam1, q4, Bytes.toBytes("a")) });
7427
7428    final AtomicReference<AssertionError> assertionError = new AtomicReference<>();
7429
7430    // Writer thread
7431    Thread writerThread = new Thread(() -> {
7432      try {
7433        while (true) {
7434          // If all the reader threads finish, then stop the writer thread
7435          if (latch.await(0, TimeUnit.MILLISECONDS)) {
7436            return;
7437          }
7438
7439          // Execute the mutations. This should be done atomically
7440          region.mutateRow(new RowMutations(row)
7441            .add(Arrays.asList(new Put(row).addColumn(fam1, q1, Bytes.toBytes(v2)),
7442              new Delete(row).addColumns(fam1, q2, deleteTimestamp.getAndIncrement()),
7443              new Increment(row).addColumn(fam1, q3, 1L),
7444              new Append(row).addColumn(fam1, q4, Bytes.toBytes("b")))));
7445
7446          // We need to ensure the timestamps of the Increment/Append operations are more than the
7447          // previous ones
7448          Result result = region.get(new Get(row).addColumn(fam1, q3).addColumn(fam1, q4));
7449          long tsIncrement = result.getColumnLatestCell(fam1, q3).getTimestamp();
7450          long tsAppend = result.getColumnLatestCell(fam1, q4).getTimestamp();
7451
7452          // Put the initial values
7453          region.batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1))
7454            .addColumn(fam1, q2, deleteTimestamp.getAndIncrement(), Bytes.toBytes(v2))
7455            .addColumn(fam1, q3, tsIncrement + 1, Bytes.toBytes(1L))
7456            .addColumn(fam1, q4, tsAppend + 1, Bytes.toBytes("a")) });
7457        }
7458      } catch (Exception e) {
7459        assertionError.set(new AssertionError(e));
7460      }
7461    });
7462    writerThread.start();
7463
7464    // Reader threads
7465    for (int i = 0; i < numReaderThreads; i++) {
7466      new Thread(() -> {
7467        try {
7468          for (int j = 0; j < 10000; j++) {
7469            // Verify the values
7470            Result result = region.get(new Get(row));
7471
7472            // The values should be equals to either the initial values or the values after
7473            // executing the mutations
7474            String q1Value = Bytes.toString(result.getValue(fam1, q1));
7475            if (v1.equals(q1Value)) {
7476              assertEquals(v2, Bytes.toString(result.getValue(fam1, q2)));
7477              assertEquals(1L, Bytes.toLong(result.getValue(fam1, q3)));
7478              assertEquals("a", Bytes.toString(result.getValue(fam1, q4)));
7479            } else if (v2.equals(q1Value)) {
7480              assertNull(Bytes.toString(result.getValue(fam1, q2)));
7481              assertEquals(2L, Bytes.toLong(result.getValue(fam1, q3)));
7482              assertEquals("ab", Bytes.toString(result.getValue(fam1, q4)));
7483            } else {
7484              fail("the qualifier " + Bytes.toString(q1) + " should be " + v1 + " or " + v2
7485                + ", but " + q1Value);
7486            }
7487          }
7488        } catch (Exception e) {
7489          assertionError.set(new AssertionError(e));
7490        } catch (AssertionError e) {
7491          assertionError.set(e);
7492        }
7493
7494        latch.countDown();
7495      }).start();
7496    }
7497
7498    writerThread.join();
7499
7500    if (assertionError.get() != null) {
7501      throw assertionError.get();
7502    }
7503  }
7504
7505  @Test
7506  public void testMutateRow_WriteRequestCount() throws Exception {
7507    byte[] row1 = Bytes.toBytes("row1");
7508    byte[] fam1 = Bytes.toBytes("fam1");
7509    byte[] qf1 = Bytes.toBytes("qualifier");
7510    byte[] val1 = Bytes.toBytes("value1");
7511
7512    RowMutations rm = new RowMutations(row1);
7513    Put put = new Put(row1);
7514    put.addColumn(fam1, qf1, val1);
7515    rm.add(put);
7516
7517    this.region = initHRegion(tableName, method, CONF, fam1);
7518    long wrcBeforeMutate = this.region.writeRequestsCount.longValue();
7519    this.region.mutateRow(rm);
7520    long wrcAfterMutate = this.region.writeRequestsCount.longValue();
7521    Assert.assertEquals(wrcBeforeMutate + rm.getMutations().size(), wrcAfterMutate);
7522  }
7523
7524  @Test
7525  public void testBulkLoadReplicationEnabled() throws IOException {
7526    TEST_UTIL.getConfiguration().setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true);
7527    final ServerName serverName = ServerName.valueOf(name.getMethodName(), 100, 42);
7528    final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName));
7529
7530    TableDescriptor tableDescriptor =
7531      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
7532        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)).build();
7533    RegionInfo hri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
7534    region = HRegion.openHRegion(hri, tableDescriptor, rss.getWAL(hri),
7535      TEST_UTIL.getConfiguration(), rss, null);
7536
7537    assertTrue(region.conf.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, false));
7538    String plugins = region.conf.get(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, "");
7539    String replicationCoprocessorClass = ReplicationObserver.class.getCanonicalName();
7540    assertTrue(plugins.contains(replicationCoprocessorClass));
7541    assertTrue(region.getCoprocessorHost().getCoprocessors()
7542      .contains(ReplicationObserver.class.getSimpleName()));
7543  }
7544
7545  /**
7546   * The same as HRegion class, the only difference is that instantiateHStore will create a
7547   * different HStore - HStoreForTesting. [HBASE-8518]
7548   */
7549  public static class HRegionForTesting extends HRegion {
7550
7551    public HRegionForTesting(final Path tableDir, final WAL wal, final FileSystem fs,
7552      final Configuration confParam, final RegionInfo regionInfo, final TableDescriptor htd,
7553      final RegionServerServices rsServices) {
7554      this(new HRegionFileSystem(confParam, fs, tableDir, regionInfo), wal, confParam, htd,
7555        rsServices);
7556    }
7557
7558    public HRegionForTesting(HRegionFileSystem fs, WAL wal, Configuration confParam,
7559      TableDescriptor htd, RegionServerServices rsServices) {
7560      super(fs, wal, confParam, htd, rsServices);
7561    }
7562
7563    /**
7564     * Create HStore instance.
7565     * @return If Mob is enabled, return HMobStore, otherwise return HStoreForTesting.
7566     */
7567    @Override
7568    protected HStore instantiateHStore(final ColumnFamilyDescriptor family, boolean warmup)
7569      throws IOException {
7570      if (family.isMobEnabled()) {
7571        if (HFile.getFormatVersion(this.conf) < HFile.MIN_FORMAT_VERSION_WITH_TAGS) {
7572          throw new IOException("A minimum HFile version of " + HFile.MIN_FORMAT_VERSION_WITH_TAGS
7573            + " is required for MOB feature. Consider setting " + HFile.FORMAT_VERSION_KEY
7574            + " accordingly.");
7575        }
7576        return new HMobStore(this, family, this.conf, warmup);
7577      }
7578      return new HStoreForTesting(this, family, this.conf, warmup);
7579    }
7580  }
7581
7582  /**
7583   * HStoreForTesting is merely the same as HStore, the difference is in the doCompaction method of
7584   * HStoreForTesting there is a checkpoint "hbase.hstore.compaction.complete" which doesn't let
7585   * hstore compaction complete. In the former edition, this config is set in HStore class inside
7586   * compact method, though this is just for testing, otherwise it doesn't do any help. In
7587   * HBASE-8518, we try to get rid of all "hbase.hstore.compaction.complete" config (except for
7588   * testing code).
7589   */
7590  public static class HStoreForTesting extends HStore {
7591
7592    protected HStoreForTesting(final HRegion region, final ColumnFamilyDescriptor family,
7593      final Configuration confParam, boolean warmup) throws IOException {
7594      super(region, family, confParam, warmup);
7595    }
7596
7597    @Override
7598    protected List<HStoreFile> doCompaction(CompactionRequestImpl cr,
7599      Collection<HStoreFile> filesToCompact, User user, long compactionStartTime,
7600      List<Path> newFiles) throws IOException {
7601      // let compaction incomplete.
7602      if (!this.conf.getBoolean("hbase.hstore.compaction.complete", true)) {
7603        LOG.warn("hbase.hstore.compaction.complete is set to false");
7604        List<HStoreFile> sfs = new ArrayList<>(newFiles.size());
7605        final boolean evictOnClose =
7606          getCacheConfig() != null ? getCacheConfig().shouldEvictOnClose() : true;
7607        for (Path newFile : newFiles) {
7608          // Create storefile around what we wrote with a reader on it.
7609          HStoreFile sf = storeEngine.createStoreFileAndReader(newFile);
7610          sf.closeStoreFile(evictOnClose);
7611          sfs.add(sf);
7612        }
7613        return sfs;
7614      }
7615      return super.doCompaction(cr, filesToCompact, user, compactionStartTime, newFiles);
7616    }
7617  }
7618
7619  @Test
7620  public void testCloseNoInterrupt() throws Exception {
7621    byte[] cf1 = Bytes.toBytes("CF1");
7622    byte[][] families = { cf1 };
7623    final int SLEEP_TIME = 10 * 1000;
7624
7625    Configuration conf = new Configuration(CONF);
7626    // Disable close thread interrupt and server abort behavior
7627    conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, false);
7628    conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, 1000);
7629    region = initHRegion(tableName, method, conf, families);
7630
7631    final CountDownLatch latch = new CountDownLatch(1);
7632    final AtomicBoolean holderInterrupted = new AtomicBoolean();
7633    Thread holder = new Thread(new Runnable() {
7634      @Override
7635      public void run() {
7636        try {
7637          LOG.info("Starting region operation holder");
7638          region.startRegionOperation(Operation.SCAN);
7639          latch.countDown();
7640          try {
7641            Thread.sleep(SLEEP_TIME);
7642          } catch (InterruptedException e) {
7643            LOG.info("Interrupted");
7644            holderInterrupted.set(true);
7645          }
7646        } catch (Exception e) {
7647          throw new RuntimeException(e);
7648        } finally {
7649          try {
7650            region.closeRegionOperation();
7651          } catch (IOException e) {
7652          }
7653          LOG.info("Stopped region operation holder");
7654        }
7655      }
7656    });
7657
7658    holder.start();
7659    latch.await();
7660    HBaseTestingUtil.closeRegionAndWAL(region);
7661    region = null;
7662    holder.join();
7663
7664    assertFalse("Region lock holder should not have been interrupted", holderInterrupted.get());
7665  }
7666
7667  @Test
7668  public void testCloseInterrupt() throws Exception {
7669    byte[] cf1 = Bytes.toBytes("CF1");
7670    byte[][] families = { cf1 };
7671    final int SLEEP_TIME = 10 * 1000;
7672
7673    Configuration conf = new Configuration(CONF);
7674    // Enable close thread interrupt and server abort behavior
7675    conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, true);
7676    // Speed up the unit test, no need to wait default 10 seconds.
7677    conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, 1000);
7678    region = initHRegion(tableName, method, conf, families);
7679
7680    final CountDownLatch latch = new CountDownLatch(1);
7681    final AtomicBoolean holderInterrupted = new AtomicBoolean();
7682    Thread holder = new Thread(new Runnable() {
7683      @Override
7684      public void run() {
7685        try {
7686          LOG.info("Starting region operation holder");
7687          region.startRegionOperation(Operation.SCAN);
7688          latch.countDown();
7689          try {
7690            Thread.sleep(SLEEP_TIME);
7691          } catch (InterruptedException e) {
7692            LOG.info("Interrupted");
7693            holderInterrupted.set(true);
7694          }
7695        } catch (Exception e) {
7696          throw new RuntimeException(e);
7697        } finally {
7698          try {
7699            region.closeRegionOperation();
7700          } catch (IOException e) {
7701          }
7702          LOG.info("Stopped region operation holder");
7703        }
7704      }
7705    });
7706
7707    holder.start();
7708    latch.await();
7709    region.close();
7710    region = null;
7711    holder.join();
7712
7713    assertTrue("Region lock holder was not interrupted", holderInterrupted.get());
7714  }
7715
7716  @Test
7717  public void testCloseAbort() throws Exception {
7718    byte[] cf1 = Bytes.toBytes("CF1");
7719    byte[][] families = { cf1 };
7720    final int SLEEP_TIME = 10 * 1000;
7721
7722    Configuration conf = new Configuration(CONF);
7723    // Enable close thread interrupt and server abort behavior.
7724    conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, true);
7725    // Set the abort interval to a fraction of sleep time so we are guaranteed to be aborted.
7726    conf.setInt(HRegion.CLOSE_WAIT_TIME, SLEEP_TIME / 2);
7727    // Set the wait interval to a fraction of sleep time so we are guaranteed to be interrupted.
7728    conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, SLEEP_TIME / 4);
7729    region = initHRegion(tableName, method, conf, families);
7730    RegionServerServices rsServices = mock(RegionServerServices.class);
7731    when(rsServices.getServerName()).thenReturn(ServerName.valueOf("localhost", 1000, 1000));
7732    region.rsServices = rsServices;
7733
7734    final CountDownLatch latch = new CountDownLatch(1);
7735    Thread holder = new Thread(new Runnable() {
7736      @Override
7737      public void run() {
7738        try {
7739          LOG.info("Starting region operation holder");
7740          region.startRegionOperation(Operation.SCAN);
7741          latch.countDown();
7742          // Hold the lock for SLEEP_TIME seconds no matter how many times we are interrupted.
7743          int timeRemaining = SLEEP_TIME;
7744          while (timeRemaining > 0) {
7745            long start = EnvironmentEdgeManager.currentTime();
7746            try {
7747              Thread.sleep(timeRemaining);
7748            } catch (InterruptedException e) {
7749              LOG.info("Interrupted");
7750            }
7751            long end = EnvironmentEdgeManager.currentTime();
7752            timeRemaining -= end - start;
7753            if (timeRemaining < 0) {
7754              timeRemaining = 0;
7755            }
7756            if (timeRemaining > 0) {
7757              LOG.info("Sleeping again, remaining time " + timeRemaining + " ms");
7758            }
7759          }
7760        } catch (Exception e) {
7761          throw new RuntimeException(e);
7762        } finally {
7763          try {
7764            region.closeRegionOperation();
7765          } catch (IOException e) {
7766          }
7767          LOG.info("Stopped region operation holder");
7768        }
7769      }
7770    });
7771
7772    holder.start();
7773    latch.await();
7774    assertThrows(IOException.class, () -> region.close());
7775    holder.join();
7776
7777    // Verify the region tried to abort the server
7778    verify(rsServices, atLeast(1)).abort(anyString(), any());
7779  }
7780
7781  @Test
7782  public void testInterruptProtection() throws Exception {
7783    byte[] cf1 = Bytes.toBytes("CF1");
7784    byte[][] families = { cf1 };
7785    final int SLEEP_TIME = 10 * 1000;
7786
7787    Configuration conf = new Configuration(CONF);
7788    // Enable close thread interrupt and server abort behavior.
7789    conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, true);
7790    conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, 1000);
7791    region = initHRegion(tableName, method, conf, families);
7792
7793    final CountDownLatch latch = new CountDownLatch(1);
7794    final AtomicBoolean holderInterrupted = new AtomicBoolean();
7795    Thread holder = new Thread(new Runnable() {
7796      @Override
7797      public void run() {
7798        try {
7799          LOG.info("Starting region operation holder");
7800          region.startRegionOperation(Operation.SCAN);
7801          LOG.info("Protecting against interrupts");
7802          region.disableInterrupts();
7803          try {
7804            latch.countDown();
7805            try {
7806              Thread.sleep(SLEEP_TIME);
7807            } catch (InterruptedException e) {
7808              LOG.info("Interrupted");
7809              holderInterrupted.set(true);
7810            }
7811          } finally {
7812            region.enableInterrupts();
7813          }
7814        } catch (Exception e) {
7815          throw new RuntimeException(e);
7816        } finally {
7817          try {
7818            region.closeRegionOperation();
7819          } catch (IOException e) {
7820          }
7821          LOG.info("Stopped region operation holder");
7822        }
7823      }
7824    });
7825
7826    holder.start();
7827    latch.await();
7828    region.close();
7829    region = null;
7830    holder.join();
7831
7832    assertFalse("Region lock holder should not have been interrupted", holderInterrupted.get());
7833  }
7834
7835  @Test
7836  public void testRegionOnCoprocessorsChange() throws IOException {
7837    byte[] cf1 = Bytes.toBytes("CF1");
7838    byte[][] families = { cf1 };
7839
7840    Configuration conf = new Configuration(CONF);
7841    region = initHRegion(tableName, method, conf, families);
7842    assertNull(region.getCoprocessorHost());
7843
7844    // set and verify the system coprocessors for region and user region
7845    Configuration newConf = new Configuration(conf);
7846    newConf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, MetaTableMetrics.class.getName());
7847    newConf.set(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
7848      NoOpRegionCoprocessor.class.getName());
7849    // trigger configuration change
7850    region.onConfigurationChange(newConf);
7851    assertTrue(region.getCoprocessorHost() != null);
7852    Set<String> coprocessors = region.getCoprocessorHost().getCoprocessors();
7853    assertTrue(coprocessors.size() == 2);
7854    assertTrue(region.getCoprocessorHost().getCoprocessors()
7855      .contains(MetaTableMetrics.class.getSimpleName()));
7856    assertTrue(region.getCoprocessorHost().getCoprocessors()
7857      .contains(NoOpRegionCoprocessor.class.getSimpleName()));
7858
7859    // remove region coprocessor and keep only user region coprocessor
7860    newConf.unset(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY);
7861    region.onConfigurationChange(newConf);
7862    assertTrue(region.getCoprocessorHost() != null);
7863    coprocessors = region.getCoprocessorHost().getCoprocessors();
7864    assertTrue(coprocessors.size() == 1);
7865    assertTrue(region.getCoprocessorHost().getCoprocessors()
7866      .contains(NoOpRegionCoprocessor.class.getSimpleName()));
7867  }
7868
7869  @Test
7870  public void testRegionOnCoprocessorsWithoutChange() throws IOException {
7871    byte[] cf1 = Bytes.toBytes("CF1");
7872    byte[][] families = { cf1 };
7873
7874    Configuration conf = new Configuration(CONF);
7875    conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
7876      MetaTableMetrics.class.getCanonicalName());
7877    region = initHRegion(tableName, method, conf, families);
7878    // region service is null in unit test, we need to load the coprocessor once
7879    region.setCoprocessorHost(new RegionCoprocessorHost(region, null, conf));
7880    RegionCoprocessor regionCoprocessor =
7881      region.getCoprocessorHost().findCoprocessor(MetaTableMetrics.class.getName());
7882
7883    // simulate when other configuration may have changed and onConfigurationChange execute once
7884    region.onConfigurationChange(conf);
7885    RegionCoprocessor regionCoprocessorAfterOnConfigurationChange =
7886      region.getCoprocessorHost().findCoprocessor(MetaTableMetrics.class.getName());
7887    assertEquals(regionCoprocessor, regionCoprocessorAfterOnConfigurationChange);
7888  }
7889
7890  public static class NoOpRegionCoprocessor implements RegionCoprocessor, RegionObserver {
7891    // a empty region coprocessor class
7892  }
7893}