001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.apache.hadoop.hbase.HBaseTestingUtil.COLUMNS;
021import static org.apache.hadoop.hbase.HBaseTestingUtil.fam1;
022import static org.apache.hadoop.hbase.HBaseTestingUtil.fam2;
023import static org.apache.hadoop.hbase.HBaseTestingUtil.fam3;
024import static org.junit.Assert.assertArrayEquals;
025import static org.junit.Assert.assertEquals;
026import static org.junit.Assert.assertFalse;
027import static org.junit.Assert.assertNotNull;
028import static org.junit.Assert.assertNull;
029import static org.junit.Assert.assertThrows;
030import static org.junit.Assert.assertTrue;
031import static org.junit.Assert.fail;
032import static org.mockito.ArgumentMatchers.any;
033import static org.mockito.ArgumentMatchers.anyLong;
034import static org.mockito.ArgumentMatchers.anyString;
035import static org.mockito.Mockito.atLeast;
036import static org.mockito.Mockito.doThrow;
037import static org.mockito.Mockito.mock;
038import static org.mockito.Mockito.never;
039import static org.mockito.Mockito.spy;
040import static org.mockito.Mockito.times;
041import static org.mockito.Mockito.verify;
042import static org.mockito.Mockito.when;
043
044import java.io.IOException;
045import java.io.InterruptedIOException;
046import java.math.BigDecimal;
047import java.security.PrivilegedExceptionAction;
048import java.util.ArrayList;
049import java.util.Arrays;
050import java.util.Collection;
051import java.util.List;
052import java.util.Map;
053import java.util.NavigableMap;
054import java.util.Objects;
055import java.util.Set;
056import java.util.TreeMap;
057import java.util.concurrent.Callable;
058import java.util.concurrent.CountDownLatch;
059import java.util.concurrent.ExecutorService;
060import java.util.concurrent.Executors;
061import java.util.concurrent.Future;
062import java.util.concurrent.TimeUnit;
063import java.util.concurrent.atomic.AtomicBoolean;
064import java.util.concurrent.atomic.AtomicInteger;
065import java.util.concurrent.atomic.AtomicLong;
066import java.util.concurrent.atomic.AtomicReference;
067import org.apache.commons.lang3.RandomStringUtils;
068import org.apache.hadoop.conf.Configuration;
069import org.apache.hadoop.fs.FSDataOutputStream;
070import org.apache.hadoop.fs.FileStatus;
071import org.apache.hadoop.fs.FileSystem;
072import org.apache.hadoop.fs.Path;
073import org.apache.hadoop.hbase.ArrayBackedTag;
074import org.apache.hadoop.hbase.Cell;
075import org.apache.hadoop.hbase.Cell.Type;
076import org.apache.hadoop.hbase.CellBuilderFactory;
077import org.apache.hadoop.hbase.CellBuilderType;
078import org.apache.hadoop.hbase.CellUtil;
079import org.apache.hadoop.hbase.CompareOperator;
080import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
081import org.apache.hadoop.hbase.DoNotRetryIOException;
082import org.apache.hadoop.hbase.DroppedSnapshotException;
083import org.apache.hadoop.hbase.ExtendedCellBuilderFactory;
084import org.apache.hadoop.hbase.HBaseClassTestRule;
085import org.apache.hadoop.hbase.HBaseConfiguration;
086import org.apache.hadoop.hbase.HBaseTestingUtil;
087import org.apache.hadoop.hbase.HConstants;
088import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
089import org.apache.hadoop.hbase.HDFSBlocksDistribution;
090import org.apache.hadoop.hbase.KeyValue;
091import org.apache.hadoop.hbase.MultithreadedTestUtil;
092import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
093import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
094import org.apache.hadoop.hbase.NotServingRegionException;
095import org.apache.hadoop.hbase.PrivateCellUtil;
096import org.apache.hadoop.hbase.RegionTooBusyException;
097import org.apache.hadoop.hbase.ServerName;
098import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
099import org.apache.hadoop.hbase.StartTestingClusterOption;
100import org.apache.hadoop.hbase.TableName;
101import org.apache.hadoop.hbase.TagType;
102import org.apache.hadoop.hbase.Waiter;
103import org.apache.hadoop.hbase.client.Append;
104import org.apache.hadoop.hbase.client.CheckAndMutate;
105import org.apache.hadoop.hbase.client.CheckAndMutateResult;
106import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
107import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
108import org.apache.hadoop.hbase.client.Delete;
109import org.apache.hadoop.hbase.client.Durability;
110import org.apache.hadoop.hbase.client.Get;
111import org.apache.hadoop.hbase.client.Increment;
112import org.apache.hadoop.hbase.client.Mutation;
113import org.apache.hadoop.hbase.client.Put;
114import org.apache.hadoop.hbase.client.RegionInfo;
115import org.apache.hadoop.hbase.client.RegionInfoBuilder;
116import org.apache.hadoop.hbase.client.Result;
117import org.apache.hadoop.hbase.client.RowMutations;
118import org.apache.hadoop.hbase.client.Scan;
119import org.apache.hadoop.hbase.client.Table;
120import org.apache.hadoop.hbase.client.TableDescriptor;
121import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
122import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
123import org.apache.hadoop.hbase.coprocessor.MetaTableMetrics;
124import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
125import org.apache.hadoop.hbase.coprocessor.RegionObserver;
126import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
127import org.apache.hadoop.hbase.filter.BigDecimalComparator;
128import org.apache.hadoop.hbase.filter.BinaryComparator;
129import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
130import org.apache.hadoop.hbase.filter.Filter;
131import org.apache.hadoop.hbase.filter.FilterBase;
132import org.apache.hadoop.hbase.filter.FilterList;
133import org.apache.hadoop.hbase.filter.NullComparator;
134import org.apache.hadoop.hbase.filter.PrefixFilter;
135import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
136import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
137import org.apache.hadoop.hbase.filter.SubstringComparator;
138import org.apache.hadoop.hbase.filter.ValueFilter;
139import org.apache.hadoop.hbase.io.TimeRange;
140import org.apache.hadoop.hbase.io.hfile.HFile;
141import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
142import org.apache.hadoop.hbase.monitoring.MonitoredTask;
143import org.apache.hadoop.hbase.monitoring.TaskMonitor;
144import org.apache.hadoop.hbase.regionserver.Region.Operation;
145import org.apache.hadoop.hbase.regionserver.Region.RowLock;
146import org.apache.hadoop.hbase.regionserver.TestHStore.FaultyFileSystem;
147import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl;
148import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
149import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
150import org.apache.hadoop.hbase.regionserver.wal.WALUtil;
151import org.apache.hadoop.hbase.replication.regionserver.ReplicationObserver;
152import org.apache.hadoop.hbase.security.User;
153import org.apache.hadoop.hbase.test.MetricsAssertHelper;
154import org.apache.hadoop.hbase.testclassification.LargeTests;
155import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests;
156import org.apache.hadoop.hbase.util.Bytes;
157import org.apache.hadoop.hbase.util.CommonFSUtils;
158import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
159import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
160import org.apache.hadoop.hbase.util.HFileArchiveUtil;
161import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
162import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
163import org.apache.hadoop.hbase.util.Threads;
164import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
165import org.apache.hadoop.hbase.wal.FaultyFSLog;
166import org.apache.hadoop.hbase.wal.NettyAsyncFSWALConfigHelper;
167import org.apache.hadoop.hbase.wal.WAL;
168import org.apache.hadoop.hbase.wal.WALEdit;
169import org.apache.hadoop.hbase.wal.WALFactory;
170import org.apache.hadoop.hbase.wal.WALKeyImpl;
171import org.apache.hadoop.hbase.wal.WALProvider;
172import org.apache.hadoop.hbase.wal.WALProvider.Writer;
173import org.apache.hadoop.hbase.wal.WALSplitUtil;
174import org.junit.After;
175import org.junit.Assert;
176import org.junit.Before;
177import org.junit.ClassRule;
178import org.junit.Rule;
179import org.junit.Test;
180import org.junit.experimental.categories.Category;
181import org.junit.rules.ExpectedException;
182import org.junit.rules.TestName;
183import org.mockito.ArgumentCaptor;
184import org.mockito.ArgumentMatcher;
185import org.mockito.Mockito;
186import org.mockito.invocation.InvocationOnMock;
187import org.mockito.stubbing.Answer;
188import org.slf4j.Logger;
189import org.slf4j.LoggerFactory;
190
191import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
192import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
193import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
194import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup;
195import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel;
196
197import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
198import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor;
199import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor;
200import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.FlushAction;
201import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor;
202import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor;
203import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.StoreDescriptor;
204
205/**
206 * Basic stand-alone testing of HRegion. No clusters! A lot of the meta information for an HRegion
207 * now lives inside other HRegions or in the HBaseMaster, so only basic testing is possible.
208 */
209@Category({ VerySlowRegionServerTests.class, LargeTests.class })
210@SuppressWarnings("deprecation")
211public class TestHRegion {
212
213  @ClassRule
214  public static final HBaseClassTestRule CLASS_RULE =
215    HBaseClassTestRule.forClass(TestHRegion.class);
216
217  // Do not spin up clusters in here. If you need to spin up a cluster, do it
218  // over in TestHRegionOnCluster.
219  private static final Logger LOG = LoggerFactory.getLogger(TestHRegion.class);
220  @Rule
221  public TestName name = new TestName();
222  @Rule
223  public final ExpectedException thrown = ExpectedException.none();
224
225  private static final String COLUMN_FAMILY = "MyCF";
226  private static final byte[] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);
227  private static final EventLoopGroup GROUP = new NioEventLoopGroup();
228
229  HRegion region = null;
230  // Do not run unit tests in parallel (? Why not? It don't work? Why not? St.Ack)
231  protected static HBaseTestingUtil TEST_UTIL;
232  public static Configuration CONF;
233  private String dir;
234  private final int MAX_VERSIONS = 2;
235
236  // Test names
237  protected TableName tableName;
238  protected String method;
239  protected final byte[] qual = Bytes.toBytes("qual");
240  protected final byte[] qual1 = Bytes.toBytes("qual1");
241  protected final byte[] qual2 = Bytes.toBytes("qual2");
242  protected final byte[] qual3 = Bytes.toBytes("qual3");
243  protected final byte[] value = Bytes.toBytes("value");
244  protected final byte[] value1 = Bytes.toBytes("value1");
245  protected final byte[] value2 = Bytes.toBytes("value2");
246  protected final byte[] row = Bytes.toBytes("rowA");
247  protected final byte[] row2 = Bytes.toBytes("rowB");
248
249  protected final MetricsAssertHelper metricsAssertHelper =
250    CompatibilitySingletonFactory.getInstance(MetricsAssertHelper.class);
251
252  @Before
253  public void setup() throws IOException {
254    TEST_UTIL = new HBaseTestingUtil();
255    CONF = TEST_UTIL.getConfiguration();
256    NettyAsyncFSWALConfigHelper.setEventLoopConfig(CONF, GROUP, NioSocketChannel.class);
257    dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
258    method = name.getMethodName();
259    tableName = TableName.valueOf(method);
260    CONF.set(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, String.valueOf(0.09));
261  }
262
263  @After
264  public void tearDown() throws IOException {
265    // Region may have been closed, but it is still no harm if we close it again here using HTU.
266    HBaseTestingUtil.closeRegionAndWAL(region);
267    EnvironmentEdgeManagerTestHelper.reset();
268    LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
269    TEST_UTIL.cleanupTestDir();
270  }
271
272  /**
273   * Test that I can use the max flushed sequence id after the close. n
274   */
275  @Test
276  public void testSequenceId() throws IOException {
277    region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES);
278    assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId());
279    // Weird. This returns 0 if no store files or no edits. Afraid to change it.
280    assertEquals(0, (long) region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES));
281    HBaseTestingUtil.closeRegionAndWAL(this.region);
282    assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId());
283    assertEquals(0, (long) region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES));
284    // Open region again.
285    region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES);
286    byte[] value = Bytes.toBytes(method);
287    // Make a random put against our cf.
288    Put put = new Put(value);
289    put.addColumn(COLUMN_FAMILY_BYTES, null, value);
290    region.put(put);
291    // No flush yet so init numbers should still be in place.
292    assertEquals(HConstants.NO_SEQNUM, region.getMaxFlushedSeqId());
293    assertEquals(0, (long) region.getMaxStoreSeqId().get(COLUMN_FAMILY_BYTES));
294    region.flush(true);
295    long max = region.getMaxFlushedSeqId();
296    HBaseTestingUtil.closeRegionAndWAL(this.region);
297    assertEquals(max, region.getMaxFlushedSeqId());
298    this.region = null;
299  }
300
301  /**
302   * Test for Bug 2 of HBASE-10466. "Bug 2: Conditions for the first flush of region close
303   * (so-called pre-flush) If memstoreSize is smaller than a certain value, or when region close
304   * starts a flush is ongoing, the first flush is skipped and only the second flush takes place.
305   * However, two flushes are required in case previous flush fails and leaves some data in
306   * snapshot. The bug could cause loss of data in current memstore. The fix is removing all
307   * conditions except abort check so we ensure 2 flushes for region close." n
308   */
309  @Test
310  public void testCloseCarryingSnapshot() throws IOException {
311    region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES);
312    HStore store = region.getStore(COLUMN_FAMILY_BYTES);
313    // Get some random bytes.
314    byte[] value = Bytes.toBytes(method);
315    // Make a random put against our cf.
316    Put put = new Put(value);
317    put.addColumn(COLUMN_FAMILY_BYTES, null, value);
318    // First put something in current memstore, which will be in snapshot after flusher.prepare()
319    region.put(put);
320    StoreFlushContext storeFlushCtx = store.createFlushContext(12345, FlushLifeCycleTracker.DUMMY);
321    storeFlushCtx.prepare();
322    // Second put something in current memstore
323    put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
324    region.put(put);
325    // Close with something in memstore and something in the snapshot. Make sure all is cleared.
326    HBaseTestingUtil.closeRegionAndWAL(region);
327    assertEquals(0, region.getMemStoreDataSize());
328    region = null;
329  }
330
331  /*
332   * This test is for verifying memstore snapshot size is correctly updated in case of rollback See
333   * HBASE-10845
334   */
335  @Test
336  public void testMemstoreSnapshotSize() throws IOException {
337    class MyFaultyFSLog extends FaultyFSLog {
338      StoreFlushContext storeFlushCtx;
339
340      public MyFaultyFSLog(FileSystem fs, Path rootDir, String logName, Configuration conf)
341        throws IOException {
342        super(fs, rootDir, logName, conf);
343      }
344
345      void setStoreFlushCtx(StoreFlushContext storeFlushCtx) {
346        this.storeFlushCtx = storeFlushCtx;
347      }
348
349      @Override
350      protected void doSync(long txid, boolean forceSync) throws IOException {
351        storeFlushCtx.prepare();
352        super.doSync(txid, forceSync);
353      }
354    }
355
356    FileSystem fs = FileSystem.get(CONF);
357    Path rootDir = new Path(dir + "testMemstoreSnapshotSize");
358    MyFaultyFSLog faultyLog = new MyFaultyFSLog(fs, rootDir, "testMemstoreSnapshotSize", CONF);
359    faultyLog.init();
360    region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, faultyLog,
361      COLUMN_FAMILY_BYTES);
362
363    HStore store = region.getStore(COLUMN_FAMILY_BYTES);
364    // Get some random bytes.
365    byte[] value = Bytes.toBytes(method);
366    faultyLog.setStoreFlushCtx(store.createFlushContext(12345, FlushLifeCycleTracker.DUMMY));
367
368    Put put = new Put(value);
369    put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
370    faultyLog.setFailureType(FaultyFSLog.FailureType.SYNC);
371    boolean threwIOE = false;
372    try {
373      region.put(put);
374    } catch (IOException ioe) {
375      threwIOE = true;
376    } finally {
377      assertTrue("The regionserver should have thrown an exception", threwIOE);
378    }
379    MemStoreSize mss = store.getFlushableSize();
380    assertTrue("flushable size should be zero, but it is " + mss, mss.getDataSize() == 0);
381  }
382
383  /**
384   * Create a WAL outside of the usual helper in
385   * {@link HBaseTestingUtil#createWal(Configuration, Path, RegionInfo)} because that method doesn't
386   * play nicely with FaultyFileSystem. Call this method before overriding {@code fs.file.impl}.
387   * @param callingMethod a unique component for the path, probably the name of the test method.
388   */
389  private static WAL createWALCompatibleWithFaultyFileSystem(String callingMethod,
390    Configuration conf, TableName tableName) throws IOException {
391    final Path logDir = TEST_UTIL.getDataTestDirOnTestFS(callingMethod + ".log");
392    final Configuration walConf = new Configuration(conf);
393    CommonFSUtils.setRootDir(walConf, logDir);
394    return new WALFactory(walConf, callingMethod)
395      .getWAL(RegionInfoBuilder.newBuilder(tableName).build());
396  }
397
398  @Test
399  public void testMemstoreSizeAccountingWithFailedPostBatchMutate() throws IOException {
400    String testName = "testMemstoreSizeAccountingWithFailedPostBatchMutate";
401    FileSystem fs = FileSystem.get(CONF);
402    Path rootDir = new Path(dir + testName);
403    FSHLog hLog = new FSHLog(fs, rootDir, testName, CONF);
404    hLog.init();
405    region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, hLog,
406      COLUMN_FAMILY_BYTES);
407    HStore store = region.getStore(COLUMN_FAMILY_BYTES);
408    assertEquals(0, region.getMemStoreDataSize());
409
410    // Put one value
411    byte[] value = Bytes.toBytes(method);
412    Put put = new Put(value);
413    put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
414    region.put(put);
415    long onePutSize = region.getMemStoreDataSize();
416    assertTrue(onePutSize > 0);
417
418    RegionCoprocessorHost mockedCPHost = Mockito.mock(RegionCoprocessorHost.class);
419    doThrow(new IOException()).when(mockedCPHost)
420      .postBatchMutate(Mockito.<MiniBatchOperationInProgress<Mutation>> any());
421    region.setCoprocessorHost(mockedCPHost);
422
423    put = new Put(value);
424    put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("dfg"), value);
425    try {
426      region.put(put);
427      fail("Should have failed with IOException");
428    } catch (IOException expected) {
429    }
430    long expectedSize = onePutSize * 2;
431    assertEquals("memstoreSize should be incremented", expectedSize, region.getMemStoreDataSize());
432    assertEquals("flushable size should be incremented", expectedSize,
433      store.getFlushableSize().getDataSize());
434
435    region.setCoprocessorHost(null);
436  }
437
438  /**
439   * A test case of HBASE-21041
440   * @throws Exception Exception
441   */
442  @Test
443  public void testFlushAndMemstoreSizeCounting() throws Exception {
444    byte[] family = Bytes.toBytes("family");
445    this.region = initHRegion(tableName, method, CONF, family);
446    final WALFactory wals = new WALFactory(CONF, method);
447    try {
448      for (byte[] row : HBaseTestingUtil.ROWS) {
449        Put put = new Put(row);
450        put.addColumn(family, family, row);
451        region.put(put);
452      }
453      region.flush(true);
454      // After flush, data size should be zero
455      assertEquals(0, region.getMemStoreDataSize());
456      // After flush, a new active mutable segment is created, so the heap size
457      // should equal to MutableSegment.DEEP_OVERHEAD
458      assertEquals(MutableSegment.DEEP_OVERHEAD, region.getMemStoreHeapSize());
459      // After flush, offheap should be zero
460      assertEquals(0, region.getMemStoreOffHeapSize());
461    } finally {
462      HBaseTestingUtil.closeRegionAndWAL(this.region);
463      this.region = null;
464      wals.close();
465    }
466  }
467
468  /**
469   * Test we do not lose data if we fail a flush and then close. Part of HBase-10466. Tests the
470   * following from the issue description: "Bug 1: Wrong calculation of HRegion.memstoreSize: When a
471   * flush fails, data to be flushed is kept in each MemStore's snapshot and wait for next flush
472   * attempt to continue on it. But when the next flush succeeds, the counter of total memstore size
473   * in HRegion is always deduced by the sum of current memstore sizes instead of snapshots left
474   * from previous failed flush. This calculation is problematic that almost every time there is
475   * failed flush, HRegion.memstoreSize gets reduced by a wrong value. If region flush could not
476   * proceed for a couple cycles, the size in current memstore could be much larger than the
477   * snapshot. It's likely to drift memstoreSize much smaller than expected. In extreme case, if the
478   * error accumulates to even bigger than HRegion's memstore size limit, any further flush is
479   * skipped because flush does not do anything if memstoreSize is not larger than 0." n
480   */
481  @Test
482  public void testFlushSizeAccounting() throws Exception {
483    final Configuration conf = HBaseConfiguration.create(CONF);
484    final WAL wal = createWALCompatibleWithFaultyFileSystem(method, conf, tableName);
485    // Only retry once.
486    conf.setInt("hbase.hstore.flush.retries.number", 1);
487    final User user = User.createUserForTesting(conf, method, new String[] { "foo" });
488    // Inject our faulty LocalFileSystem
489    conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
490    user.runAs(new PrivilegedExceptionAction<Object>() {
491      @Override
492      public Object run() throws Exception {
493        // Make sure it worked (above is sensitive to caching details in hadoop core)
494        FileSystem fs = FileSystem.get(conf);
495        Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
496        FaultyFileSystem ffs = (FaultyFileSystem) fs;
497        HRegion region = null;
498        try {
499          // Initialize region
500          region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, wal,
501            COLUMN_FAMILY_BYTES);
502          long size = region.getMemStoreDataSize();
503          Assert.assertEquals(0, size);
504          // Put one item into memstore. Measure the size of one item in memstore.
505          Put p1 = new Put(row);
506          p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[]) null));
507          region.put(p1);
508          final long sizeOfOnePut = region.getMemStoreDataSize();
509          // Fail a flush which means the current memstore will hang out as memstore 'snapshot'.
510          try {
511            LOG.info("Flushing");
512            region.flush(true);
513            Assert.fail("Didn't bubble up IOE!");
514          } catch (DroppedSnapshotException dse) {
515            // What we are expecting
516            region.closing.set(false); // this is needed for the rest of the test to work
517          }
518          // Make it so all writes succeed from here on out
519          ffs.fault.set(false);
520          // Check sizes. Should still be the one entry.
521          Assert.assertEquals(sizeOfOnePut, region.getMemStoreDataSize());
522          // Now add two entries so that on this next flush that fails, we can see if we
523          // subtract the right amount, the snapshot size only.
524          Put p2 = new Put(row);
525          p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[]) null));
526          p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[]) null));
527          region.put(p2);
528          long expectedSize = sizeOfOnePut * 3;
529          Assert.assertEquals(expectedSize, region.getMemStoreDataSize());
530          // Do a successful flush. It will clear the snapshot only. Thats how flushes work.
531          // If already a snapshot, we clear it else we move the memstore to be snapshot and flush
532          // it
533          region.flush(true);
534          // Make sure our memory accounting is right.
535          Assert.assertEquals(sizeOfOnePut * 2, region.getMemStoreDataSize());
536        } finally {
537          HBaseTestingUtil.closeRegionAndWAL(region);
538        }
539        return null;
540      }
541    });
542    FileSystem.closeAllForUGI(user.getUGI());
543  }
544
545  @Test
546  public void testCloseWithFailingFlush() throws Exception {
547    final Configuration conf = HBaseConfiguration.create(CONF);
548    final WAL wal = createWALCompatibleWithFaultyFileSystem(method, conf, tableName);
549    // Only retry once.
550    conf.setInt("hbase.hstore.flush.retries.number", 1);
551    final User user = User.createUserForTesting(conf, this.method, new String[] { "foo" });
552    // Inject our faulty LocalFileSystem
553    conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class);
554    user.runAs(new PrivilegedExceptionAction<Object>() {
555      @Override
556      public Object run() throws Exception {
557        // Make sure it worked (above is sensitive to caching details in hadoop core)
558        FileSystem fs = FileSystem.get(conf);
559        Assert.assertEquals(FaultyFileSystem.class, fs.getClass());
560        FaultyFileSystem ffs = (FaultyFileSystem) fs;
561        HRegion region = null;
562        try {
563          // Initialize region
564          region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, wal,
565            COLUMN_FAMILY_BYTES);
566          long size = region.getMemStoreDataSize();
567          Assert.assertEquals(0, size);
568          // Put one item into memstore. Measure the size of one item in memstore.
569          Put p1 = new Put(row);
570          p1.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual1, 1, (byte[]) null));
571          region.put(p1);
572          // Manufacture an outstanding snapshot -- fake a failed flush by doing prepare step only.
573          HStore store = region.getStore(COLUMN_FAMILY_BYTES);
574          StoreFlushContext storeFlushCtx =
575            store.createFlushContext(12345, FlushLifeCycleTracker.DUMMY);
576          storeFlushCtx.prepare();
577          // Now add two entries to the foreground memstore.
578          Put p2 = new Put(row);
579          p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual2, 2, (byte[]) null));
580          p2.add(new KeyValue(row, COLUMN_FAMILY_BYTES, qual3, 3, (byte[]) null));
581          region.put(p2);
582          // Now try close on top of a failing flush.
583          HBaseTestingUtil.closeRegionAndWAL(region);
584          region = null;
585          fail();
586        } catch (DroppedSnapshotException dse) {
587          // Expected
588          LOG.info("Expected DroppedSnapshotException");
589        } finally {
590          // Make it so all writes succeed from here on out so can close clean
591          ffs.fault.set(false);
592          HBaseTestingUtil.closeRegionAndWAL(region);
593        }
594        return null;
595      }
596    });
597    FileSystem.closeAllForUGI(user.getUGI());
598  }
599
600  @Test
601  public void testCompactionAffectedByScanners() throws Exception {
602    byte[] family = Bytes.toBytes("family");
603    this.region = initHRegion(tableName, method, CONF, family);
604
605    Put put = new Put(Bytes.toBytes("r1"));
606    put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
607    region.put(put);
608    region.flush(true);
609
610    Scan scan = new Scan();
611    scan.readVersions(3);
612    // open the first scanner
613    RegionScanner scanner1 = region.getScanner(scan);
614
615    Delete delete = new Delete(Bytes.toBytes("r1"));
616    region.delete(delete);
617    region.flush(true);
618
619    // open the second scanner
620    RegionScanner scanner2 = region.getScanner(scan);
621
622    List<Cell> results = new ArrayList<>();
623
624    System.out.println("Smallest read point:" + region.getSmallestReadPoint());
625
626    // make a major compaction
627    region.compact(true);
628
629    // open the third scanner
630    RegionScanner scanner3 = region.getScanner(scan);
631
632    // get data from scanner 1, 2, 3 after major compaction
633    scanner1.next(results);
634    System.out.println(results);
635    assertEquals(1, results.size());
636
637    results.clear();
638    scanner2.next(results);
639    System.out.println(results);
640    assertEquals(0, results.size());
641
642    results.clear();
643    scanner3.next(results);
644    System.out.println(results);
645    assertEquals(0, results.size());
646  }
647
648  @Test
649  public void testToShowNPEOnRegionScannerReseek() throws Exception {
650    byte[] family = Bytes.toBytes("family");
651    this.region = initHRegion(tableName, method, CONF, family);
652
653    Put put = new Put(Bytes.toBytes("r1"));
654    put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
655    region.put(put);
656    put = new Put(Bytes.toBytes("r2"));
657    put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
658    region.put(put);
659    region.flush(true);
660
661    Scan scan = new Scan();
662    scan.readVersions(3);
663    // open the first scanner
664    RegionScanner scanner1 = region.getScanner(scan);
665
666    System.out.println("Smallest read point:" + region.getSmallestReadPoint());
667
668    region.compact(true);
669
670    scanner1.reseek(Bytes.toBytes("r2"));
671    List<Cell> results = new ArrayList<>();
672    scanner1.next(results);
673    Cell keyValue = results.get(0);
674    Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0);
675    scanner1.close();
676  }
677
678  @Test
679  public void testArchiveRecoveredEditsReplay() throws Exception {
680    byte[] family = Bytes.toBytes("family");
681    this.region = initHRegion(tableName, method, CONF, family);
682    final WALFactory wals = new WALFactory(CONF, method);
683    try {
684      Path regiondir = region.getRegionFileSystem().getRegionDir();
685      FileSystem fs = region.getRegionFileSystem().getFileSystem();
686      byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
687
688      Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir);
689
690      long maxSeqId = 1050;
691      long minSeqId = 1000;
692
693      for (long i = minSeqId; i <= maxSeqId; i += 10) {
694        Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
695        fs.create(recoveredEdits);
696        WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
697
698        long time = System.nanoTime();
699        WALEdit edit = new WALEdit();
700        edit.add(
701          new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i)));
702        writer.append(new WAL.Entry(
703          new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit));
704
705        writer.close();
706      }
707      MonitoredTask status = TaskMonitor.get().createStatus(method);
708      Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);
709      for (HStore store : region.getStores()) {
710        maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId - 1);
711      }
712      CONF.set("hbase.region.archive.recovered.edits", "true");
713      CONF.set(CommonFSUtils.HBASE_WAL_DIR, "/custom_wal_dir");
714      long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status);
715      assertEquals(maxSeqId, seqId);
716      region.getMVCC().advanceTo(seqId);
717      String fakeFamilyName = recoveredEditsDir.getName();
718      Path rootDir = new Path(CONF.get(HConstants.HBASE_DIR));
719      Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePathForRootDir(rootDir,
720        region.getRegionInfo(), Bytes.toBytes(fakeFamilyName));
721      FileStatus[] list = TEST_UTIL.getTestFileSystem().listStatus(storeArchiveDir);
722      assertEquals(6, list.length);
723    } finally {
724      CONF.set("hbase.region.archive.recovered.edits", "false");
725      CONF.set(CommonFSUtils.HBASE_WAL_DIR, "");
726      HBaseTestingUtil.closeRegionAndWAL(this.region);
727      this.region = null;
728      wals.close();
729    }
730  }
731
732  @Test
733  public void testSkipRecoveredEditsReplay() throws Exception {
734    byte[] family = Bytes.toBytes("family");
735    this.region = initHRegion(tableName, method, CONF, family);
736    final WALFactory wals = new WALFactory(CONF, method);
737    try {
738      Path regiondir = region.getRegionFileSystem().getRegionDir();
739      FileSystem fs = region.getRegionFileSystem().getFileSystem();
740      byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
741
742      Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir);
743
744      long maxSeqId = 1050;
745      long minSeqId = 1000;
746
747      for (long i = minSeqId; i <= maxSeqId; i += 10) {
748        Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
749        fs.create(recoveredEdits);
750        WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
751
752        long time = System.nanoTime();
753        WALEdit edit = new WALEdit();
754        edit.add(
755          new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i)));
756        writer.append(new WAL.Entry(
757          new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit));
758
759        writer.close();
760      }
761      MonitoredTask status = TaskMonitor.get().createStatus(method);
762      Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);
763      for (HStore store : region.getStores()) {
764        maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId - 1);
765      }
766      long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status);
767      assertEquals(maxSeqId, seqId);
768      region.getMVCC().advanceTo(seqId);
769      Get get = new Get(row);
770      Result result = region.get(get);
771      for (long i = minSeqId; i <= maxSeqId; i += 10) {
772        List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
773        assertEquals(1, kvs.size());
774        assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
775      }
776    } finally {
777      HBaseTestingUtil.closeRegionAndWAL(this.region);
778      this.region = null;
779      wals.close();
780    }
781  }
782
783  @Test
784  public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
785    byte[] family = Bytes.toBytes("family");
786    this.region = initHRegion(tableName, method, CONF, family);
787    final WALFactory wals = new WALFactory(CONF, method);
788    try {
789      Path regiondir = region.getRegionFileSystem().getRegionDir();
790      FileSystem fs = region.getRegionFileSystem().getFileSystem();
791      byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
792
793      Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir);
794
795      long maxSeqId = 1050;
796      long minSeqId = 1000;
797
798      for (long i = minSeqId; i <= maxSeqId; i += 10) {
799        Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
800        fs.create(recoveredEdits);
801        WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
802
803        long time = System.nanoTime();
804        WALEdit edit = new WALEdit();
805        edit.add(
806          new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i)));
807        writer.append(new WAL.Entry(
808          new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit));
809
810        writer.close();
811      }
812      long recoverSeqId = 1030;
813      MonitoredTask status = TaskMonitor.get().createStatus(method);
814      Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);
815      for (HStore store : region.getStores()) {
816        maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), recoverSeqId - 1);
817      }
818      long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status);
819      assertEquals(maxSeqId, seqId);
820      region.getMVCC().advanceTo(seqId);
821      Get get = new Get(row);
822      Result result = region.get(get);
823      for (long i = minSeqId; i <= maxSeqId; i += 10) {
824        List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
825        if (i < recoverSeqId) {
826          assertEquals(0, kvs.size());
827        } else {
828          assertEquals(1, kvs.size());
829          assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
830        }
831      }
832    } finally {
833      HBaseTestingUtil.closeRegionAndWAL(this.region);
834      this.region = null;
835      wals.close();
836    }
837  }
838
839  @Test
840  public void testSkipRecoveredEditsReplayAllIgnored() throws Exception {
841    byte[] family = Bytes.toBytes("family");
842    this.region = initHRegion(tableName, method, CONF, family);
843    Path regiondir = region.getRegionFileSystem().getRegionDir();
844    FileSystem fs = region.getRegionFileSystem().getFileSystem();
845
846    Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir);
847    for (int i = 1000; i < 1050; i += 10) {
848      Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
849      FSDataOutputStream dos = fs.create(recoveredEdits);
850      dos.writeInt(i);
851      dos.close();
852    }
853    long minSeqId = 2000;
854    Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1));
855    FSDataOutputStream dos = fs.create(recoveredEdits);
856    dos.close();
857
858    Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);
859    for (HStore store : region.getStores()) {
860      maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId);
861    }
862    long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, null);
863    assertEquals(minSeqId, seqId);
864  }
865
866  @Test
867  public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception {
868    byte[] family = Bytes.toBytes("family");
869    this.region = initHRegion(tableName, method, CONF, family);
870    final WALFactory wals = new WALFactory(CONF, method);
871    try {
872      Path regiondir = region.getRegionFileSystem().getRegionDir();
873      FileSystem fs = region.getRegionFileSystem().getFileSystem();
874      byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
875      byte[][] columns = region.getTableDescriptor().getColumnFamilyNames().toArray(new byte[0][]);
876
877      assertEquals(0, region.getStoreFileList(columns).size());
878
879      Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir);
880
881      long maxSeqId = 1050;
882      long minSeqId = 1000;
883
884      for (long i = minSeqId; i <= maxSeqId; i += 10) {
885        Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
886        fs.create(recoveredEdits);
887        WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
888
889        long time = System.nanoTime();
890        WALEdit edit = null;
891        if (i == maxSeqId) {
892          edit = WALEdit.createCompaction(region.getRegionInfo(),
893            CompactionDescriptor.newBuilder().setTableName(ByteString.copyFrom(tableName.getName()))
894              .setFamilyName(ByteString.copyFrom(regionName))
895              .setEncodedRegionName(ByteString.copyFrom(regionName))
896              .setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString())))
897              .setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName())).build());
898        } else {
899          edit = new WALEdit();
900          edit.add(
901            new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i)));
902        }
903        writer.append(new WAL.Entry(
904          new WALKeyImpl(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit));
905        writer.close();
906      }
907
908      long recoverSeqId = 1030;
909      Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);
910      MonitoredTask status = TaskMonitor.get().createStatus(method);
911      for (HStore store : region.getStores()) {
912        maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), recoverSeqId - 1);
913      }
914      long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, status);
915      assertEquals(maxSeqId, seqId);
916
917      // assert that the files are flushed
918      assertEquals(1, region.getStoreFileList(columns).size());
919
920    } finally {
921      HBaseTestingUtil.closeRegionAndWAL(this.region);
922      this.region = null;
923      wals.close();
924    }
925  }
926
927  @Test
928  public void testRecoveredEditsReplayCompaction() throws Exception {
929    testRecoveredEditsReplayCompaction(false);
930    testRecoveredEditsReplayCompaction(true);
931  }
932
933  public void testRecoveredEditsReplayCompaction(boolean mismatchedRegionName) throws Exception {
934    CONF.setClass(HConstants.REGION_IMPL, HRegionForTesting.class, Region.class);
935    byte[] family = Bytes.toBytes("family");
936    this.region = initHRegion(tableName, method, CONF, family);
937    final WALFactory wals = new WALFactory(CONF, method);
938    try {
939      Path regiondir = region.getRegionFileSystem().getRegionDir();
940      FileSystem fs = region.getRegionFileSystem().getFileSystem();
941      byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
942
943      long maxSeqId = 3;
944      long minSeqId = 0;
945
946      for (long i = minSeqId; i < maxSeqId; i++) {
947        Put put = new Put(Bytes.toBytes(i));
948        put.addColumn(family, Bytes.toBytes(i), Bytes.toBytes(i));
949        region.put(put);
950        region.flush(true);
951      }
952
953      // this will create a region with 3 files
954      assertEquals(3, region.getStore(family).getStorefilesCount());
955      List<Path> storeFiles = new ArrayList<>(3);
956      for (HStoreFile sf : region.getStore(family).getStorefiles()) {
957        storeFiles.add(sf.getPath());
958      }
959
960      // disable compaction completion
961      CONF.setBoolean("hbase.hstore.compaction.complete", false);
962      region.compactStores();
963
964      // ensure that nothing changed
965      assertEquals(3, region.getStore(family).getStorefilesCount());
966
967      // now find the compacted file, and manually add it to the recovered edits
968      Path tmpDir = new Path(region.getRegionFileSystem().getTempDir(), Bytes.toString(family));
969      FileStatus[] files = CommonFSUtils.listStatus(fs, tmpDir);
970      String errorMsg = "Expected to find 1 file in the region temp directory "
971        + "from the compaction, could not find any";
972      assertNotNull(errorMsg, files);
973      assertEquals(errorMsg, 1, files.length);
974      // move the file inside region dir
975      Path newFile =
976        region.getRegionFileSystem().commitStoreFile(Bytes.toString(family), files[0].getPath());
977
978      byte[] encodedNameAsBytes = this.region.getRegionInfo().getEncodedNameAsBytes();
979      byte[] fakeEncodedNameAsBytes = new byte[encodedNameAsBytes.length];
980      for (int i = 0; i < encodedNameAsBytes.length; i++) {
981        // Mix the byte array to have a new encodedName
982        fakeEncodedNameAsBytes[i] = (byte) (encodedNameAsBytes[i] + 1);
983      }
984
985      CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(
986        this.region.getRegionInfo(), mismatchedRegionName ? fakeEncodedNameAsBytes : null, family,
987        storeFiles, Lists.newArrayList(newFile),
988        region.getRegionFileSystem().getStoreDir(Bytes.toString(family)));
989
990      WALUtil.writeCompactionMarker(region.getWAL(), this.region.getReplicationScope(),
991        this.region.getRegionInfo(), compactionDescriptor, region.getMVCC(), null);
992
993      Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir);
994
995      Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
996      fs.create(recoveredEdits);
997      WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
998
999      long time = System.nanoTime();
1000
1001      writer.append(new WAL.Entry(
1002        new WALKeyImpl(regionName, tableName, 10, time, HConstants.DEFAULT_CLUSTER_ID),
1003        WALEdit.createCompaction(region.getRegionInfo(), compactionDescriptor)));
1004      writer.close();
1005
1006      // close the region now, and reopen again
1007      region.getTableDescriptor();
1008      region.getRegionInfo();
1009      HBaseTestingUtil.closeRegionAndWAL(this.region);
1010      try {
1011        region = HRegion.openHRegion(region, null);
1012      } catch (WrongRegionException wre) {
1013        fail("Matching encoded region name should not have produced WrongRegionException");
1014      }
1015
1016      // now check whether we have only one store file, the compacted one
1017      Collection<HStoreFile> sfs = region.getStore(family).getStorefiles();
1018      for (HStoreFile sf : sfs) {
1019        LOG.info(Objects.toString(sf.getPath()));
1020      }
1021      if (!mismatchedRegionName) {
1022        assertEquals(1, region.getStore(family).getStorefilesCount());
1023      }
1024      files = CommonFSUtils.listStatus(fs, tmpDir);
1025      assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);
1026
1027      for (long i = minSeqId; i < maxSeqId; i++) {
1028        Get get = new Get(Bytes.toBytes(i));
1029        Result result = region.get(get);
1030        byte[] value = result.getValue(family, Bytes.toBytes(i));
1031        assertArrayEquals(Bytes.toBytes(i), value);
1032      }
1033    } finally {
1034      HBaseTestingUtil.closeRegionAndWAL(this.region);
1035      this.region = null;
1036      wals.close();
1037      CONF.setClass(HConstants.REGION_IMPL, HRegion.class, Region.class);
1038    }
1039  }
1040
1041  @Test
1042  public void testFlushMarkers() throws Exception {
1043    // tests that flush markers are written to WAL and handled at recovered edits
1044    byte[] family = Bytes.toBytes("family");
1045    Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + ".log");
1046    final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration());
1047    CommonFSUtils.setRootDir(walConf, logDir);
1048    final WALFactory wals = new WALFactory(walConf, method);
1049    final WAL wal = wals.getWAL(RegionInfoBuilder.newBuilder(tableName).build());
1050
1051    this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF,
1052      false, Durability.USE_DEFAULT, wal, family);
1053    try {
1054      Path regiondir = region.getRegionFileSystem().getRegionDir();
1055      FileSystem fs = region.getRegionFileSystem().getFileSystem();
1056      byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
1057
1058      long maxSeqId = 3;
1059      long minSeqId = 0;
1060
1061      for (long i = minSeqId; i < maxSeqId; i++) {
1062        Put put = new Put(Bytes.toBytes(i));
1063        put.addColumn(family, Bytes.toBytes(i), Bytes.toBytes(i));
1064        region.put(put);
1065        region.flush(true);
1066      }
1067
1068      // this will create a region with 3 files from flush
1069      assertEquals(3, region.getStore(family).getStorefilesCount());
1070      List<String> storeFiles = new ArrayList<>(3);
1071      for (HStoreFile sf : region.getStore(family).getStorefiles()) {
1072        storeFiles.add(sf.getPath().getName());
1073      }
1074
1075      // now verify that the flush markers are written
1076      wal.shutdown();
1077      WAL.Reader reader = WALFactory.createReader(fs, AbstractFSWALProvider.getCurrentFileName(wal),
1078        TEST_UTIL.getConfiguration());
1079      try {
1080        List<WAL.Entry> flushDescriptors = new ArrayList<>();
1081        long lastFlushSeqId = -1;
1082        while (true) {
1083          WAL.Entry entry = reader.next();
1084          if (entry == null) {
1085            break;
1086          }
1087          Cell cell = entry.getEdit().getCells().get(0);
1088          if (WALEdit.isMetaEditFamily(cell)) {
1089            FlushDescriptor flushDesc = WALEdit.getFlushDescriptor(cell);
1090            assertNotNull(flushDesc);
1091            assertArrayEquals(tableName.getName(), flushDesc.getTableName().toByteArray());
1092            if (flushDesc.getAction() == FlushAction.START_FLUSH) {
1093              assertTrue(flushDesc.getFlushSequenceNumber() > lastFlushSeqId);
1094            } else if (flushDesc.getAction() == FlushAction.COMMIT_FLUSH) {
1095              assertTrue(flushDesc.getFlushSequenceNumber() == lastFlushSeqId);
1096            }
1097            lastFlushSeqId = flushDesc.getFlushSequenceNumber();
1098            assertArrayEquals(regionName, flushDesc.getEncodedRegionName().toByteArray());
1099            assertEquals(1, flushDesc.getStoreFlushesCount()); // only one store
1100            StoreFlushDescriptor storeFlushDesc = flushDesc.getStoreFlushes(0);
1101            assertArrayEquals(family, storeFlushDesc.getFamilyName().toByteArray());
1102            assertEquals("family", storeFlushDesc.getStoreHomeDir());
1103            if (flushDesc.getAction() == FlushAction.START_FLUSH) {
1104              assertEquals(0, storeFlushDesc.getFlushOutputCount());
1105            } else {
1106              assertEquals(1, storeFlushDesc.getFlushOutputCount()); // only one file from flush
1107              assertTrue(storeFiles.contains(storeFlushDesc.getFlushOutput(0)));
1108            }
1109
1110            flushDescriptors.add(entry);
1111          }
1112        }
1113
1114        assertEquals(3 * 2, flushDescriptors.size()); // START_FLUSH and COMMIT_FLUSH per flush
1115
1116        // now write those markers to the recovered edits again.
1117
1118        Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir);
1119
1120        Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
1121        fs.create(recoveredEdits);
1122        WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
1123
1124        for (WAL.Entry entry : flushDescriptors) {
1125          writer.append(entry);
1126        }
1127        writer.close();
1128      } finally {
1129        if (null != reader) {
1130          try {
1131            reader.close();
1132          } catch (IOException exception) {
1133            LOG.warn("Problem closing wal: " + exception.getMessage());
1134            LOG.debug("exception details", exception);
1135          }
1136        }
1137      }
1138
1139      // close the region now, and reopen again
1140      HBaseTestingUtil.closeRegionAndWAL(this.region);
1141      region = HRegion.openHRegion(region, null);
1142
1143      // now check whether we have can read back the data from region
1144      for (long i = minSeqId; i < maxSeqId; i++) {
1145        Get get = new Get(Bytes.toBytes(i));
1146        Result result = region.get(get);
1147        byte[] value = result.getValue(family, Bytes.toBytes(i));
1148        assertArrayEquals(Bytes.toBytes(i), value);
1149      }
1150    } finally {
1151      HBaseTestingUtil.closeRegionAndWAL(this.region);
1152      this.region = null;
1153      wals.close();
1154    }
1155  }
1156
1157  static class IsFlushWALMarker implements ArgumentMatcher<WALEdit> {
1158    volatile FlushAction[] actions;
1159
1160    public IsFlushWALMarker(FlushAction... actions) {
1161      this.actions = actions;
1162    }
1163
1164    @Override
1165    public boolean matches(WALEdit edit) {
1166      List<Cell> cells = edit.getCells();
1167      if (cells.isEmpty()) {
1168        return false;
1169      }
1170      if (WALEdit.isMetaEditFamily(cells.get(0))) {
1171        FlushDescriptor desc;
1172        try {
1173          desc = WALEdit.getFlushDescriptor(cells.get(0));
1174        } catch (IOException e) {
1175          LOG.warn(e.toString(), e);
1176          return false;
1177        }
1178        if (desc != null) {
1179          for (FlushAction action : actions) {
1180            if (desc.getAction() == action) {
1181              return true;
1182            }
1183          }
1184        }
1185      }
1186      return false;
1187    }
1188
1189    public IsFlushWALMarker set(FlushAction... actions) {
1190      this.actions = actions;
1191      return this;
1192    }
1193  }
1194
1195  @Test
1196  public void testFlushMarkersWALFail() throws Exception {
1197    // test the cases where the WAL append for flush markers fail.
1198    byte[] family = Bytes.toBytes("family");
1199
1200    // spy an actual WAL implementation to throw exception (was not able to mock)
1201    Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + "log");
1202
1203    final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration());
1204    CommonFSUtils.setRootDir(walConf, logDir);
1205    // Make up a WAL that we can manipulate at append time.
1206    class FailAppendFlushMarkerWAL extends FSHLog {
1207      volatile FlushAction[] flushActions = null;
1208
1209      public FailAppendFlushMarkerWAL(FileSystem fs, Path root, String logDir, Configuration conf)
1210        throws IOException {
1211        super(fs, root, logDir, conf);
1212      }
1213
1214      @Override
1215      protected Writer createWriterInstance(Path path) throws IOException {
1216        final Writer w = super.createWriterInstance(path);
1217        return new Writer() {
1218          @Override
1219          public void close() throws IOException {
1220            w.close();
1221          }
1222
1223          @Override
1224          public void sync(boolean forceSync) throws IOException {
1225            w.sync(forceSync);
1226          }
1227
1228          @Override
1229          public void append(Entry entry) throws IOException {
1230            List<Cell> cells = entry.getEdit().getCells();
1231            if (WALEdit.isMetaEditFamily(cells.get(0))) {
1232              FlushDescriptor desc = WALEdit.getFlushDescriptor(cells.get(0));
1233              if (desc != null) {
1234                for (FlushAction flushAction : flushActions) {
1235                  if (desc.getAction().equals(flushAction)) {
1236                    throw new IOException("Failed to append flush marker! " + flushAction);
1237                  }
1238                }
1239              }
1240            }
1241            w.append(entry);
1242          }
1243
1244          @Override
1245          public long getLength() {
1246            return w.getLength();
1247          }
1248
1249          @Override
1250          public long getSyncedLength() {
1251            return w.getSyncedLength();
1252          }
1253        };
1254      }
1255    }
1256    FailAppendFlushMarkerWAL wal = new FailAppendFlushMarkerWAL(FileSystem.get(walConf),
1257      CommonFSUtils.getRootDir(walConf), method, walConf);
1258    wal.init();
1259    this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF,
1260      false, Durability.USE_DEFAULT, wal, family);
1261    int i = 0;
1262    Put put = new Put(Bytes.toBytes(i));
1263    put.setDurability(Durability.SKIP_WAL); // have to skip mocked wal
1264    put.addColumn(family, Bytes.toBytes(i), Bytes.toBytes(i));
1265    region.put(put);
1266
1267    // 1. Test case where START_FLUSH throws exception
1268    wal.flushActions = new FlushAction[] { FlushAction.START_FLUSH };
1269
1270    // start cache flush will throw exception
1271    try {
1272      region.flush(true);
1273      fail("This should have thrown exception");
1274    } catch (DroppedSnapshotException unexpected) {
1275      // this should not be a dropped snapshot exception. Meaning that RS will not abort
1276      throw unexpected;
1277    } catch (IOException expected) {
1278      // expected
1279    }
1280    // The WAL is hosed now. It has two edits appended. We cannot roll the log without it
1281    // throwing a DroppedSnapshotException to force an abort. Just clean up the mess.
1282    region.close(true);
1283    wal.close();
1284
1285    // 2. Test case where START_FLUSH succeeds but COMMIT_FLUSH will throw exception
1286    wal.flushActions = new FlushAction[] { FlushAction.COMMIT_FLUSH };
1287    wal = new FailAppendFlushMarkerWAL(FileSystem.get(walConf), CommonFSUtils.getRootDir(walConf),
1288      method, walConf);
1289    wal.init();
1290    this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF,
1291      false, Durability.USE_DEFAULT, wal, family);
1292    region.put(put);
1293    // 3. Test case where ABORT_FLUSH will throw exception.
1294    // Even if ABORT_FLUSH throws exception, we should not fail with IOE, but continue with
1295    // DroppedSnapshotException. Below COMMIT_FLUSH will cause flush to abort
1296    wal.flushActions = new FlushAction[] { FlushAction.COMMIT_FLUSH, FlushAction.ABORT_FLUSH };
1297
1298    try {
1299      region.flush(true);
1300      fail("This should have thrown exception");
1301    } catch (DroppedSnapshotException expected) {
1302      // we expect this exception, since we were able to write the snapshot, but failed to
1303      // write the flush marker to WAL
1304    } catch (IOException unexpected) {
1305      throw unexpected;
1306    }
1307  }
1308
1309  @Test
1310  public void testGetWhileRegionClose() throws IOException {
1311    Configuration hc = initSplit();
1312    int numRows = 100;
1313    byte[][] families = { fam1, fam2, fam3 };
1314
1315    // Setting up region
1316    this.region = initHRegion(tableName, method, hc, families);
1317    // Put data in region
1318    final int startRow = 100;
1319    putData(startRow, numRows, qual1, families);
1320    putData(startRow, numRows, qual2, families);
1321    putData(startRow, numRows, qual3, families);
1322    final AtomicBoolean done = new AtomicBoolean(false);
1323    final AtomicInteger gets = new AtomicInteger(0);
1324    GetTillDoneOrException[] threads = new GetTillDoneOrException[10];
1325    try {
1326      // Set ten threads running concurrently getting from the region.
1327      for (int i = 0; i < threads.length / 2; i++) {
1328        threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
1329        threads[i].setDaemon(true);
1330        threads[i].start();
1331      }
1332      // Artificially make the condition by setting closing flag explicitly.
1333      // I can't make the issue happen with a call to region.close().
1334      this.region.closing.set(true);
1335      for (int i = threads.length / 2; i < threads.length; i++) {
1336        threads[i] = new GetTillDoneOrException(i, Bytes.toBytes("" + startRow), done, gets);
1337        threads[i].setDaemon(true);
1338        threads[i].start();
1339      }
1340    } finally {
1341      if (this.region != null) {
1342        HBaseTestingUtil.closeRegionAndWAL(this.region);
1343        this.region = null;
1344      }
1345    }
1346    done.set(true);
1347    for (GetTillDoneOrException t : threads) {
1348      try {
1349        t.join();
1350      } catch (InterruptedException e) {
1351        e.printStackTrace();
1352      }
1353      if (t.e != null) {
1354        LOG.info("Exception=" + t.e);
1355        assertFalse("Found a NPE in " + t.getName(), t.e instanceof NullPointerException);
1356      }
1357    }
1358  }
1359
1360  /*
1361   * Thread that does get on single row until 'done' flag is flipped. If an exception causes us to
1362   * fail, it records it.
1363   */
1364  class GetTillDoneOrException extends Thread {
1365    private final Get g;
1366    private final AtomicBoolean done;
1367    private final AtomicInteger count;
1368    private Exception e;
1369
1370    GetTillDoneOrException(final int i, final byte[] r, final AtomicBoolean d,
1371      final AtomicInteger c) {
1372      super("getter." + i);
1373      this.g = new Get(r);
1374      this.done = d;
1375      this.count = c;
1376    }
1377
1378    @Override
1379    public void run() {
1380      while (!this.done.get()) {
1381        try {
1382          assertTrue(region.get(g).size() > 0);
1383          this.count.incrementAndGet();
1384        } catch (Exception e) {
1385          this.e = e;
1386          break;
1387        }
1388      }
1389    }
1390  }
1391
1392  /*
1393   * An involved filter test. Has multiple column families and deletes in mix.
1394   */
1395  @Test
1396  public void testWeirdCacheBehaviour() throws Exception {
1397    final TableName tableName = TableName.valueOf(name.getMethodName());
1398    byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), Bytes.toBytes("trans-type"),
1399      Bytes.toBytes("trans-date"), Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
1400    this.region = initHRegion(tableName, method, CONF, FAMILIES);
1401    String value = "this is the value";
1402    String value2 = "this is some other value";
1403    String keyPrefix1 = "prefix1";
1404    String keyPrefix2 = "prefix2";
1405    String keyPrefix3 = "prefix3";
1406    putRows(this.region, 3, value, keyPrefix1);
1407    putRows(this.region, 3, value, keyPrefix2);
1408    putRows(this.region, 3, value, keyPrefix3);
1409    putRows(this.region, 3, value2, keyPrefix1);
1410    putRows(this.region, 3, value2, keyPrefix2);
1411    putRows(this.region, 3, value2, keyPrefix3);
1412    System.out.println("Checking values for key: " + keyPrefix1);
1413    assertEquals("Got back incorrect number of rows from scan", 3,
1414      getNumberOfRows(keyPrefix1, value2, this.region));
1415    System.out.println("Checking values for key: " + keyPrefix2);
1416    assertEquals("Got back incorrect number of rows from scan", 3,
1417      getNumberOfRows(keyPrefix2, value2, this.region));
1418    System.out.println("Checking values for key: " + keyPrefix3);
1419    assertEquals("Got back incorrect number of rows from scan", 3,
1420      getNumberOfRows(keyPrefix3, value2, this.region));
1421    deleteColumns(this.region, value2, keyPrefix1);
1422    deleteColumns(this.region, value2, keyPrefix2);
1423    deleteColumns(this.region, value2, keyPrefix3);
1424    System.out.println("Starting important checks.....");
1425    assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, 0,
1426      getNumberOfRows(keyPrefix1, value2, this.region));
1427    assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, 0,
1428      getNumberOfRows(keyPrefix2, value2, this.region));
1429    assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, 0,
1430      getNumberOfRows(keyPrefix3, value2, this.region));
1431  }
1432
1433  @Test
1434  public void testAppendWithReadOnlyTable() throws Exception {
1435    final TableName tableName = TableName.valueOf(name.getMethodName());
1436    this.region = initHRegion(tableName, method, CONF, true, Bytes.toBytes("somefamily"));
1437    boolean exceptionCaught = false;
1438    Append append = new Append(Bytes.toBytes("somerow"));
1439    append.setDurability(Durability.SKIP_WAL);
1440    append.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"),
1441      Bytes.toBytes("somevalue"));
1442    try {
1443      region.append(append);
1444    } catch (IOException e) {
1445      exceptionCaught = true;
1446    }
1447    assertTrue(exceptionCaught == true);
1448  }
1449
1450  @Test
1451  public void testIncrWithReadOnlyTable() throws Exception {
1452    final TableName tableName = TableName.valueOf(name.getMethodName());
1453    this.region = initHRegion(tableName, method, CONF, true, Bytes.toBytes("somefamily"));
1454    boolean exceptionCaught = false;
1455    Increment inc = new Increment(Bytes.toBytes("somerow"));
1456    inc.setDurability(Durability.SKIP_WAL);
1457    inc.addColumn(Bytes.toBytes("somefamily"), Bytes.toBytes("somequalifier"), 1L);
1458    try {
1459      region.increment(inc);
1460    } catch (IOException e) {
1461      exceptionCaught = true;
1462    }
1463    assertTrue(exceptionCaught == true);
1464  }
1465
1466  private void deleteColumns(HRegion r, String value, String keyPrefix) throws IOException {
1467    InternalScanner scanner = buildScanner(keyPrefix, value, r);
1468    int count = 0;
1469    boolean more = false;
1470    List<Cell> results = new ArrayList<>();
1471    do {
1472      more = scanner.next(results);
1473      if (results != null && !results.isEmpty()) count++;
1474      else break;
1475      Delete delete = new Delete(CellUtil.cloneRow(results.get(0)));
1476      delete.addColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
1477      r.delete(delete);
1478      results.clear();
1479    } while (more);
1480    assertEquals("Did not perform correct number of deletes", 3, count);
1481  }
1482
1483  private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
1484    InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
1485    int numberOfResults = 0;
1486    List<Cell> results = new ArrayList<>();
1487    boolean more = false;
1488    do {
1489      more = resultScanner.next(results);
1490      if (results != null && !results.isEmpty()) numberOfResults++;
1491      else break;
1492      for (Cell kv : results) {
1493        System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv)));
1494      }
1495      results.clear();
1496    } while (more);
1497    return numberOfResults;
1498  }
1499
1500  private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
1501    throws IOException {
1502    // Defaults FilterList.Operator.MUST_PASS_ALL.
1503    FilterList allFilters = new FilterList();
1504    allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
1505    // Only return rows where this column value exists in the row.
1506    SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
1507      Bytes.toBytes("qual2"), CompareOperator.EQUAL, Bytes.toBytes(value));
1508    filter.setFilterIfMissing(true);
1509    allFilters.addFilter(filter);
1510    Scan scan = new Scan();
1511    scan.addFamily(Bytes.toBytes("trans-blob"));
1512    scan.addFamily(Bytes.toBytes("trans-type"));
1513    scan.addFamily(Bytes.toBytes("trans-date"));
1514    scan.addFamily(Bytes.toBytes("trans-tags"));
1515    scan.addFamily(Bytes.toBytes("trans-group"));
1516    scan.setFilter(allFilters);
1517    return r.getScanner(scan);
1518  }
1519
1520  private void putRows(HRegion r, int numRows, String value, String key) throws IOException {
1521    for (int i = 0; i < numRows; i++) {
1522      String row = key + "_" + i/* UUID.randomUUID().toString() */;
1523      System.out.println(String.format("Saving row: %s, with value %s", row, value));
1524      Put put = new Put(Bytes.toBytes(row));
1525      put.setDurability(Durability.SKIP_WAL);
1526      put.addColumn(Bytes.toBytes("trans-blob"), null, Bytes.toBytes("value for blob"));
1527      put.addColumn(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
1528      put.addColumn(Bytes.toBytes("trans-date"), null, Bytes.toBytes("20090921010101999"));
1529      put.addColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes.toBytes(value));
1530      put.addColumn(Bytes.toBytes("trans-group"), null, Bytes.toBytes("adhocTransactionGroupId"));
1531      r.put(put);
1532    }
1533  }
1534
1535  @Test
1536  public void testFamilyWithAndWithoutColon() throws Exception {
1537    byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
1538    this.region = initHRegion(tableName, method, CONF, cf);
1539    Put p = new Put(tableName.toBytes());
1540    byte[] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
1541    p.addColumn(cfwithcolon, cfwithcolon, cfwithcolon);
1542    boolean exception = false;
1543    try {
1544      this.region.put(p);
1545    } catch (NoSuchColumnFamilyException e) {
1546      exception = true;
1547    }
1548    assertTrue(exception);
1549  }
1550
1551  @Test
1552  public void testBatchPut_whileNoRowLocksHeld() throws IOException {
1553    final Put[] puts = new Put[10];
1554    MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1555    long syncs = prepareRegionForBachPut(puts, source, false);
1556
1557    OperationStatus[] codes = this.region.batchMutate(puts);
1558    assertEquals(10, codes.length);
1559    for (int i = 0; i < 10; i++) {
1560      assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
1561    }
1562    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
1563
1564    LOG.info("Next a batch put with one invalid family");
1565    puts[5].addColumn(Bytes.toBytes("BAD_CF"), qual, value);
1566    codes = this.region.batchMutate(puts);
1567    assertEquals(10, codes.length);
1568    for (int i = 0; i < 10; i++) {
1569      assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1570        codes[i].getOperationStatusCode());
1571    }
1572
1573    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
1574  }
1575
1576  @Test
1577  public void testBatchPut_whileMultipleRowLocksHeld() throws Exception {
1578    final Put[] puts = new Put[10];
1579    MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1580    long syncs = prepareRegionForBachPut(puts, source, false);
1581
1582    puts[5].addColumn(Bytes.toBytes("BAD_CF"), qual, value);
1583
1584    LOG.info("batchPut will have to break into four batches to avoid row locks");
1585    RowLock rowLock1 = region.getRowLock(Bytes.toBytes("row_2"));
1586    RowLock rowLock2 = region.getRowLock(Bytes.toBytes("row_1"));
1587    RowLock rowLock3 = region.getRowLock(Bytes.toBytes("row_3"));
1588    RowLock rowLock4 = region.getRowLock(Bytes.toBytes("row_3"), true);
1589
1590    MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
1591    final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<>();
1592    final CountDownLatch startingPuts = new CountDownLatch(1);
1593    final CountDownLatch startingClose = new CountDownLatch(1);
1594    TestThread putter = new TestThread(ctx) {
1595      @Override
1596      public void doWork() throws IOException {
1597        startingPuts.countDown();
1598        retFromThread.set(region.batchMutate(puts));
1599      }
1600    };
1601    LOG.info("...starting put thread while holding locks");
1602    ctx.addThread(putter);
1603    ctx.startThreads();
1604
1605    // Now attempt to close the region from another thread. Prior to HBASE-12565
1606    // this would cause the in-progress batchMutate operation to to fail with
1607    // exception because it use to release and re-acquire the close-guard lock
1608    // between batches. Caller then didn't get status indicating which writes succeeded.
1609    // We now expect this thread to block until the batchMutate call finishes.
1610    Thread regionCloseThread = new TestThread(ctx) {
1611      @Override
1612      public void doWork() {
1613        try {
1614          startingPuts.await();
1615          // Give some time for the batch mutate to get in.
1616          // We don't want to race with the mutate
1617          Thread.sleep(10);
1618          startingClose.countDown();
1619          HBaseTestingUtil.closeRegionAndWAL(region);
1620          region = null;
1621        } catch (IOException e) {
1622          throw new RuntimeException(e);
1623        } catch (InterruptedException e) {
1624          throw new RuntimeException(e);
1625        }
1626      }
1627    };
1628    regionCloseThread.start();
1629
1630    startingClose.await();
1631    startingPuts.await();
1632    Thread.sleep(100);
1633    LOG.info("...releasing row lock 1, which should let put thread continue");
1634    rowLock1.release();
1635    rowLock2.release();
1636    rowLock3.release();
1637    waitForCounter(source, "syncTimeNumOps", syncs + 1);
1638
1639    LOG.info("...joining on put thread");
1640    ctx.stop();
1641    regionCloseThread.join();
1642
1643    OperationStatus[] codes = retFromThread.get();
1644    for (int i = 0; i < codes.length; i++) {
1645      assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
1646        codes[i].getOperationStatusCode());
1647    }
1648    rowLock4.release();
1649  }
1650
1651  private void waitForCounter(MetricsWALSource source, String metricName, long expectedCount)
1652    throws InterruptedException {
1653    long startWait = EnvironmentEdgeManager.currentTime();
1654    long currentCount;
1655    while ((currentCount = metricsAssertHelper.getCounter(metricName, source)) < expectedCount) {
1656      Thread.sleep(100);
1657      if (EnvironmentEdgeManager.currentTime() - startWait > 10000) {
1658        fail(String.format("Timed out waiting for '%s' >= '%s', currentCount=%s", metricName,
1659          expectedCount, currentCount));
1660      }
1661    }
1662  }
1663
1664  @Test
1665  public void testAtomicBatchPut() throws IOException {
1666    final Put[] puts = new Put[10];
1667    MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1668    long syncs = prepareRegionForBachPut(puts, source, false);
1669
1670    // 1. Straight forward case, should succeed
1671    OperationStatus[] codes = this.region.batchMutate(puts, true);
1672    assertEquals(10, codes.length);
1673    for (int i = 0; i < 10; i++) {
1674      assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
1675    }
1676    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
1677
1678    // 2. Failed to get lock
1679    RowLock lock = region.getRowLock(Bytes.toBytes("row_" + 3));
1680    // Method {@link HRegion#getRowLock(byte[])} is reentrant. As 'row_3' is locked in this
1681    // thread, need to run {@link HRegion#batchMutate(HRegion.BatchOperation)} in different thread
1682    MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
1683    final AtomicReference<IOException> retFromThread = new AtomicReference<>();
1684    final CountDownLatch finishedPuts = new CountDownLatch(1);
1685    TestThread putter = new TestThread(ctx) {
1686      @Override
1687      public void doWork() throws IOException {
1688        try {
1689          region.batchMutate(puts, true);
1690        } catch (IOException ioe) {
1691          LOG.error("test failed!", ioe);
1692          retFromThread.set(ioe);
1693        }
1694        finishedPuts.countDown();
1695      }
1696    };
1697    LOG.info("...starting put thread while holding locks");
1698    ctx.addThread(putter);
1699    ctx.startThreads();
1700    LOG.info("...waiting for batch puts while holding locks");
1701    try {
1702      finishedPuts.await();
1703    } catch (InterruptedException e) {
1704      LOG.error("Interrupted!", e);
1705    } finally {
1706      if (lock != null) {
1707        lock.release();
1708      }
1709    }
1710    assertNotNull(retFromThread.get());
1711    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);
1712
1713    // 3. Exception thrown in validation
1714    LOG.info("Next a batch put with one invalid family");
1715    puts[5].addColumn(Bytes.toBytes("BAD_CF"), qual, value);
1716    thrown.expect(NoSuchColumnFamilyException.class);
1717    this.region.batchMutate(puts, true);
1718  }
1719
1720  @Test
1721  public void testBatchPutWithTsSlop() throws Exception {
1722    // add data with a timestamp that is too recent for range. Ensure assert
1723    CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
1724    final Put[] puts = new Put[10];
1725    MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
1726
1727    long syncs = prepareRegionForBachPut(puts, source, true);
1728
1729    OperationStatus[] codes = this.region.batchMutate(puts);
1730    assertEquals(10, codes.length);
1731    for (int i = 0; i < 10; i++) {
1732      assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
1733    }
1734    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1735  }
1736
1737  /**
1738   * @return syncs initial syncTimeNumOps
1739   */
1740  private long prepareRegionForBachPut(final Put[] puts, final MetricsWALSource source,
1741    boolean slop) throws IOException {
1742    this.region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES);
1743
1744    LOG.info("First a batch put with all valid puts");
1745    for (int i = 0; i < puts.length; i++) {
1746      puts[i] = slop
1747        ? new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100)
1748        : new Put(Bytes.toBytes("row_" + i));
1749      puts[i].addColumn(COLUMN_FAMILY_BYTES, qual, value);
1750    }
1751
1752    long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
1753    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
1754    return syncs;
1755  }
1756
1757  // ////////////////////////////////////////////////////////////////////////////
1758  // checkAndMutate tests
1759  // ////////////////////////////////////////////////////////////////////////////
1760  @Test
1761  @Deprecated
1762  public void testCheckAndMutate_WithEmptyRowValue() throws IOException {
1763    byte[] row1 = Bytes.toBytes("row1");
1764    byte[] fam1 = Bytes.toBytes("fam1");
1765    byte[] qf1 = Bytes.toBytes("qualifier");
1766    byte[] emptyVal = new byte[] {};
1767    byte[] val1 = Bytes.toBytes("value1");
1768    byte[] val2 = Bytes.toBytes("value2");
1769
1770    // Setting up region
1771    this.region = initHRegion(tableName, method, CONF, fam1);
1772    // Putting empty data in key
1773    Put put = new Put(row1);
1774    put.addColumn(fam1, qf1, emptyVal);
1775
1776    // checkAndPut with empty value
1777    boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
1778      new BinaryComparator(emptyVal), put);
1779    assertTrue(res);
1780
1781    // Putting data in key
1782    put = new Put(row1);
1783    put.addColumn(fam1, qf1, val1);
1784
1785    // checkAndPut with correct value
1786    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
1787      new BinaryComparator(emptyVal), put);
1788    assertTrue(res);
1789
1790    // not empty anymore
1791    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
1792      new BinaryComparator(emptyVal), put);
1793    assertFalse(res);
1794
1795    Delete delete = new Delete(row1);
1796    delete.addColumn(fam1, qf1);
1797    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
1798      new BinaryComparator(emptyVal), delete);
1799    assertFalse(res);
1800
1801    put = new Put(row1);
1802    put.addColumn(fam1, qf1, val2);
1803    // checkAndPut with correct value
1804    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val1),
1805      put);
1806    assertTrue(res);
1807
1808    // checkAndDelete with correct value
1809    delete = new Delete(row1);
1810    delete.addColumn(fam1, qf1);
1811    delete.addColumn(fam1, qf1);
1812    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val2),
1813      delete);
1814    assertTrue(res);
1815
1816    delete = new Delete(row1);
1817    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
1818      new BinaryComparator(emptyVal), delete);
1819    assertTrue(res);
1820
1821    // checkAndPut looking for a null value
1822    put = new Put(row1);
1823    put.addColumn(fam1, qf1, val1);
1824
1825    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new NullComparator(), put);
1826    assertTrue(res);
1827  }
1828
1829  @Test
1830  @Deprecated
1831  public void testCheckAndMutate_WithWrongValue() throws IOException {
1832    byte[] row1 = Bytes.toBytes("row1");
1833    byte[] fam1 = Bytes.toBytes("fam1");
1834    byte[] qf1 = Bytes.toBytes("qualifier");
1835    byte[] val1 = Bytes.toBytes("value1");
1836    byte[] val2 = Bytes.toBytes("value2");
1837    BigDecimal bd1 = new BigDecimal(Double.MAX_VALUE);
1838    BigDecimal bd2 = new BigDecimal(Double.MIN_VALUE);
1839
1840    // Setting up region
1841    this.region = initHRegion(tableName, method, CONF, fam1);
1842    // Putting data in key
1843    Put put = new Put(row1);
1844    put.addColumn(fam1, qf1, val1);
1845    region.put(put);
1846
1847    // checkAndPut with wrong value
1848    boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
1849      new BinaryComparator(val2), put);
1850    assertEquals(false, res);
1851
1852    // checkAndDelete with wrong value
1853    Delete delete = new Delete(row1);
1854    delete.addFamily(fam1);
1855    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val2),
1856      put);
1857    assertEquals(false, res);
1858
1859    // Putting data in key
1860    put = new Put(row1);
1861    put.addColumn(fam1, qf1, Bytes.toBytes(bd1));
1862    region.put(put);
1863
1864    // checkAndPut with wrong value
1865    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
1866      new BigDecimalComparator(bd2), put);
1867    assertEquals(false, res);
1868
1869    // checkAndDelete with wrong value
1870    delete = new Delete(row1);
1871    delete.addFamily(fam1);
1872    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
1873      new BigDecimalComparator(bd2), put);
1874    assertEquals(false, res);
1875  }
1876
1877  @Test
1878  @Deprecated
1879  public void testCheckAndMutate_WithCorrectValue() throws IOException {
1880    byte[] row1 = Bytes.toBytes("row1");
1881    byte[] fam1 = Bytes.toBytes("fam1");
1882    byte[] qf1 = Bytes.toBytes("qualifier");
1883    byte[] val1 = Bytes.toBytes("value1");
1884    BigDecimal bd1 = new BigDecimal(Double.MIN_VALUE);
1885
1886    // Setting up region
1887    this.region = initHRegion(tableName, method, CONF, fam1);
1888    // Putting data in key
1889    long now = EnvironmentEdgeManager.currentTime();
1890    Put put = new Put(row1);
1891    put.addColumn(fam1, qf1, now, val1);
1892    region.put(put);
1893
1894    // checkAndPut with correct value
1895    boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
1896      new BinaryComparator(val1), put);
1897    assertEquals("First", true, res);
1898
1899    // checkAndDelete with correct value
1900    Delete delete = new Delete(row1, now + 1);
1901    delete.addColumn(fam1, qf1);
1902    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val1),
1903      delete);
1904    assertEquals("Delete", true, res);
1905
1906    // Putting data in key
1907    put = new Put(row1);
1908    put.addColumn(fam1, qf1, now + 2, Bytes.toBytes(bd1));
1909    region.put(put);
1910
1911    // checkAndPut with correct value
1912    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
1913      new BigDecimalComparator(bd1), put);
1914    assertEquals("Second put", true, res);
1915
1916    // checkAndDelete with correct value
1917    delete = new Delete(row1, now + 3);
1918    delete.addColumn(fam1, qf1);
1919    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
1920      new BigDecimalComparator(bd1), delete);
1921    assertEquals("Second delete", true, res);
1922  }
1923
1924  @Test
1925  @Deprecated
1926  public void testCheckAndMutate_WithNonEqualCompareOp() throws IOException {
1927    byte[] row1 = Bytes.toBytes("row1");
1928    byte[] fam1 = Bytes.toBytes("fam1");
1929    byte[] qf1 = Bytes.toBytes("qualifier");
1930    byte[] val1 = Bytes.toBytes("value1");
1931    byte[] val2 = Bytes.toBytes("value2");
1932    byte[] val3 = Bytes.toBytes("value3");
1933    byte[] val4 = Bytes.toBytes("value4");
1934
1935    // Setting up region
1936    this.region = initHRegion(tableName, method, CONF, fam1);
1937    // Putting val3 in key
1938    Put put = new Put(row1);
1939    put.addColumn(fam1, qf1, val3);
1940    region.put(put);
1941
1942    // Test CompareOp.LESS: original = val3, compare with val3, fail
1943    boolean res =
1944      region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS, new BinaryComparator(val3), put);
1945    assertEquals(false, res);
1946
1947    // Test CompareOp.LESS: original = val3, compare with val4, fail
1948    res =
1949      region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS, new BinaryComparator(val4), put);
1950    assertEquals(false, res);
1951
1952    // Test CompareOp.LESS: original = val3, compare with val2,
1953    // succeed (now value = val2)
1954    put = new Put(row1);
1955    put.addColumn(fam1, qf1, val2);
1956    res =
1957      region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS, new BinaryComparator(val2), put);
1958    assertEquals(true, res);
1959
1960    // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val3, fail
1961    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS_OR_EQUAL,
1962      new BinaryComparator(val3), put);
1963    assertEquals(false, res);
1964
1965    // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val2,
1966    // succeed (value still = val2)
1967    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS_OR_EQUAL,
1968      new BinaryComparator(val2), put);
1969    assertEquals(true, res);
1970
1971    // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val1,
1972    // succeed (now value = val3)
1973    put = new Put(row1);
1974    put.addColumn(fam1, qf1, val3);
1975    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.LESS_OR_EQUAL,
1976      new BinaryComparator(val1), put);
1977    assertEquals(true, res);
1978
1979    // Test CompareOp.GREATER: original = val3, compare with val3, fail
1980    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER,
1981      new BinaryComparator(val3), put);
1982    assertEquals(false, res);
1983
1984    // Test CompareOp.GREATER: original = val3, compare with val2, fail
1985    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER,
1986      new BinaryComparator(val2), put);
1987    assertEquals(false, res);
1988
1989    // Test CompareOp.GREATER: original = val3, compare with val4,
1990    // succeed (now value = val2)
1991    put = new Put(row1);
1992    put.addColumn(fam1, qf1, val2);
1993    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER,
1994      new BinaryComparator(val4), put);
1995    assertEquals(true, res);
1996
1997    // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val1, fail
1998    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER_OR_EQUAL,
1999      new BinaryComparator(val1), put);
2000    assertEquals(false, res);
2001
2002    // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val2,
2003    // succeed (value still = val2)
2004    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER_OR_EQUAL,
2005      new BinaryComparator(val2), put);
2006    assertEquals(true, res);
2007
2008    // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val3, succeed
2009    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.GREATER_OR_EQUAL,
2010      new BinaryComparator(val3), put);
2011    assertEquals(true, res);
2012  }
2013
2014  @Test
2015  @Deprecated
2016  public void testCheckAndPut_ThatPutWasWritten() throws IOException {
2017    byte[] row1 = Bytes.toBytes("row1");
2018    byte[] fam1 = Bytes.toBytes("fam1");
2019    byte[] fam2 = Bytes.toBytes("fam2");
2020    byte[] qf1 = Bytes.toBytes("qualifier");
2021    byte[] val1 = Bytes.toBytes("value1");
2022    byte[] val2 = Bytes.toBytes("value2");
2023
2024    byte[][] families = { fam1, fam2 };
2025
2026    // Setting up region
2027    this.region = initHRegion(tableName, method, CONF, families);
2028    // Putting data in the key to check
2029    Put put = new Put(row1);
2030    put.addColumn(fam1, qf1, val1);
2031    region.put(put);
2032
2033    // Creating put to add
2034    long ts = EnvironmentEdgeManager.currentTime();
2035    KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
2036    put = new Put(row1);
2037    put.add(kv);
2038
2039    // checkAndPut with wrong value
2040    boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
2041      new BinaryComparator(val1), put);
2042    assertEquals(true, res);
2043
2044    Get get = new Get(row1);
2045    get.addColumn(fam2, qf1);
2046    Cell[] actual = region.get(get).rawCells();
2047
2048    Cell[] expected = { kv };
2049
2050    assertEquals(expected.length, actual.length);
2051    for (int i = 0; i < actual.length; i++) {
2052      assertEquals(expected[i], actual[i]);
2053    }
2054  }
2055
2056  @Test
2057  @Deprecated
2058  public void testCheckAndPut_wrongRowInPut() throws IOException {
2059    this.region = initHRegion(tableName, method, CONF, COLUMNS);
2060    Put put = new Put(row2);
2061    put.addColumn(fam1, qual1, value1);
2062    try {
2063      region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(value2),
2064        put);
2065      fail();
2066    } catch (org.apache.hadoop.hbase.DoNotRetryIOException expected) {
2067      // expected exception.
2068    }
2069  }
2070
2071  @Test
2072  @Deprecated
2073  public void testCheckAndDelete_ThatDeleteWasWritten() throws IOException {
2074    byte[] row1 = Bytes.toBytes("row1");
2075    byte[] fam1 = Bytes.toBytes("fam1");
2076    byte[] fam2 = Bytes.toBytes("fam2");
2077    byte[] qf1 = Bytes.toBytes("qualifier1");
2078    byte[] qf2 = Bytes.toBytes("qualifier2");
2079    byte[] qf3 = Bytes.toBytes("qualifier3");
2080    byte[] val1 = Bytes.toBytes("value1");
2081    byte[] val2 = Bytes.toBytes("value2");
2082    byte[] val3 = Bytes.toBytes("value3");
2083    byte[] emptyVal = new byte[] {};
2084
2085    byte[][] families = { fam1, fam2 };
2086
2087    // Setting up region
2088    this.region = initHRegion(tableName, method, CONF, families);
2089    // Put content
2090    Put put = new Put(row1);
2091    put.addColumn(fam1, qf1, val1);
2092    region.put(put);
2093    Threads.sleep(2);
2094
2095    put = new Put(row1);
2096    put.addColumn(fam1, qf1, val2);
2097    put.addColumn(fam2, qf1, val3);
2098    put.addColumn(fam2, qf2, val2);
2099    put.addColumn(fam2, qf3, val1);
2100    put.addColumn(fam1, qf3, val1);
2101    region.put(put);
2102
2103    LOG.info("get={}", region.get(new Get(row1).addColumn(fam1, qf1)).toString());
2104
2105    // Multi-column delete
2106    Delete delete = new Delete(row1);
2107    delete.addColumn(fam1, qf1);
2108    delete.addColumn(fam2, qf1);
2109    delete.addColumn(fam1, qf3);
2110    boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL,
2111      new BinaryComparator(val2), delete);
2112    assertEquals(true, res);
2113
2114    Get get = new Get(row1);
2115    get.addColumn(fam1, qf1);
2116    get.addColumn(fam1, qf3);
2117    get.addColumn(fam2, qf2);
2118    Result r = region.get(get);
2119    assertEquals(2, r.size());
2120    assertArrayEquals(val1, r.getValue(fam1, qf1));
2121    assertArrayEquals(val2, r.getValue(fam2, qf2));
2122
2123    // Family delete
2124    delete = new Delete(row1);
2125    delete.addFamily(fam2);
2126    res = region.checkAndMutate(row1, fam2, qf1, CompareOperator.EQUAL,
2127      new BinaryComparator(emptyVal), delete);
2128    assertEquals(true, res);
2129
2130    get = new Get(row1);
2131    r = region.get(get);
2132    assertEquals(1, r.size());
2133    assertArrayEquals(val1, r.getValue(fam1, qf1));
2134
2135    // Row delete
2136    delete = new Delete(row1);
2137    res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val1),
2138      delete);
2139    assertEquals(true, res);
2140    get = new Get(row1);
2141    r = region.get(get);
2142    assertEquals(0, r.size());
2143  }
2144
2145  @Test
2146  @Deprecated
2147  public void testCheckAndMutate_WithFilters() throws Throwable {
2148    final byte[] FAMILY = Bytes.toBytes("fam");
2149
2150    // Setting up region
2151    this.region = initHRegion(tableName, method, CONF, FAMILY);
2152
2153    // Put one row
2154    Put put = new Put(row);
2155    put.addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a"));
2156    put.addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b"));
2157    put.addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c"));
2158    region.put(put);
2159
2160    // Put with success
2161    boolean ok = region.checkAndMutate(row,
2162      new FilterList(
2163        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2164          Bytes.toBytes("a")),
2165        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL,
2166          Bytes.toBytes("b"))),
2167      new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d")));
2168    assertTrue(ok);
2169
2170    Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D")));
2171    assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D"))));
2172
2173    // Put with failure
2174    ok = region.checkAndMutate(row,
2175      new FilterList(
2176        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2177          Bytes.toBytes("a")),
2178        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL,
2179          Bytes.toBytes("c"))),
2180      new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e")));
2181    assertFalse(ok);
2182
2183    assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E"))).isEmpty());
2184
2185    // Delete with success
2186    ok = region.checkAndMutate(row,
2187      new FilterList(
2188        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2189          Bytes.toBytes("a")),
2190        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL,
2191          Bytes.toBytes("b"))),
2192      new Delete(row).addColumns(FAMILY, Bytes.toBytes("D")));
2193    assertTrue(ok);
2194
2195    assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))).isEmpty());
2196
2197    // Mutate with success
2198    ok = region.checkAndRowMutate(row,
2199      new FilterList(
2200        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2201          Bytes.toBytes("a")),
2202        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL,
2203          Bytes.toBytes("b"))),
2204      new RowMutations(row)
2205        .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e")))
2206        .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A"))));
2207    assertTrue(ok);
2208
2209    result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E")));
2210    assertEquals("e", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("E"))));
2211
2212    assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty());
2213  }
2214
2215  @Test
2216  @Deprecated
2217  public void testCheckAndMutate_WithFiltersAndTimeRange() throws Throwable {
2218    final byte[] FAMILY = Bytes.toBytes("fam");
2219
2220    // Setting up region
2221    this.region = initHRegion(tableName, method, CONF, FAMILY);
2222
2223    // Put with specifying the timestamp
2224    region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), 100, Bytes.toBytes("a")));
2225
2226    // Put with success
2227    boolean ok = region.checkAndMutate(row,
2228      new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2229        Bytes.toBytes("a")),
2230      TimeRange.between(0, 101),
2231      new Put(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")));
2232    assertTrue(ok);
2233
2234    Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B")));
2235    assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B"))));
2236
2237    // Put with failure
2238    ok = region.checkAndMutate(row,
2239      new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2240        Bytes.toBytes("a")),
2241      TimeRange.between(0, 100),
2242      new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")));
2243    assertFalse(ok);
2244
2245    assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("C"))).isEmpty());
2246
2247    // Mutate with success
2248    ok = region.checkAndRowMutate(row,
2249      new SingleColumnValueFilter(
2250        FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL, Bytes.toBytes("a")),
2251      TimeRange.between(0, 101),
2252      new RowMutations(row)
2253        .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d")))
2254        .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A"))));
2255    assertTrue(ok);
2256
2257    result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D")));
2258    assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D"))));
2259
2260    assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty());
2261  }
2262
2263  @Test
2264  @Deprecated
2265  public void testCheckAndMutate_wrongMutationType() throws Throwable {
2266    // Setting up region
2267    this.region = initHRegion(tableName, method, CONF, fam1);
2268
2269    try {
2270      region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(value1),
2271        new Increment(row).addColumn(fam1, qual1, 1));
2272      fail("should throw DoNotRetryIOException");
2273    } catch (DoNotRetryIOException e) {
2274      assertEquals("Unsupported mutate type: INCREMENT", e.getMessage());
2275    }
2276
2277    try {
2278      region.checkAndMutate(row,
2279        new SingleColumnValueFilter(fam1, qual1, CompareOperator.EQUAL, value1),
2280        new Increment(row).addColumn(fam1, qual1, 1));
2281      fail("should throw DoNotRetryIOException");
2282    } catch (DoNotRetryIOException e) {
2283      assertEquals("Unsupported mutate type: INCREMENT", e.getMessage());
2284    }
2285  }
2286
2287  @Test
2288  @Deprecated
2289  public void testCheckAndMutate_wrongRow() throws Throwable {
2290    final byte[] wrongRow = Bytes.toBytes("wrongRow");
2291
2292    // Setting up region
2293    this.region = initHRegion(tableName, method, CONF, fam1);
2294
2295    try {
2296      region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(value1),
2297        new Put(wrongRow).addColumn(fam1, qual1, value1));
2298      fail("should throw DoNotRetryIOException");
2299    } catch (DoNotRetryIOException e) {
2300      assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>",
2301        e.getMessage());
2302    }
2303
2304    try {
2305      region.checkAndMutate(row,
2306        new SingleColumnValueFilter(fam1, qual1, CompareOperator.EQUAL, value1),
2307        new Put(wrongRow).addColumn(fam1, qual1, value1));
2308      fail("should throw DoNotRetryIOException");
2309    } catch (DoNotRetryIOException e) {
2310      assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>",
2311        e.getMessage());
2312    }
2313
2314    try {
2315      region.checkAndRowMutate(row, fam1, qual1, CompareOperator.EQUAL,
2316        new BinaryComparator(value1),
2317        new RowMutations(wrongRow).add((Mutation) new Put(wrongRow).addColumn(fam1, qual1, value1))
2318          .add((Mutation) new Delete(wrongRow).addColumns(fam1, qual2)));
2319      fail("should throw DoNotRetryIOException");
2320    } catch (DoNotRetryIOException e) {
2321      assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>",
2322        e.getMessage());
2323    }
2324
2325    try {
2326      region.checkAndRowMutate(row,
2327        new SingleColumnValueFilter(fam1, qual1, CompareOperator.EQUAL, value1),
2328        new RowMutations(wrongRow).add((Mutation) new Put(wrongRow).addColumn(fam1, qual1, value1))
2329          .add((Mutation) new Delete(wrongRow).addColumns(fam1, qual2)));
2330      fail("should throw DoNotRetryIOException");
2331    } catch (DoNotRetryIOException e) {
2332      assertEquals("The row of the action <wrongRow> doesn't match the original one <rowA>",
2333        e.getMessage());
2334    }
2335  }
2336
2337  @Test
2338  public void testCheckAndMutateWithEmptyRowValue() throws IOException {
2339    byte[] row1 = Bytes.toBytes("row1");
2340    byte[] fam1 = Bytes.toBytes("fam1");
2341    byte[] qf1 = Bytes.toBytes("qualifier");
2342    byte[] emptyVal = new byte[] {};
2343    byte[] val1 = Bytes.toBytes("value1");
2344    byte[] val2 = Bytes.toBytes("value2");
2345
2346    // Setting up region
2347    this.region = initHRegion(tableName, method, CONF, fam1);
2348    // Putting empty data in key
2349    Put put = new Put(row1);
2350    put.addColumn(fam1, qf1, emptyVal);
2351
2352    // checkAndPut with empty value
2353    CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2354      .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(put));
2355    assertTrue(res.isSuccess());
2356    assertNull(res.getResult());
2357
2358    // Putting data in key
2359    put = new Put(row1);
2360    put.addColumn(fam1, qf1, val1);
2361
2362    // checkAndPut with correct value
2363    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2364      .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(put));
2365    assertTrue(res.isSuccess());
2366    assertNull(res.getResult());
2367
2368    // not empty anymore
2369    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2370      .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(put));
2371    assertFalse(res.isSuccess());
2372    assertNull(res.getResult());
2373
2374    Delete delete = new Delete(row1);
2375    delete.addColumn(fam1, qf1);
2376    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2377      .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(delete));
2378    assertFalse(res.isSuccess());
2379    assertNull(res.getResult());
2380
2381    put = new Put(row1);
2382    put.addColumn(fam1, qf1, val2);
2383    // checkAndPut with correct value
2384    res = region.checkAndMutate(
2385      CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(put));
2386    assertTrue(res.isSuccess());
2387    assertNull(res.getResult());
2388
2389    // checkAndDelete with correct value
2390    delete = new Delete(row1);
2391    delete.addColumn(fam1, qf1);
2392    delete.addColumn(fam1, qf1);
2393    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2394      .ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(delete));
2395    assertTrue(res.isSuccess());
2396    assertNull(res.getResult());
2397
2398    delete = new Delete(row1);
2399    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2400      .ifMatches(fam1, qf1, CompareOperator.EQUAL, emptyVal).build(delete));
2401    assertTrue(res.isSuccess());
2402    assertNull(res.getResult());
2403
2404    // checkAndPut looking for a null value
2405    put = new Put(row1);
2406    put.addColumn(fam1, qf1, val1);
2407
2408    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1).ifNotExists(fam1, qf1).build(put));
2409    assertTrue(res.isSuccess());
2410    assertNull(res.getResult());
2411  }
2412
2413  @Test
2414  public void testCheckAndMutateWithWrongValue() throws IOException {
2415    byte[] row1 = Bytes.toBytes("row1");
2416    byte[] fam1 = Bytes.toBytes("fam1");
2417    byte[] qf1 = Bytes.toBytes("qualifier");
2418    byte[] val1 = Bytes.toBytes("value1");
2419    byte[] val2 = Bytes.toBytes("value2");
2420    BigDecimal bd1 = new BigDecimal(Double.MAX_VALUE);
2421    BigDecimal bd2 = new BigDecimal(Double.MIN_VALUE);
2422
2423    // Setting up region
2424    this.region = initHRegion(tableName, method, CONF, fam1);
2425    // Putting data in key
2426    Put put = new Put(row1);
2427    put.addColumn(fam1, qf1, val1);
2428    region.put(put);
2429
2430    // checkAndPut with wrong value
2431    CheckAndMutateResult res = region.checkAndMutate(
2432      CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(put));
2433    assertFalse(res.isSuccess());
2434    assertNull(res.getResult());
2435
2436    // checkAndDelete with wrong value
2437    Delete delete = new Delete(row1);
2438    delete.addFamily(fam1);
2439    res = region.checkAndMutate(
2440      CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(put));
2441    assertFalse(res.isSuccess());
2442    assertNull(res.getResult());
2443
2444    // Putting data in key
2445    put = new Put(row1);
2446    put.addColumn(fam1, qf1, Bytes.toBytes(bd1));
2447    region.put(put);
2448
2449    // checkAndPut with wrong value
2450    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2451      .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd2)).build(put));
2452    assertFalse(res.isSuccess());
2453    assertNull(res.getResult());
2454
2455    // checkAndDelete with wrong value
2456    delete = new Delete(row1);
2457    delete.addFamily(fam1);
2458    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2459      .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd2)).build(delete));
2460    assertFalse(res.isSuccess());
2461    assertNull(res.getResult());
2462  }
2463
2464  @Test
2465  public void testCheckAndMutateWithCorrectValue() throws IOException {
2466    byte[] row1 = Bytes.toBytes("row1");
2467    byte[] fam1 = Bytes.toBytes("fam1");
2468    byte[] qf1 = Bytes.toBytes("qualifier");
2469    byte[] val1 = Bytes.toBytes("value1");
2470    BigDecimal bd1 = new BigDecimal(Double.MIN_VALUE);
2471
2472    // Setting up region
2473    this.region = initHRegion(tableName, method, CONF, fam1);
2474    // Putting data in key
2475    long now = EnvironmentEdgeManager.currentTime();
2476    Put put = new Put(row1);
2477    put.addColumn(fam1, qf1, now, val1);
2478    region.put(put);
2479
2480    // checkAndPut with correct value
2481    CheckAndMutateResult res = region.checkAndMutate(
2482      CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(put));
2483    assertTrue("First", res.isSuccess());
2484
2485    // checkAndDelete with correct value
2486    Delete delete = new Delete(row1, now + 1);
2487    delete.addColumn(fam1, qf1);
2488    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2489      .ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(delete));
2490    assertTrue("Delete", res.isSuccess());
2491    assertNull(res.getResult());
2492
2493    // Putting data in key
2494    put = new Put(row1);
2495    put.addColumn(fam1, qf1, now + 2, Bytes.toBytes(bd1));
2496    region.put(put);
2497
2498    // checkAndPut with correct value
2499    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2500      .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd1)).build(put));
2501    assertTrue("Second put", res.isSuccess());
2502    assertNull(res.getResult());
2503
2504    // checkAndDelete with correct value
2505    delete = new Delete(row1, now + 3);
2506    delete.addColumn(fam1, qf1);
2507    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2508      .ifMatches(fam1, qf1, CompareOperator.EQUAL, Bytes.toBytes(bd1)).build(delete));
2509    assertTrue("Second delete", res.isSuccess());
2510    assertNull(res.getResult());
2511  }
2512
2513  @Test
2514  public void testCheckAndMutateWithNonEqualCompareOp() throws IOException {
2515    byte[] row1 = Bytes.toBytes("row1");
2516    byte[] fam1 = Bytes.toBytes("fam1");
2517    byte[] qf1 = Bytes.toBytes("qualifier");
2518    byte[] val1 = Bytes.toBytes("value1");
2519    byte[] val2 = Bytes.toBytes("value2");
2520    byte[] val3 = Bytes.toBytes("value3");
2521    byte[] val4 = Bytes.toBytes("value4");
2522
2523    // Setting up region
2524    this.region = initHRegion(tableName, method, CONF, fam1);
2525    // Putting val3 in key
2526    Put put = new Put(row1);
2527    put.addColumn(fam1, qf1, val3);
2528    region.put(put);
2529
2530    // Test CompareOp.LESS: original = val3, compare with val3, fail
2531    CheckAndMutateResult res = region.checkAndMutate(
2532      CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.LESS, val3).build(put));
2533    assertFalse(res.isSuccess());
2534    assertNull(res.getResult());
2535
2536    // Test CompareOp.LESS: original = val3, compare with val4, fail
2537    res = region.checkAndMutate(
2538      CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.LESS, val4).build(put));
2539    assertFalse(res.isSuccess());
2540    assertNull(res.getResult());
2541
2542    // Test CompareOp.LESS: original = val3, compare with val2,
2543    // succeed (now value = val2)
2544    put = new Put(row1);
2545    put.addColumn(fam1, qf1, val2);
2546    res = region.checkAndMutate(
2547      CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.LESS, val2).build(put));
2548    assertTrue(res.isSuccess());
2549    assertNull(res.getResult());
2550
2551    // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val3, fail
2552    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2553      .ifMatches(fam1, qf1, CompareOperator.LESS_OR_EQUAL, val3).build(put));
2554    assertFalse(res.isSuccess());
2555    assertNull(res.getResult());
2556
2557    // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val2,
2558    // succeed (value still = val2)
2559    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2560      .ifMatches(fam1, qf1, CompareOperator.LESS_OR_EQUAL, val2).build(put));
2561    assertTrue(res.isSuccess());
2562    assertNull(res.getResult());
2563
2564    // Test CompareOp.LESS_OR_EQUAL: original = val2, compare with val1,
2565    // succeed (now value = val3)
2566    put = new Put(row1);
2567    put.addColumn(fam1, qf1, val3);
2568    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2569      .ifMatches(fam1, qf1, CompareOperator.LESS_OR_EQUAL, val1).build(put));
2570    assertTrue(res.isSuccess());
2571    assertNull(res.getResult());
2572
2573    // Test CompareOp.GREATER: original = val3, compare with val3, fail
2574    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2575      .ifMatches(fam1, qf1, CompareOperator.GREATER, val3).build(put));
2576    assertFalse(res.isSuccess());
2577    assertNull(res.getResult());
2578
2579    // Test CompareOp.GREATER: original = val3, compare with val2, fail
2580    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2581      .ifMatches(fam1, qf1, CompareOperator.GREATER, val2).build(put));
2582    assertFalse(res.isSuccess());
2583    assertNull(res.getResult());
2584
2585    // Test CompareOp.GREATER: original = val3, compare with val4,
2586    // succeed (now value = val2)
2587    put = new Put(row1);
2588    put.addColumn(fam1, qf1, val2);
2589    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2590      .ifMatches(fam1, qf1, CompareOperator.GREATER, val4).build(put));
2591    assertTrue(res.isSuccess());
2592    assertNull(res.getResult());
2593
2594    // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val1, fail
2595    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2596      .ifMatches(fam1, qf1, CompareOperator.GREATER_OR_EQUAL, val1).build(put));
2597    assertFalse(res.isSuccess());
2598    assertNull(res.getResult());
2599
2600    // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val2,
2601    // succeed (value still = val2)
2602    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2603      .ifMatches(fam1, qf1, CompareOperator.GREATER_OR_EQUAL, val2).build(put));
2604    assertTrue(res.isSuccess());
2605    assertNull(res.getResult());
2606
2607    // Test CompareOp.GREATER_OR_EQUAL: original = val2, compare with val3, succeed
2608    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2609      .ifMatches(fam1, qf1, CompareOperator.GREATER_OR_EQUAL, val3).build(put));
2610    assertTrue(res.isSuccess());
2611    assertNull(res.getResult());
2612  }
2613
2614  @Test
2615  public void testCheckAndPutThatPutWasWritten() throws IOException {
2616    byte[] row1 = Bytes.toBytes("row1");
2617    byte[] fam1 = Bytes.toBytes("fam1");
2618    byte[] fam2 = Bytes.toBytes("fam2");
2619    byte[] qf1 = Bytes.toBytes("qualifier");
2620    byte[] val1 = Bytes.toBytes("value1");
2621    byte[] val2 = Bytes.toBytes("value2");
2622
2623    byte[][] families = { fam1, fam2 };
2624
2625    // Setting up region
2626    this.region = initHRegion(tableName, method, CONF, families);
2627    // Putting data in the key to check
2628    Put put = new Put(row1);
2629    put.addColumn(fam1, qf1, val1);
2630    region.put(put);
2631
2632    // Creating put to add
2633    long ts = EnvironmentEdgeManager.currentTime();
2634    KeyValue kv = new KeyValue(row1, fam2, qf1, ts, KeyValue.Type.Put, val2);
2635    put = new Put(row1);
2636    put.add(kv);
2637
2638    // checkAndPut with wrong value
2639    CheckAndMutateResult res = region.checkAndMutate(
2640      CheckAndMutate.newBuilder(row1).ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(put));
2641    assertTrue(res.isSuccess());
2642    assertNull(res.getResult());
2643
2644    Get get = new Get(row1);
2645    get.addColumn(fam2, qf1);
2646    Cell[] actual = region.get(get).rawCells();
2647
2648    Cell[] expected = { kv };
2649
2650    assertEquals(expected.length, actual.length);
2651    for (int i = 0; i < actual.length; i++) {
2652      assertEquals(expected[i], actual[i]);
2653    }
2654  }
2655
2656  @Test
2657  public void testCheckAndDeleteThatDeleteWasWritten() throws IOException {
2658    byte[] row1 = Bytes.toBytes("row1");
2659    byte[] fam1 = Bytes.toBytes("fam1");
2660    byte[] fam2 = Bytes.toBytes("fam2");
2661    byte[] qf1 = Bytes.toBytes("qualifier1");
2662    byte[] qf2 = Bytes.toBytes("qualifier2");
2663    byte[] qf3 = Bytes.toBytes("qualifier3");
2664    byte[] val1 = Bytes.toBytes("value1");
2665    byte[] val2 = Bytes.toBytes("value2");
2666    byte[] val3 = Bytes.toBytes("value3");
2667    byte[] emptyVal = new byte[] {};
2668
2669    byte[][] families = { fam1, fam2 };
2670
2671    // Setting up region
2672    this.region = initHRegion(tableName, method, CONF, families);
2673    // Put content
2674    Put put = new Put(row1);
2675    put.addColumn(fam1, qf1, val1);
2676    region.put(put);
2677    Threads.sleep(2);
2678
2679    put = new Put(row1);
2680    put.addColumn(fam1, qf1, val2);
2681    put.addColumn(fam2, qf1, val3);
2682    put.addColumn(fam2, qf2, val2);
2683    put.addColumn(fam2, qf3, val1);
2684    put.addColumn(fam1, qf3, val1);
2685    region.put(put);
2686
2687    LOG.info("get={}", region.get(new Get(row1).addColumn(fam1, qf1)).toString());
2688
2689    // Multi-column delete
2690    Delete delete = new Delete(row1);
2691    delete.addColumn(fam1, qf1);
2692    delete.addColumn(fam2, qf1);
2693    delete.addColumn(fam1, qf3);
2694    CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2695      .ifMatches(fam1, qf1, CompareOperator.EQUAL, val2).build(delete));
2696    assertTrue(res.isSuccess());
2697    assertNull(res.getResult());
2698
2699    Get get = new Get(row1);
2700    get.addColumn(fam1, qf1);
2701    get.addColumn(fam1, qf3);
2702    get.addColumn(fam2, qf2);
2703    Result r = region.get(get);
2704    assertEquals(2, r.size());
2705    assertArrayEquals(val1, r.getValue(fam1, qf1));
2706    assertArrayEquals(val2, r.getValue(fam2, qf2));
2707
2708    // Family delete
2709    delete = new Delete(row1);
2710    delete.addFamily(fam2);
2711    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2712      .ifMatches(fam2, qf1, CompareOperator.EQUAL, emptyVal).build(delete));
2713    assertTrue(res.isSuccess());
2714    assertNull(res.getResult());
2715
2716    get = new Get(row1);
2717    r = region.get(get);
2718    assertEquals(1, r.size());
2719    assertArrayEquals(val1, r.getValue(fam1, qf1));
2720
2721    // Row delete
2722    delete = new Delete(row1);
2723    res = region.checkAndMutate(CheckAndMutate.newBuilder(row1)
2724      .ifMatches(fam1, qf1, CompareOperator.EQUAL, val1).build(delete));
2725    assertTrue(res.isSuccess());
2726    assertNull(res.getResult());
2727
2728    get = new Get(row1);
2729    r = region.get(get);
2730    assertEquals(0, r.size());
2731  }
2732
2733  @Test
2734  public void testCheckAndMutateWithFilters() throws Throwable {
2735    final byte[] FAMILY = Bytes.toBytes("fam");
2736
2737    // Setting up region
2738    this.region = initHRegion(tableName, method, CONF, FAMILY);
2739
2740    // Put one row
2741    Put put = new Put(row);
2742    put.addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a"));
2743    put.addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b"));
2744    put.addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c"));
2745    region.put(put);
2746
2747    // Put with success
2748    CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row)
2749      .ifMatches(new FilterList(
2750        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2751          Bytes.toBytes("a")),
2752        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL,
2753          Bytes.toBytes("b"))))
2754      .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d"))));
2755    assertTrue(res.isSuccess());
2756    assertNull(res.getResult());
2757
2758    Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D")));
2759    assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D"))));
2760
2761    // Put with failure
2762    res = region.checkAndMutate(CheckAndMutate.newBuilder(row)
2763      .ifMatches(new FilterList(
2764        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2765          Bytes.toBytes("a")),
2766        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL,
2767          Bytes.toBytes("c"))))
2768      .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e"))));
2769    assertFalse(res.isSuccess());
2770    assertNull(res.getResult());
2771
2772    assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E"))).isEmpty());
2773
2774    // Delete with success
2775    res = region.checkAndMutate(CheckAndMutate.newBuilder(row)
2776      .ifMatches(new FilterList(
2777        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2778          Bytes.toBytes("a")),
2779        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL,
2780          Bytes.toBytes("b"))))
2781      .build(new Delete(row).addColumns(FAMILY, Bytes.toBytes("D"))));
2782    assertTrue(res.isSuccess());
2783    assertNull(res.getResult());
2784
2785    assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D"))).isEmpty());
2786
2787    // Mutate with success
2788    res = region.checkAndMutate(CheckAndMutate.newBuilder(row)
2789      .ifMatches(new FilterList(
2790        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2791          Bytes.toBytes("a")),
2792        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("B"), CompareOperator.EQUAL,
2793          Bytes.toBytes("b"))))
2794      .build(new RowMutations(row)
2795        .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e")))
2796        .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A")))));
2797    assertTrue(res.isSuccess());
2798    assertNull(res.getResult());
2799
2800    result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("E")));
2801    assertEquals("e", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("E"))));
2802
2803    assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty());
2804  }
2805
2806  @Test
2807  public void testCheckAndMutateWithFiltersAndTimeRange() throws Throwable {
2808    final byte[] FAMILY = Bytes.toBytes("fam");
2809
2810    // Setting up region
2811    this.region = initHRegion(tableName, method, CONF, FAMILY);
2812
2813    // Put with specifying the timestamp
2814    region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), 100, Bytes.toBytes("a")));
2815
2816    // Put with success
2817    CheckAndMutateResult res = region.checkAndMutate(CheckAndMutate.newBuilder(row)
2818      .ifMatches(new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2819        Bytes.toBytes("a")))
2820      .timeRange(TimeRange.between(0, 101))
2821      .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b"))));
2822    assertTrue(res.isSuccess());
2823    assertNull(res.getResult());
2824
2825    Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B")));
2826    assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B"))));
2827
2828    // Put with failure
2829    res = region.checkAndMutate(CheckAndMutate.newBuilder(row)
2830      .ifMatches(new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2831        Bytes.toBytes("a")))
2832      .timeRange(TimeRange.between(0, 100))
2833      .build(new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c"))));
2834    assertFalse(res.isSuccess());
2835    assertNull(res.getResult());
2836
2837    assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("C"))).isEmpty());
2838
2839    // RowMutations with success
2840    res = region.checkAndMutate(CheckAndMutate.newBuilder(row)
2841      .ifMatches(new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2842        Bytes.toBytes("a")))
2843      .timeRange(TimeRange.between(0, 101))
2844      .build(new RowMutations(row)
2845        .add((Mutation) new Put(row).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d")))
2846        .add((Mutation) new Delete(row).addColumns(FAMILY, Bytes.toBytes("A")))));
2847    assertTrue(res.isSuccess());
2848    assertNull(res.getResult());
2849
2850    result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("D")));
2851    assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D"))));
2852
2853    assertTrue(region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("A"))).isEmpty());
2854  }
2855
2856  @Test
2857  public void testCheckAndIncrement() throws Throwable {
2858    final byte[] FAMILY = Bytes.toBytes("fam");
2859
2860    // Setting up region
2861    this.region = initHRegion(tableName, method, CONF, FAMILY);
2862
2863    region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")));
2864
2865    // CheckAndIncrement with correct value
2866    CheckAndMutateResult res = region.checkAndMutate(
2867      CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a"))
2868        .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 1)));
2869    assertTrue(res.isSuccess());
2870    assertEquals(1, Bytes.toLong(res.getResult().getValue(FAMILY, Bytes.toBytes("B"))));
2871
2872    Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B")));
2873    assertEquals(1, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B"))));
2874
2875    // CheckAndIncrement with wrong value
2876    res = region.checkAndMutate(
2877      CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("b"))
2878        .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 1)));
2879    assertFalse(res.isSuccess());
2880    assertNull(res.getResult());
2881
2882    result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B")));
2883    assertEquals(1, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B"))));
2884
2885    region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")));
2886
2887    // CheckAndIncrement with a filter and correct value
2888    res = region.checkAndMutate(CheckAndMutate.newBuilder(row)
2889      .ifMatches(new FilterList(
2890        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2891          Bytes.toBytes("a")),
2892        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL,
2893          Bytes.toBytes("c"))))
2894      .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 2)));
2895    assertTrue(res.isSuccess());
2896    assertEquals(3, Bytes.toLong(res.getResult().getValue(FAMILY, Bytes.toBytes("B"))));
2897
2898    result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B")));
2899    assertEquals(3, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B"))));
2900
2901    // CheckAndIncrement with a filter and correct value
2902    res = region.checkAndMutate(CheckAndMutate.newBuilder(row)
2903      .ifMatches(new FilterList(
2904        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2905          Bytes.toBytes("b")),
2906        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL,
2907          Bytes.toBytes("d"))))
2908      .build(new Increment(row).addColumn(FAMILY, Bytes.toBytes("B"), 2)));
2909    assertFalse(res.isSuccess());
2910    assertNull(res.getResult());
2911
2912    result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B")));
2913    assertEquals(3, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("B"))));
2914  }
2915
2916  @Test
2917  public void testCheckAndAppend() throws Throwable {
2918    final byte[] FAMILY = Bytes.toBytes("fam");
2919
2920    // Setting up region
2921    this.region = initHRegion(tableName, method, CONF, FAMILY);
2922
2923    region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")));
2924
2925    // CheckAndAppend with correct value
2926    CheckAndMutateResult res = region.checkAndMutate(
2927      CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a"))
2928        .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b"))));
2929    assertTrue(res.isSuccess());
2930    assertEquals("b", Bytes.toString(res.getResult().getValue(FAMILY, Bytes.toBytes("B"))));
2931
2932    Result result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B")));
2933    assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B"))));
2934
2935    // CheckAndAppend with wrong value
2936    res = region.checkAndMutate(
2937      CheckAndMutate.newBuilder(row).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("b"))
2938        .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b"))));
2939    assertFalse(res.isSuccess());
2940    assertNull(res.getResult());
2941
2942    result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B")));
2943    assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B"))));
2944
2945    region.put(new Put(row).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")));
2946
2947    // CheckAndAppend with a filter and correct value
2948    res = region.checkAndMutate(CheckAndMutate.newBuilder(row)
2949      .ifMatches(new FilterList(
2950        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2951          Bytes.toBytes("a")),
2952        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL,
2953          Bytes.toBytes("c"))))
2954      .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("bb"))));
2955    assertTrue(res.isSuccess());
2956    assertEquals("bbb", Bytes.toString(res.getResult().getValue(FAMILY, Bytes.toBytes("B"))));
2957
2958    result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B")));
2959    assertEquals("bbb", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B"))));
2960
2961    // CheckAndAppend with a filter and wrong value
2962    res = region.checkAndMutate(CheckAndMutate.newBuilder(row)
2963      .ifMatches(new FilterList(
2964        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("A"), CompareOperator.EQUAL,
2965          Bytes.toBytes("b")),
2966        new SingleColumnValueFilter(FAMILY, Bytes.toBytes("C"), CompareOperator.EQUAL,
2967          Bytes.toBytes("d"))))
2968      .build(new Append(row).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("bb"))));
2969    assertFalse(res.isSuccess());
2970    assertNull(res.getResult());
2971
2972    result = region.get(new Get(row).addColumn(FAMILY, Bytes.toBytes("B")));
2973    assertEquals("bbb", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B"))));
2974  }
2975
2976  @Test
2977  public void testCheckAndIncrementAndAppend() throws Throwable {
2978    // Setting up region
2979    this.region = initHRegion(tableName, method, CONF, fam1);
2980
2981    // CheckAndMutate with Increment and Append
2982    CheckAndMutate checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, qual)
2983      .build(new RowMutations(row).add((Mutation) new Increment(row).addColumn(fam1, qual1, 1L))
2984        .add((Mutation) new Append(row).addColumn(fam1, qual2, Bytes.toBytes("a"))));
2985
2986    CheckAndMutateResult result = region.checkAndMutate(checkAndMutate);
2987    assertTrue(result.isSuccess());
2988    assertEquals(1L, Bytes.toLong(result.getResult().getValue(fam1, qual1)));
2989    assertEquals("a", Bytes.toString(result.getResult().getValue(fam1, qual2)));
2990
2991    Result r = region.get(new Get(row));
2992    assertEquals(1L, Bytes.toLong(r.getValue(fam1, qual1)));
2993    assertEquals("a", Bytes.toString(r.getValue(fam1, qual2)));
2994
2995    // Set return results to false
2996    checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, qual)
2997      .build(new RowMutations(row)
2998        .add((Mutation) new Increment(row).addColumn(fam1, qual1, 1L).setReturnResults(false))
2999        .add((Mutation) new Append(row).addColumn(fam1, qual2, Bytes.toBytes("a"))
3000          .setReturnResults(false)));
3001
3002    result = region.checkAndMutate(checkAndMutate);
3003    assertTrue(result.isSuccess());
3004    assertNull(result.getResult().getValue(fam1, qual1));
3005    assertNull(result.getResult().getValue(fam1, qual2));
3006
3007    r = region.get(new Get(row));
3008    assertEquals(2L, Bytes.toLong(r.getValue(fam1, qual1)));
3009    assertEquals("aa", Bytes.toString(r.getValue(fam1, qual2)));
3010
3011    checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, qual)
3012      .build(new RowMutations(row).add((Mutation) new Increment(row).addColumn(fam1, qual1, 1L))
3013        .add((Mutation) new Append(row).addColumn(fam1, qual2, Bytes.toBytes("a"))
3014          .setReturnResults(false)));
3015
3016    result = region.checkAndMutate(checkAndMutate);
3017    assertTrue(result.isSuccess());
3018    assertEquals(3L, Bytes.toLong(result.getResult().getValue(fam1, qual1)));
3019    assertNull(result.getResult().getValue(fam1, qual2));
3020
3021    r = region.get(new Get(row));
3022    assertEquals(3L, Bytes.toLong(r.getValue(fam1, qual1)));
3023    assertEquals("aaa", Bytes.toString(r.getValue(fam1, qual2)));
3024  }
3025
3026  @Test
3027  public void testCheckAndRowMutations() throws Throwable {
3028    final byte[] row = Bytes.toBytes("row");
3029    final byte[] q1 = Bytes.toBytes("q1");
3030    final byte[] q2 = Bytes.toBytes("q2");
3031    final byte[] q3 = Bytes.toBytes("q3");
3032    final byte[] q4 = Bytes.toBytes("q4");
3033    final String v1 = "v1";
3034
3035    region = initHRegion(tableName, method, CONF, fam1);
3036
3037    // Initial values
3038    region
3039      .batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q2, Bytes.toBytes("toBeDeleted")),
3040        new Put(row).addColumn(fam1, q3, Bytes.toBytes(5L)),
3041        new Put(row).addColumn(fam1, q4, Bytes.toBytes("a")), });
3042
3043    // Do CheckAndRowMutations
3044    CheckAndMutate checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, q1).build(
3045      new RowMutations(row).add(Arrays.asList(new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1)),
3046        new Delete(row).addColumns(fam1, q2), new Increment(row).addColumn(fam1, q3, 1),
3047        new Append(row).addColumn(fam1, q4, Bytes.toBytes("b")))));
3048
3049    CheckAndMutateResult result = region.checkAndMutate(checkAndMutate);
3050    assertTrue(result.isSuccess());
3051    assertEquals(6L, Bytes.toLong(result.getResult().getValue(fam1, q3)));
3052    assertEquals("ab", Bytes.toString(result.getResult().getValue(fam1, q4)));
3053
3054    // Verify the value
3055    Result r = region.get(new Get(row));
3056    assertEquals(v1, Bytes.toString(r.getValue(fam1, q1)));
3057    assertNull(r.getValue(fam1, q2));
3058    assertEquals(6L, Bytes.toLong(r.getValue(fam1, q3)));
3059    assertEquals("ab", Bytes.toString(r.getValue(fam1, q4)));
3060
3061    // Do CheckAndRowMutations again
3062    checkAndMutate = CheckAndMutate.newBuilder(row).ifNotExists(fam1, q1)
3063      .build(new RowMutations(row).add(Arrays.asList(new Delete(row).addColumns(fam1, q1),
3064        new Put(row).addColumn(fam1, q2, Bytes.toBytes(v1)),
3065        new Increment(row).addColumn(fam1, q3, 1),
3066        new Append(row).addColumn(fam1, q4, Bytes.toBytes("b")))));
3067
3068    result = region.checkAndMutate(checkAndMutate);
3069    assertFalse(result.isSuccess());
3070    assertNull(result.getResult());
3071
3072    // Verify the value
3073    r = region.get(new Get(row));
3074    assertEquals(v1, Bytes.toString(r.getValue(fam1, q1)));
3075    assertNull(r.getValue(fam1, q2));
3076    assertEquals(6L, Bytes.toLong(r.getValue(fam1, q3)));
3077    assertEquals("ab", Bytes.toString(r.getValue(fam1, q4)));
3078  }
3079
3080  // ////////////////////////////////////////////////////////////////////////////
3081  // Delete tests
3082  // ////////////////////////////////////////////////////////////////////////////
3083  @Test
3084  public void testDelete_multiDeleteColumn() throws IOException {
3085    byte[] row1 = Bytes.toBytes("row1");
3086    byte[] fam1 = Bytes.toBytes("fam1");
3087    byte[] qual = Bytes.toBytes("qualifier");
3088    byte[] value = Bytes.toBytes("value");
3089
3090    Put put = new Put(row1);
3091    put.addColumn(fam1, qual, 1, value);
3092    put.addColumn(fam1, qual, 2, value);
3093
3094    this.region = initHRegion(tableName, method, CONF, fam1);
3095    region.put(put);
3096
3097    // We do support deleting more than 1 'latest' version
3098    Delete delete = new Delete(row1);
3099    delete.addColumn(fam1, qual);
3100    delete.addColumn(fam1, qual);
3101    region.delete(delete);
3102
3103    Get get = new Get(row1);
3104    get.addFamily(fam1);
3105    Result r = region.get(get);
3106    assertEquals(0, r.size());
3107  }
3108
3109  @Test
3110  public void testDelete_CheckFamily() throws IOException {
3111    byte[] row1 = Bytes.toBytes("row1");
3112    byte[] fam1 = Bytes.toBytes("fam1");
3113    byte[] fam2 = Bytes.toBytes("fam2");
3114    byte[] fam3 = Bytes.toBytes("fam3");
3115    byte[] fam4 = Bytes.toBytes("fam4");
3116
3117    // Setting up region
3118    this.region = initHRegion(tableName, method, CONF, fam1, fam2, fam3);
3119    List<Cell> kvs = new ArrayList<>();
3120    kvs.add(new KeyValue(row1, fam4, null, null));
3121
3122    byte[] forUnitTestsOnly = Bytes.toBytes("ForUnitTestsOnly");
3123
3124    // testing existing family
3125    NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
3126    deleteMap.put(fam2, kvs);
3127    region.delete(new Delete(forUnitTestsOnly, HConstants.LATEST_TIMESTAMP, deleteMap));
3128
3129    // testing non existing family
3130    NavigableMap<byte[], List<Cell>> deleteMap2 = new TreeMap<>(Bytes.BYTES_COMPARATOR);
3131    deleteMap2.put(fam4, kvs);
3132    assertThrows("Family " + Bytes.toString(fam4) + " does exist",
3133      NoSuchColumnFamilyException.class,
3134      () -> region.delete(new Delete(forUnitTestsOnly, HConstants.LATEST_TIMESTAMP, deleteMap2)));
3135  }
3136
3137  @Test
3138  public void testDelete_mixed() throws IOException, InterruptedException {
3139    byte[] fam = Bytes.toBytes("info");
3140    byte[][] families = { fam };
3141    this.region = initHRegion(tableName, method, CONF, families);
3142    EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
3143
3144    byte[] row = Bytes.toBytes("table_name");
3145    // column names
3146    byte[] serverinfo = Bytes.toBytes("serverinfo");
3147    byte[] splitA = Bytes.toBytes("splitA");
3148    byte[] splitB = Bytes.toBytes("splitB");
3149
3150    // add some data:
3151    Put put = new Put(row);
3152    put.addColumn(fam, splitA, Bytes.toBytes("reference_A"));
3153    region.put(put);
3154
3155    put = new Put(row);
3156    put.addColumn(fam, splitB, Bytes.toBytes("reference_B"));
3157    region.put(put);
3158
3159    put = new Put(row);
3160    put.addColumn(fam, serverinfo, Bytes.toBytes("ip_address"));
3161    region.put(put);
3162
3163    // ok now delete a split:
3164    Delete delete = new Delete(row);
3165    delete.addColumns(fam, splitA);
3166    region.delete(delete);
3167
3168    // assert some things:
3169    Get get = new Get(row).addColumn(fam, serverinfo);
3170    Result result = region.get(get);
3171    assertEquals(1, result.size());
3172
3173    get = new Get(row).addColumn(fam, splitA);
3174    result = region.get(get);
3175    assertEquals(0, result.size());
3176
3177    get = new Get(row).addColumn(fam, splitB);
3178    result = region.get(get);
3179    assertEquals(1, result.size());
3180
3181    // Assert that after a delete, I can put.
3182    put = new Put(row);
3183    put.addColumn(fam, splitA, Bytes.toBytes("reference_A"));
3184    region.put(put);
3185    get = new Get(row);
3186    result = region.get(get);
3187    assertEquals(3, result.size());
3188
3189    // Now delete all... then test I can add stuff back
3190    delete = new Delete(row);
3191    region.delete(delete);
3192    assertEquals(0, region.get(get).size());
3193
3194    region.put(new Put(row).addColumn(fam, splitA, Bytes.toBytes("reference_A")));
3195    result = region.get(get);
3196    assertEquals(1, result.size());
3197  }
3198
3199  @Test
3200  public void testDeleteRowWithFutureTs() throws IOException {
3201    byte[] fam = Bytes.toBytes("info");
3202    byte[][] families = { fam };
3203    this.region = initHRegion(tableName, method, CONF, families);
3204    byte[] row = Bytes.toBytes("table_name");
3205    // column names
3206    byte[] serverinfo = Bytes.toBytes("serverinfo");
3207
3208    // add data in the far future
3209    Put put = new Put(row);
3210    put.addColumn(fam, serverinfo, HConstants.LATEST_TIMESTAMP - 5, Bytes.toBytes("value"));
3211    region.put(put);
3212
3213    // now delete something in the present
3214    Delete delete = new Delete(row);
3215    region.delete(delete);
3216
3217    // make sure we still see our data
3218    Get get = new Get(row).addColumn(fam, serverinfo);
3219    Result result = region.get(get);
3220    assertEquals(1, result.size());
3221
3222    // delete the future row
3223    delete = new Delete(row, HConstants.LATEST_TIMESTAMP - 3);
3224    region.delete(delete);
3225
3226    // make sure it is gone
3227    get = new Get(row).addColumn(fam, serverinfo);
3228    result = region.get(get);
3229    assertEquals(0, result.size());
3230  }
3231
3232  /**
3233   * Tests that the special LATEST_TIMESTAMP option for puts gets replaced by the actual timestamp
3234   */
3235  @Test
3236  public void testPutWithLatestTS() throws IOException {
3237    byte[] fam = Bytes.toBytes("info");
3238    byte[][] families = { fam };
3239    this.region = initHRegion(tableName, method, CONF, families);
3240    byte[] row = Bytes.toBytes("row1");
3241    // column names
3242    byte[] qual = Bytes.toBytes("qual");
3243
3244    // add data with LATEST_TIMESTAMP, put without WAL
3245    Put put = new Put(row);
3246    put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
3247    region.put(put);
3248
3249    // Make sure it shows up with an actual timestamp
3250    Get get = new Get(row).addColumn(fam, qual);
3251    Result result = region.get(get);
3252    assertEquals(1, result.size());
3253    Cell kv = result.rawCells()[0];
3254    LOG.info("Got: " + kv);
3255    assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
3256      kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
3257
3258    // Check same with WAL enabled (historically these took different
3259    // code paths, so check both)
3260    row = Bytes.toBytes("row2");
3261    put = new Put(row);
3262    put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
3263    region.put(put);
3264
3265    // Make sure it shows up with an actual timestamp
3266    get = new Get(row).addColumn(fam, qual);
3267    result = region.get(get);
3268    assertEquals(1, result.size());
3269    kv = result.rawCells()[0];
3270    LOG.info("Got: " + kv);
3271    assertTrue("LATEST_TIMESTAMP was not replaced with real timestamp",
3272      kv.getTimestamp() != HConstants.LATEST_TIMESTAMP);
3273  }
3274
3275  /**
3276   * Tests that there is server-side filtering for invalid timestamp upper bound. Note that the
3277   * timestamp lower bound is automatically handled for us by the TTL field.
3278   */
3279  @Test
3280  public void testPutWithTsSlop() throws IOException {
3281    byte[] fam = Bytes.toBytes("info");
3282    byte[][] families = { fam };
3283
3284    // add data with a timestamp that is too recent for range. Ensure assert
3285    CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
3286    this.region = initHRegion(tableName, method, CONF, families);
3287    boolean caughtExcep = false;
3288    try {
3289      // no TS specified == use latest. should not error
3290      region.put(new Put(row).addColumn(fam, Bytes.toBytes("qual"), Bytes.toBytes("value")));
3291      // TS out of range. should error
3292      region.put(new Put(row).addColumn(fam, Bytes.toBytes("qual"),
3293        EnvironmentEdgeManager.currentTime() + 2000, Bytes.toBytes("value")));
3294      fail("Expected IOE for TS out of configured timerange");
3295    } catch (FailedSanityCheckException ioe) {
3296      LOG.debug("Received expected exception", ioe);
3297      caughtExcep = true;
3298    }
3299    assertTrue("Should catch FailedSanityCheckException", caughtExcep);
3300  }
3301
3302  @Test
3303  public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
3304    byte[] fam1 = Bytes.toBytes("columnA");
3305    byte[] fam2 = Bytes.toBytes("columnB");
3306    this.region = initHRegion(tableName, method, CONF, fam1, fam2);
3307    byte[] rowA = Bytes.toBytes("rowA");
3308    byte[] rowB = Bytes.toBytes("rowB");
3309
3310    byte[] value = Bytes.toBytes("value");
3311
3312    Delete delete = new Delete(rowA);
3313    delete.addFamily(fam1);
3314
3315    region.delete(delete);
3316
3317    // now create data.
3318    Put put = new Put(rowA);
3319    put.addColumn(fam2, null, value);
3320    region.put(put);
3321
3322    put = new Put(rowB);
3323    put.addColumn(fam1, null, value);
3324    put.addColumn(fam2, null, value);
3325    region.put(put);
3326
3327    Scan scan = new Scan();
3328    scan.addFamily(fam1).addFamily(fam2);
3329    InternalScanner s = region.getScanner(scan);
3330    List<Cell> results = new ArrayList<>();
3331    s.next(results);
3332    assertTrue(CellUtil.matchingRows(results.get(0), rowA));
3333
3334    results.clear();
3335    s.next(results);
3336    assertTrue(CellUtil.matchingRows(results.get(0), rowB));
3337  }
3338
3339  @Test
3340  public void testDataInMemoryWithoutWAL() throws IOException {
3341    FileSystem fs = FileSystem.get(CONF);
3342    Path rootDir = new Path(dir + "testDataInMemoryWithoutWAL");
3343    FSHLog hLog = new FSHLog(fs, rootDir, "testDataInMemoryWithoutWAL", CONF);
3344    hLog.init();
3345    // This chunk creation is done throughout the code base. Do we want to move it into core?
3346    // It is missing from this test. W/o it we NPE.
3347    region = initHRegion(tableName, null, null, CONF, false, Durability.SYNC_WAL, hLog,
3348      COLUMN_FAMILY_BYTES);
3349
3350    Cell originalCell = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row)
3351      .setFamily(COLUMN_FAMILY_BYTES).setQualifier(qual1)
3352      .setTimestamp(EnvironmentEdgeManager.currentTime()).setType(KeyValue.Type.Put.getCode())
3353      .setValue(value1).build();
3354    final long originalSize = originalCell.getSerializedSize();
3355
3356    Cell addCell = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row)
3357      .setFamily(COLUMN_FAMILY_BYTES).setQualifier(qual1)
3358      .setTimestamp(EnvironmentEdgeManager.currentTime()).setType(KeyValue.Type.Put.getCode())
3359      .setValue(Bytes.toBytes("xxxxxxxxxx")).build();
3360    final long addSize = addCell.getSerializedSize();
3361
3362    LOG.info("originalSize:" + originalSize + ", addSize:" + addSize);
3363    // start test. We expect that the addPut's durability will be replaced
3364    // by originalPut's durability.
3365
3366    // case 1:
3367    testDataInMemoryWithoutWAL(region,
3368      new Put(row).add(originalCell).setDurability(Durability.SKIP_WAL),
3369      new Put(row).add(addCell).setDurability(Durability.SKIP_WAL), originalSize + addSize);
3370
3371    // case 2:
3372    testDataInMemoryWithoutWAL(region,
3373      new Put(row).add(originalCell).setDurability(Durability.SKIP_WAL),
3374      new Put(row).add(addCell).setDurability(Durability.SYNC_WAL), originalSize + addSize);
3375
3376    // case 3:
3377    testDataInMemoryWithoutWAL(region,
3378      new Put(row).add(originalCell).setDurability(Durability.SYNC_WAL),
3379      new Put(row).add(addCell).setDurability(Durability.SKIP_WAL), 0);
3380
3381    // case 4:
3382    testDataInMemoryWithoutWAL(region,
3383      new Put(row).add(originalCell).setDurability(Durability.SYNC_WAL),
3384      new Put(row).add(addCell).setDurability(Durability.SYNC_WAL), 0);
3385  }
3386
3387  private static void testDataInMemoryWithoutWAL(HRegion region, Put originalPut, final Put addPut,
3388    long delta) throws IOException {
3389    final long initSize = region.getDataInMemoryWithoutWAL();
3390    // save normalCPHost and replaced by mockedCPHost
3391    RegionCoprocessorHost normalCPHost = region.getCoprocessorHost();
3392    RegionCoprocessorHost mockedCPHost = Mockito.mock(RegionCoprocessorHost.class);
3393    // Because the preBatchMutate returns void, we can't do usual Mockito when...then form. Must
3394    // do below format (from Mockito doc).
3395    Mockito.doAnswer(new Answer<Void>() {
3396      @Override
3397      public Void answer(InvocationOnMock invocation) throws Throwable {
3398        MiniBatchOperationInProgress<Mutation> mb = invocation.getArgument(0);
3399        mb.addOperationsFromCP(0, new Mutation[] { addPut });
3400        return null;
3401      }
3402    }).when(mockedCPHost).preBatchMutate(Mockito.isA(MiniBatchOperationInProgress.class));
3403    ColumnFamilyDescriptorBuilder builder =
3404      ColumnFamilyDescriptorBuilder.newBuilder(COLUMN_FAMILY_BYTES);
3405    ScanInfo info = new ScanInfo(CONF, builder.build(), Long.MAX_VALUE, Long.MAX_VALUE,
3406      region.getCellComparator());
3407    Mockito.when(mockedCPHost.preFlushScannerOpen(Mockito.any(HStore.class), Mockito.any()))
3408      .thenReturn(info);
3409    Mockito
3410      .when(mockedCPHost.preFlush(Mockito.any(), Mockito.any(StoreScanner.class), Mockito.any()))
3411      .thenAnswer(i -> i.getArgument(1));
3412    region.setCoprocessorHost(mockedCPHost);
3413
3414    region.put(originalPut);
3415    region.setCoprocessorHost(normalCPHost);
3416    final long finalSize = region.getDataInMemoryWithoutWAL();
3417    assertEquals("finalSize:" + finalSize + ", initSize:" + initSize + ", delta:" + delta,
3418      finalSize, initSize + delta);
3419  }
3420
3421  @Test
3422  public void testDeleteColumns_PostInsert() throws IOException, InterruptedException {
3423    Delete delete = new Delete(row);
3424    delete.addColumns(fam1, qual1);
3425    doTestDelete_AndPostInsert(delete);
3426  }
3427
3428  @Test
3429  public void testaddFamily_PostInsert() throws IOException, InterruptedException {
3430    Delete delete = new Delete(row);
3431    delete.addFamily(fam1);
3432    doTestDelete_AndPostInsert(delete);
3433  }
3434
3435  public void doTestDelete_AndPostInsert(Delete delete) throws IOException, InterruptedException {
3436    this.region = initHRegion(tableName, method, CONF, fam1);
3437    EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
3438    Put put = new Put(row);
3439    put.addColumn(fam1, qual1, value1);
3440    region.put(put);
3441
3442    // now delete the value:
3443    region.delete(delete);
3444
3445    // ok put data:
3446    put = new Put(row);
3447    put.addColumn(fam1, qual1, value2);
3448    region.put(put);
3449
3450    // ok get:
3451    Get get = new Get(row);
3452    get.addColumn(fam1, qual1);
3453
3454    Result r = region.get(get);
3455    assertEquals(1, r.size());
3456    assertArrayEquals(value2, r.getValue(fam1, qual1));
3457
3458    // next:
3459    Scan scan = new Scan().withStartRow(row);
3460    scan.addColumn(fam1, qual1);
3461    InternalScanner s = region.getScanner(scan);
3462
3463    List<Cell> results = new ArrayList<>();
3464    assertEquals(false, s.next(results));
3465    assertEquals(1, results.size());
3466    Cell kv = results.get(0);
3467
3468    assertArrayEquals(value2, CellUtil.cloneValue(kv));
3469    assertArrayEquals(fam1, CellUtil.cloneFamily(kv));
3470    assertArrayEquals(qual1, CellUtil.cloneQualifier(kv));
3471    assertArrayEquals(row, CellUtil.cloneRow(kv));
3472  }
3473
3474  @Test
3475  public void testDelete_CheckTimestampUpdated() throws IOException {
3476    byte[] row1 = Bytes.toBytes("row1");
3477    byte[] col1 = Bytes.toBytes("col1");
3478    byte[] col2 = Bytes.toBytes("col2");
3479    byte[] col3 = Bytes.toBytes("col3");
3480
3481    byte[] forUnitTestsOnly = Bytes.toBytes("ForUnitTestsOnly");
3482
3483    // Setting up region
3484    this.region = initHRegion(tableName, method, CONF, fam1);
3485    // Building checkerList
3486    List<Cell> kvs = new ArrayList<>();
3487    kvs.add(new KeyValue(row1, fam1, col1, null));
3488    kvs.add(new KeyValue(row1, fam1, col2, null));
3489    kvs.add(new KeyValue(row1, fam1, col3, null));
3490
3491    NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
3492    deleteMap.put(fam1, kvs);
3493    region.delete(new Delete(forUnitTestsOnly, HConstants.LATEST_TIMESTAMP, deleteMap));
3494
3495    // extract the key values out the memstore:
3496    // This is kinda hacky, but better than nothing...
3497    long now = EnvironmentEdgeManager.currentTime();
3498    AbstractMemStore memstore = (AbstractMemStore) region.getStore(fam1).memstore;
3499    Cell firstCell = memstore.getActive().first();
3500    assertTrue(firstCell.getTimestamp() <= now);
3501    now = firstCell.getTimestamp();
3502    for (Cell cell : memstore.getActive().getCellSet()) {
3503      assertTrue(cell.getTimestamp() <= now);
3504      now = cell.getTimestamp();
3505    }
3506  }
3507
3508  // ////////////////////////////////////////////////////////////////////////////
3509  // Get tests
3510  // ////////////////////////////////////////////////////////////////////////////
3511  @Test
3512  public void testGet_FamilyChecker() throws IOException {
3513    byte[] row1 = Bytes.toBytes("row1");
3514    byte[] fam1 = Bytes.toBytes("fam1");
3515    byte[] fam2 = Bytes.toBytes("False");
3516    byte[] col1 = Bytes.toBytes("col1");
3517
3518    // Setting up region
3519    this.region = initHRegion(tableName, method, CONF, fam1);
3520    Get get = new Get(row1);
3521    get.addColumn(fam2, col1);
3522
3523    // Test
3524    try {
3525      region.get(get);
3526      fail("Expecting DoNotRetryIOException in get but did not get any");
3527    } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
3528      LOG.info("Got expected DoNotRetryIOException successfully");
3529    }
3530  }
3531
3532  @Test
3533  public void testGet_Basic() throws IOException {
3534    byte[] row1 = Bytes.toBytes("row1");
3535    byte[] fam1 = Bytes.toBytes("fam1");
3536    byte[] col1 = Bytes.toBytes("col1");
3537    byte[] col2 = Bytes.toBytes("col2");
3538    byte[] col3 = Bytes.toBytes("col3");
3539    byte[] col4 = Bytes.toBytes("col4");
3540    byte[] col5 = Bytes.toBytes("col5");
3541
3542    // Setting up region
3543    this.region = initHRegion(tableName, method, CONF, fam1);
3544    // Add to memstore
3545    Put put = new Put(row1);
3546    put.addColumn(fam1, col1, null);
3547    put.addColumn(fam1, col2, null);
3548    put.addColumn(fam1, col3, null);
3549    put.addColumn(fam1, col4, null);
3550    put.addColumn(fam1, col5, null);
3551    region.put(put);
3552
3553    Get get = new Get(row1);
3554    get.addColumn(fam1, col2);
3555    get.addColumn(fam1, col4);
3556    // Expected result
3557    KeyValue kv1 = new KeyValue(row1, fam1, col2);
3558    KeyValue kv2 = new KeyValue(row1, fam1, col4);
3559    KeyValue[] expected = { kv1, kv2 };
3560
3561    // Test
3562    Result res = region.get(get);
3563    assertEquals(expected.length, res.size());
3564    for (int i = 0; i < res.size(); i++) {
3565      assertTrue(CellUtil.matchingRows(expected[i], res.rawCells()[i]));
3566      assertTrue(CellUtil.matchingFamily(expected[i], res.rawCells()[i]));
3567      assertTrue(CellUtil.matchingQualifier(expected[i], res.rawCells()[i]));
3568    }
3569
3570    // Test using a filter on a Get
3571    Get g = new Get(row1);
3572    final int count = 2;
3573    g.setFilter(new ColumnCountGetFilter(count));
3574    res = region.get(g);
3575    assertEquals(count, res.size());
3576  }
3577
3578  @Test
3579  public void testGet_Empty() throws IOException {
3580    byte[] row = Bytes.toBytes("row");
3581    byte[] fam = Bytes.toBytes("fam");
3582
3583    this.region = initHRegion(tableName, method, CONF, fam);
3584    Get get = new Get(row);
3585    get.addFamily(fam);
3586    Result r = region.get(get);
3587
3588    assertTrue(r.isEmpty());
3589  }
3590
3591  @Test
3592  public void testGetWithFilter() throws IOException, InterruptedException {
3593    byte[] row1 = Bytes.toBytes("row1");
3594    byte[] fam1 = Bytes.toBytes("fam1");
3595    byte[] col1 = Bytes.toBytes("col1");
3596    byte[] value1 = Bytes.toBytes("value1");
3597    byte[] value2 = Bytes.toBytes("value2");
3598
3599    final int maxVersions = 3;
3600    TableDescriptor tableDescriptor =
3601      TableDescriptorBuilder.newBuilder(TableName.valueOf("testFilterAndColumnTracker"))
3602        .setColumnFamily(
3603          ColumnFamilyDescriptorBuilder.newBuilder(fam1).setMaxVersions(maxVersions).build())
3604        .build();
3605    ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null,
3606      MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);
3607    RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
3608    Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + ".log");
3609    final WAL wal = HBaseTestingUtil.createWal(TEST_UTIL.getConfiguration(), logDir, info);
3610    this.region = TEST_UTIL.createLocalHRegion(info, CONF, tableDescriptor, wal);
3611
3612    // Put 4 version to memstore
3613    long ts = 0;
3614    Put put = new Put(row1, ts);
3615    put.addColumn(fam1, col1, value1);
3616    region.put(put);
3617    put = new Put(row1, ts + 1);
3618    put.addColumn(fam1, col1, Bytes.toBytes("filter1"));
3619    region.put(put);
3620    put = new Put(row1, ts + 2);
3621    put.addColumn(fam1, col1, Bytes.toBytes("filter2"));
3622    region.put(put);
3623    put = new Put(row1, ts + 3);
3624    put.addColumn(fam1, col1, value2);
3625    region.put(put);
3626
3627    Get get = new Get(row1);
3628    get.readAllVersions();
3629    Result res = region.get(get);
3630    // Get 3 versions, the oldest version has gone from user view
3631    assertEquals(maxVersions, res.size());
3632
3633    get.setFilter(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("value")));
3634    res = region.get(get);
3635    // When use value filter, the oldest version should still gone from user view and it
3636    // should only return one key vaule
3637    assertEquals(1, res.size());
3638    assertTrue(CellUtil.matchingValue(new KeyValue(row1, fam1, col1, value2), res.rawCells()[0]));
3639    assertEquals(ts + 3, res.rawCells()[0].getTimestamp());
3640
3641    region.flush(true);
3642    region.compact(true);
3643    Thread.sleep(1000);
3644    res = region.get(get);
3645    // After flush and compact, the result should be consistent with previous result
3646    assertEquals(1, res.size());
3647    assertTrue(CellUtil.matchingValue(new KeyValue(row1, fam1, col1, value2), res.rawCells()[0]));
3648  }
3649
3650  // ////////////////////////////////////////////////////////////////////////////
3651  // Scanner tests
3652  // ////////////////////////////////////////////////////////////////////////////
3653  @Test
3654  public void testGetScanner_WithOkFamilies() throws IOException {
3655    byte[] fam1 = Bytes.toBytes("fam1");
3656    byte[] fam2 = Bytes.toBytes("fam2");
3657
3658    byte[][] families = { fam1, fam2 };
3659
3660    // Setting up region
3661    this.region = initHRegion(tableName, method, CONF, families);
3662    Scan scan = new Scan();
3663    scan.addFamily(fam1);
3664    scan.addFamily(fam2);
3665    try {
3666      region.getScanner(scan);
3667    } catch (Exception e) {
3668      assertTrue("Families could not be found in Region", false);
3669    }
3670  }
3671
3672  @Test
3673  public void testGetScanner_WithNotOkFamilies() throws IOException {
3674    byte[] fam1 = Bytes.toBytes("fam1");
3675    byte[] fam2 = Bytes.toBytes("fam2");
3676
3677    byte[][] families = { fam1 };
3678
3679    // Setting up region
3680    this.region = initHRegion(tableName, method, CONF, families);
3681    Scan scan = new Scan();
3682    scan.addFamily(fam2);
3683    boolean ok = false;
3684    try {
3685      region.getScanner(scan);
3686    } catch (Exception e) {
3687      ok = true;
3688    }
3689    assertTrue("Families could not be found in Region", ok);
3690  }
3691
3692  @Test
3693  public void testGetScanner_WithNoFamilies() throws IOException {
3694    byte[] row1 = Bytes.toBytes("row1");
3695    byte[] fam1 = Bytes.toBytes("fam1");
3696    byte[] fam2 = Bytes.toBytes("fam2");
3697    byte[] fam3 = Bytes.toBytes("fam3");
3698    byte[] fam4 = Bytes.toBytes("fam4");
3699
3700    byte[][] families = { fam1, fam2, fam3, fam4 };
3701
3702    // Setting up region
3703    this.region = initHRegion(tableName, method, CONF, families);
3704    // Putting data in Region
3705    Put put = new Put(row1);
3706    put.addColumn(fam1, null, null);
3707    put.addColumn(fam2, null, null);
3708    put.addColumn(fam3, null, null);
3709    put.addColumn(fam4, null, null);
3710    region.put(put);
3711
3712    Scan scan = null;
3713    RegionScannerImpl is = null;
3714
3715    // Testing to see how many scanners that is produced by getScanner,
3716    // starting
3717    // with known number, 2 - current = 1
3718    scan = new Scan();
3719    scan.addFamily(fam2);
3720    scan.addFamily(fam4);
3721    is = region.getScanner(scan);
3722    assertEquals(1, is.storeHeap.getHeap().size());
3723
3724    scan = new Scan();
3725    is = region.getScanner(scan);
3726    assertEquals(families.length - 1, is.storeHeap.getHeap().size());
3727  }
3728
3729  /**
3730   * This method tests https://issues.apache.org/jira/browse/HBASE-2516. n
3731   */
3732  @Test
3733  public void testGetScanner_WithRegionClosed() throws IOException {
3734    byte[] fam1 = Bytes.toBytes("fam1");
3735    byte[] fam2 = Bytes.toBytes("fam2");
3736
3737    byte[][] families = { fam1, fam2 };
3738
3739    // Setting up region
3740    try {
3741      this.region = initHRegion(tableName, method, CONF, families);
3742    } catch (IOException e) {
3743      e.printStackTrace();
3744      fail("Got IOException during initHRegion, " + e.getMessage());
3745    }
3746    region.closed.set(true);
3747    try {
3748      region.getScanner(null);
3749      fail("Expected to get an exception during getScanner on a region that is closed");
3750    } catch (NotServingRegionException e) {
3751      // this is the correct exception that is expected
3752    } catch (IOException e) {
3753      fail("Got wrong type of exception - should be a NotServingRegionException, "
3754        + "but was an IOException: " + e.getMessage());
3755    }
3756  }
3757
3758  @Test
3759  public void testRegionScanner_Next() throws IOException {
3760    byte[] row1 = Bytes.toBytes("row1");
3761    byte[] row2 = Bytes.toBytes("row2");
3762    byte[] fam1 = Bytes.toBytes("fam1");
3763    byte[] fam2 = Bytes.toBytes("fam2");
3764    byte[] fam3 = Bytes.toBytes("fam3");
3765    byte[] fam4 = Bytes.toBytes("fam4");
3766
3767    byte[][] families = { fam1, fam2, fam3, fam4 };
3768    long ts = EnvironmentEdgeManager.currentTime();
3769
3770    // Setting up region
3771    this.region = initHRegion(tableName, method, CONF, families);
3772    // Putting data in Region
3773    Put put = null;
3774    put = new Put(row1);
3775    put.addColumn(fam1, (byte[]) null, ts, null);
3776    put.addColumn(fam2, (byte[]) null, ts, null);
3777    put.addColumn(fam3, (byte[]) null, ts, null);
3778    put.addColumn(fam4, (byte[]) null, ts, null);
3779    region.put(put);
3780
3781    put = new Put(row2);
3782    put.addColumn(fam1, (byte[]) null, ts, null);
3783    put.addColumn(fam2, (byte[]) null, ts, null);
3784    put.addColumn(fam3, (byte[]) null, ts, null);
3785    put.addColumn(fam4, (byte[]) null, ts, null);
3786    region.put(put);
3787
3788    Scan scan = new Scan();
3789    scan.addFamily(fam2);
3790    scan.addFamily(fam4);
3791    InternalScanner is = region.getScanner(scan);
3792
3793    List<Cell> res = null;
3794
3795    // Result 1
3796    List<Cell> expected1 = new ArrayList<>();
3797    expected1.add(new KeyValue(row1, fam2, null, ts, KeyValue.Type.Put, null));
3798    expected1.add(new KeyValue(row1, fam4, null, ts, KeyValue.Type.Put, null));
3799
3800    res = new ArrayList<>();
3801    is.next(res);
3802    for (int i = 0; i < res.size(); i++) {
3803      assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected1.get(i), res.get(i)));
3804    }
3805
3806    // Result 2
3807    List<Cell> expected2 = new ArrayList<>();
3808    expected2.add(new KeyValue(row2, fam2, null, ts, KeyValue.Type.Put, null));
3809    expected2.add(new KeyValue(row2, fam4, null, ts, KeyValue.Type.Put, null));
3810
3811    res = new ArrayList<>();
3812    is.next(res);
3813    for (int i = 0; i < res.size(); i++) {
3814      assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected2.get(i), res.get(i)));
3815    }
3816  }
3817
3818  @Test
3819  public void testScanner_ExplicitColumns_FromMemStore_EnforceVersions() throws IOException {
3820    byte[] row1 = Bytes.toBytes("row1");
3821    byte[] qf1 = Bytes.toBytes("qualifier1");
3822    byte[] qf2 = Bytes.toBytes("qualifier2");
3823    byte[] fam1 = Bytes.toBytes("fam1");
3824    byte[][] families = { fam1 };
3825
3826    long ts1 = EnvironmentEdgeManager.currentTime();
3827    long ts2 = ts1 + 1;
3828    long ts3 = ts1 + 2;
3829
3830    // Setting up region
3831    this.region = initHRegion(tableName, method, CONF, families);
3832    // Putting data in Region
3833    Put put = null;
3834    KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
3835    KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
3836    KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
3837
3838    KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
3839    KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
3840    KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
3841
3842    put = new Put(row1);
3843    put.add(kv13);
3844    put.add(kv12);
3845    put.add(kv11);
3846    put.add(kv23);
3847    put.add(kv22);
3848    put.add(kv21);
3849    region.put(put);
3850
3851    // Expected
3852    List<Cell> expected = new ArrayList<>();
3853    expected.add(kv13);
3854    expected.add(kv12);
3855
3856    Scan scan = new Scan().withStartRow(row1);
3857    scan.addColumn(fam1, qf1);
3858    scan.readVersions(MAX_VERSIONS);
3859    List<Cell> actual = new ArrayList<>();
3860    InternalScanner scanner = region.getScanner(scan);
3861
3862    boolean hasNext = scanner.next(actual);
3863    assertEquals(false, hasNext);
3864
3865    // Verify result
3866    for (int i = 0; i < expected.size(); i++) {
3867      assertEquals(expected.get(i), actual.get(i));
3868    }
3869  }
3870
3871  @Test
3872  public void testScanner_ExplicitColumns_FromFilesOnly_EnforceVersions() throws IOException {
3873    byte[] row1 = Bytes.toBytes("row1");
3874    byte[] qf1 = Bytes.toBytes("qualifier1");
3875    byte[] qf2 = Bytes.toBytes("qualifier2");
3876    byte[] fam1 = Bytes.toBytes("fam1");
3877    byte[][] families = { fam1 };
3878
3879    long ts1 = 1;
3880    long ts2 = ts1 + 1;
3881    long ts3 = ts1 + 2;
3882
3883    // Setting up region
3884    this.region = initHRegion(tableName, method, CONF, families);
3885    // Putting data in Region
3886    Put put = null;
3887    KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
3888    KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
3889    KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
3890
3891    KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
3892    KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
3893    KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
3894
3895    put = new Put(row1);
3896    put.add(kv13);
3897    put.add(kv12);
3898    put.add(kv11);
3899    put.add(kv23);
3900    put.add(kv22);
3901    put.add(kv21);
3902    region.put(put);
3903    region.flush(true);
3904
3905    // Expected
3906    List<Cell> expected = new ArrayList<>();
3907    expected.add(kv13);
3908    expected.add(kv12);
3909    expected.add(kv23);
3910    expected.add(kv22);
3911
3912    Scan scan = new Scan().withStartRow(row1);
3913    scan.addColumn(fam1, qf1);
3914    scan.addColumn(fam1, qf2);
3915    scan.readVersions(MAX_VERSIONS);
3916    List<Cell> actual = new ArrayList<>();
3917    InternalScanner scanner = region.getScanner(scan);
3918
3919    boolean hasNext = scanner.next(actual);
3920    assertEquals(false, hasNext);
3921
3922    // Verify result
3923    for (int i = 0; i < expected.size(); i++) {
3924      assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
3925    }
3926  }
3927
3928  @Test
3929  public void testScanner_ExplicitColumns_FromMemStoreAndFiles_EnforceVersions()
3930    throws IOException {
3931    byte[] row1 = Bytes.toBytes("row1");
3932    byte[] fam1 = Bytes.toBytes("fam1");
3933    byte[][] families = { fam1 };
3934    byte[] qf1 = Bytes.toBytes("qualifier1");
3935    byte[] qf2 = Bytes.toBytes("qualifier2");
3936
3937    long ts1 = 1;
3938    long ts2 = ts1 + 1;
3939    long ts3 = ts1 + 2;
3940    long ts4 = ts1 + 3;
3941
3942    // Setting up region
3943    this.region = initHRegion(tableName, method, CONF, families);
3944    // Putting data in Region
3945    KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
3946    KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
3947    KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
3948    KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
3949
3950    KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
3951    KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
3952    KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
3953    KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
3954
3955    Put put = null;
3956    put = new Put(row1);
3957    put.add(kv14);
3958    put.add(kv24);
3959    region.put(put);
3960    region.flush(true);
3961
3962    put = new Put(row1);
3963    put.add(kv23);
3964    put.add(kv13);
3965    region.put(put);
3966    region.flush(true);
3967
3968    put = new Put(row1);
3969    put.add(kv22);
3970    put.add(kv12);
3971    region.put(put);
3972    region.flush(true);
3973
3974    put = new Put(row1);
3975    put.add(kv21);
3976    put.add(kv11);
3977    region.put(put);
3978
3979    // Expected
3980    List<Cell> expected = new ArrayList<>();
3981    expected.add(kv14);
3982    expected.add(kv13);
3983    expected.add(kv12);
3984    expected.add(kv24);
3985    expected.add(kv23);
3986    expected.add(kv22);
3987
3988    Scan scan = new Scan().withStartRow(row1);
3989    scan.addColumn(fam1, qf1);
3990    scan.addColumn(fam1, qf2);
3991    int versions = 3;
3992    scan.readVersions(versions);
3993    List<Cell> actual = new ArrayList<>();
3994    InternalScanner scanner = region.getScanner(scan);
3995
3996    boolean hasNext = scanner.next(actual);
3997    assertEquals(false, hasNext);
3998
3999    // Verify result
4000    for (int i = 0; i < expected.size(); i++) {
4001      assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
4002    }
4003  }
4004
4005  @Test
4006  public void testScanner_Wildcard_FromMemStore_EnforceVersions() throws IOException {
4007    byte[] row1 = Bytes.toBytes("row1");
4008    byte[] qf1 = Bytes.toBytes("qualifier1");
4009    byte[] qf2 = Bytes.toBytes("qualifier2");
4010    byte[] fam1 = Bytes.toBytes("fam1");
4011    byte[][] families = { fam1 };
4012
4013    long ts1 = EnvironmentEdgeManager.currentTime();
4014    long ts2 = ts1 + 1;
4015    long ts3 = ts1 + 2;
4016
4017    // Setting up region
4018    this.region = initHRegion(tableName, method, CONF, families);
4019    // Putting data in Region
4020    Put put = null;
4021    KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
4022    KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
4023    KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
4024
4025    KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
4026    KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
4027    KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
4028
4029    put = new Put(row1);
4030    put.add(kv13);
4031    put.add(kv12);
4032    put.add(kv11);
4033    put.add(kv23);
4034    put.add(kv22);
4035    put.add(kv21);
4036    region.put(put);
4037
4038    // Expected
4039    List<Cell> expected = new ArrayList<>();
4040    expected.add(kv13);
4041    expected.add(kv12);
4042    expected.add(kv23);
4043    expected.add(kv22);
4044
4045    Scan scan = new Scan().withStartRow(row1);
4046    scan.addFamily(fam1);
4047    scan.readVersions(MAX_VERSIONS);
4048    List<Cell> actual = new ArrayList<>();
4049    InternalScanner scanner = region.getScanner(scan);
4050
4051    boolean hasNext = scanner.next(actual);
4052    assertEquals(false, hasNext);
4053
4054    // Verify result
4055    for (int i = 0; i < expected.size(); i++) {
4056      assertEquals(expected.get(i), actual.get(i));
4057    }
4058  }
4059
4060  @Test
4061  public void testScanner_Wildcard_FromFilesOnly_EnforceVersions() throws IOException {
4062    byte[] row1 = Bytes.toBytes("row1");
4063    byte[] qf1 = Bytes.toBytes("qualifier1");
4064    byte[] qf2 = Bytes.toBytes("qualifier2");
4065    byte[] fam1 = Bytes.toBytes("fam1");
4066
4067    long ts1 = 1;
4068    long ts2 = ts1 + 1;
4069    long ts3 = ts1 + 2;
4070
4071    // Setting up region
4072    this.region = initHRegion(tableName, method, CONF, fam1);
4073    // Putting data in Region
4074    Put put = null;
4075    KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
4076    KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
4077    KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
4078
4079    KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
4080    KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
4081    KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
4082
4083    put = new Put(row1);
4084    put.add(kv13);
4085    put.add(kv12);
4086    put.add(kv11);
4087    put.add(kv23);
4088    put.add(kv22);
4089    put.add(kv21);
4090    region.put(put);
4091    region.flush(true);
4092
4093    // Expected
4094    List<Cell> expected = new ArrayList<>();
4095    expected.add(kv13);
4096    expected.add(kv12);
4097    expected.add(kv23);
4098    expected.add(kv22);
4099
4100    Scan scan = new Scan().withStartRow(row1);
4101    scan.addFamily(fam1);
4102    scan.readVersions(MAX_VERSIONS);
4103    List<Cell> actual = new ArrayList<>();
4104    InternalScanner scanner = region.getScanner(scan);
4105
4106    boolean hasNext = scanner.next(actual);
4107    assertEquals(false, hasNext);
4108
4109    // Verify result
4110    for (int i = 0; i < expected.size(); i++) {
4111      assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
4112    }
4113  }
4114
4115  @Test
4116  public void testScanner_StopRow1542() throws IOException {
4117    byte[] family = Bytes.toBytes("testFamily");
4118    this.region = initHRegion(tableName, method, CONF, family);
4119    byte[] row1 = Bytes.toBytes("row111");
4120    byte[] row2 = Bytes.toBytes("row222");
4121    byte[] row3 = Bytes.toBytes("row333");
4122    byte[] row4 = Bytes.toBytes("row444");
4123    byte[] row5 = Bytes.toBytes("row555");
4124
4125    byte[] col1 = Bytes.toBytes("Pub111");
4126    byte[] col2 = Bytes.toBytes("Pub222");
4127
4128    Put put = new Put(row1);
4129    put.addColumn(family, col1, Bytes.toBytes(10L));
4130    region.put(put);
4131
4132    put = new Put(row2);
4133    put.addColumn(family, col1, Bytes.toBytes(15L));
4134    region.put(put);
4135
4136    put = new Put(row3);
4137    put.addColumn(family, col2, Bytes.toBytes(20L));
4138    region.put(put);
4139
4140    put = new Put(row4);
4141    put.addColumn(family, col2, Bytes.toBytes(30L));
4142    region.put(put);
4143
4144    put = new Put(row5);
4145    put.addColumn(family, col1, Bytes.toBytes(40L));
4146    region.put(put);
4147
4148    Scan scan = new Scan().withStartRow(row3).withStopRow(row4);
4149    scan.readAllVersions();
4150    scan.addColumn(family, col1);
4151    InternalScanner s = region.getScanner(scan);
4152
4153    List<Cell> results = new ArrayList<>();
4154    assertEquals(false, s.next(results));
4155    assertEquals(0, results.size());
4156  }
4157
4158  @Test
4159  public void testScanner_Wildcard_FromMemStoreAndFiles_EnforceVersions() throws IOException {
4160    byte[] row1 = Bytes.toBytes("row1");
4161    byte[] fam1 = Bytes.toBytes("fam1");
4162    byte[] qf1 = Bytes.toBytes("qualifier1");
4163    byte[] qf2 = Bytes.toBytes("quateslifier2");
4164
4165    long ts1 = 1;
4166    long ts2 = ts1 + 1;
4167    long ts3 = ts1 + 2;
4168    long ts4 = ts1 + 3;
4169
4170    // Setting up region
4171    this.region = initHRegion(tableName, method, CONF, fam1);
4172    // Putting data in Region
4173    KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
4174    KeyValue kv13 = new KeyValue(row1, fam1, qf1, ts3, KeyValue.Type.Put, null);
4175    KeyValue kv12 = new KeyValue(row1, fam1, qf1, ts2, KeyValue.Type.Put, null);
4176    KeyValue kv11 = new KeyValue(row1, fam1, qf1, ts1, KeyValue.Type.Put, null);
4177
4178    KeyValue kv24 = new KeyValue(row1, fam1, qf2, ts4, KeyValue.Type.Put, null);
4179    KeyValue kv23 = new KeyValue(row1, fam1, qf2, ts3, KeyValue.Type.Put, null);
4180    KeyValue kv22 = new KeyValue(row1, fam1, qf2, ts2, KeyValue.Type.Put, null);
4181    KeyValue kv21 = new KeyValue(row1, fam1, qf2, ts1, KeyValue.Type.Put, null);
4182
4183    Put put = null;
4184    put = new Put(row1);
4185    put.add(kv14);
4186    put.add(kv24);
4187    region.put(put);
4188    region.flush(true);
4189
4190    put = new Put(row1);
4191    put.add(kv23);
4192    put.add(kv13);
4193    region.put(put);
4194    region.flush(true);
4195
4196    put = new Put(row1);
4197    put.add(kv22);
4198    put.add(kv12);
4199    region.put(put);
4200    region.flush(true);
4201
4202    put = new Put(row1);
4203    put.add(kv21);
4204    put.add(kv11);
4205    region.put(put);
4206
4207    // Expected
4208    List<KeyValue> expected = new ArrayList<>();
4209    expected.add(kv14);
4210    expected.add(kv13);
4211    expected.add(kv12);
4212    expected.add(kv24);
4213    expected.add(kv23);
4214    expected.add(kv22);
4215
4216    Scan scan = new Scan().withStartRow(row1);
4217    int versions = 3;
4218    scan.readVersions(versions);
4219    List<Cell> actual = new ArrayList<>();
4220    InternalScanner scanner = region.getScanner(scan);
4221
4222    boolean hasNext = scanner.next(actual);
4223    assertEquals(false, hasNext);
4224
4225    // Verify result
4226    for (int i = 0; i < expected.size(); i++) {
4227      assertTrue(PrivateCellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i)));
4228    }
4229  }
4230
4231  /**
4232   * Added for HBASE-5416 Here we test scan optimization when only subset of CFs are used in filter
4233   * conditions.
4234   */
4235  @Test
4236  public void testScanner_JoinedScanners() throws IOException {
4237    byte[] cf_essential = Bytes.toBytes("essential");
4238    byte[] cf_joined = Bytes.toBytes("joined");
4239    byte[] cf_alpha = Bytes.toBytes("alpha");
4240    this.region = initHRegion(tableName, method, CONF, cf_essential, cf_joined, cf_alpha);
4241    byte[] row1 = Bytes.toBytes("row1");
4242    byte[] row2 = Bytes.toBytes("row2");
4243    byte[] row3 = Bytes.toBytes("row3");
4244
4245    byte[] col_normal = Bytes.toBytes("d");
4246    byte[] col_alpha = Bytes.toBytes("a");
4247
4248    byte[] filtered_val = Bytes.toBytes(3);
4249
4250    Put put = new Put(row1);
4251    put.addColumn(cf_essential, col_normal, Bytes.toBytes(1));
4252    put.addColumn(cf_joined, col_alpha, Bytes.toBytes(1));
4253    region.put(put);
4254
4255    put = new Put(row2);
4256    put.addColumn(cf_essential, col_alpha, Bytes.toBytes(2));
4257    put.addColumn(cf_joined, col_normal, Bytes.toBytes(2));
4258    put.addColumn(cf_alpha, col_alpha, Bytes.toBytes(2));
4259    region.put(put);
4260
4261    put = new Put(row3);
4262    put.addColumn(cf_essential, col_normal, filtered_val);
4263    put.addColumn(cf_joined, col_normal, filtered_val);
4264    region.put(put);
4265
4266    // Check two things:
4267    // 1. result list contains expected values
4268    // 2. result list is sorted properly
4269
4270    Scan scan = new Scan();
4271    Filter filter = new SingleColumnValueExcludeFilter(cf_essential, col_normal,
4272      CompareOperator.NOT_EQUAL, filtered_val);
4273    scan.setFilter(filter);
4274    scan.setLoadColumnFamiliesOnDemand(true);
4275    InternalScanner s = region.getScanner(scan);
4276
4277    List<Cell> results = new ArrayList<>();
4278    assertTrue(s.next(results));
4279    assertEquals(1, results.size());
4280    results.clear();
4281
4282    assertTrue(s.next(results));
4283    assertEquals(3, results.size());
4284    assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
4285    assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
4286    assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
4287    results.clear();
4288
4289    assertFalse(s.next(results));
4290    assertEquals(0, results.size());
4291  }
4292
4293  /**
4294   * HBASE-5416 Test case when scan limits amount of KVs returned on each next() call.
4295   */
4296  @Test
4297  public void testScanner_JoinedScannersWithLimits() throws IOException {
4298    final byte[] cf_first = Bytes.toBytes("first");
4299    final byte[] cf_second = Bytes.toBytes("second");
4300
4301    this.region = initHRegion(tableName, method, CONF, cf_first, cf_second);
4302    final byte[] col_a = Bytes.toBytes("a");
4303    final byte[] col_b = Bytes.toBytes("b");
4304
4305    Put put;
4306
4307    for (int i = 0; i < 10; i++) {
4308      put = new Put(Bytes.toBytes("r" + Integer.toString(i)));
4309      put.addColumn(cf_first, col_a, Bytes.toBytes(i));
4310      if (i < 5) {
4311        put.addColumn(cf_first, col_b, Bytes.toBytes(i));
4312        put.addColumn(cf_second, col_a, Bytes.toBytes(i));
4313        put.addColumn(cf_second, col_b, Bytes.toBytes(i));
4314      }
4315      region.put(put);
4316    }
4317
4318    Scan scan = new Scan();
4319    scan.setLoadColumnFamiliesOnDemand(true);
4320    Filter bogusFilter = new FilterBase() {
4321      @Override
4322      public ReturnCode filterCell(final Cell ignored) throws IOException {
4323        return ReturnCode.INCLUDE;
4324      }
4325
4326      @Override
4327      public boolean isFamilyEssential(byte[] name) {
4328        return Bytes.equals(name, cf_first);
4329      }
4330    };
4331
4332    scan.setFilter(bogusFilter);
4333    InternalScanner s = region.getScanner(scan);
4334
4335    // Our data looks like this:
4336    // r0: first:a, first:b, second:a, second:b
4337    // r1: first:a, first:b, second:a, second:b
4338    // r2: first:a, first:b, second:a, second:b
4339    // r3: first:a, first:b, second:a, second:b
4340    // r4: first:a, first:b, second:a, second:b
4341    // r5: first:a
4342    // r6: first:a
4343    // r7: first:a
4344    // r8: first:a
4345    // r9: first:a
4346
4347    // But due to next's limit set to 3, we should get this:
4348    // r0: first:a, first:b, second:a
4349    // r0: second:b
4350    // r1: first:a, first:b, second:a
4351    // r1: second:b
4352    // r2: first:a, first:b, second:a
4353    // r2: second:b
4354    // r3: first:a, first:b, second:a
4355    // r3: second:b
4356    // r4: first:a, first:b, second:a
4357    // r4: second:b
4358    // r5: first:a
4359    // r6: first:a
4360    // r7: first:a
4361    // r8: first:a
4362    // r9: first:a
4363
4364    List<Cell> results = new ArrayList<>();
4365    int index = 0;
4366    ScannerContext scannerContext = ScannerContext.newBuilder().setBatchLimit(3).build();
4367    while (true) {
4368      boolean more = s.next(results, scannerContext);
4369      if ((index >> 1) < 5) {
4370        if (index % 2 == 0) {
4371          assertEquals(3, results.size());
4372        } else {
4373          assertEquals(1, results.size());
4374        }
4375      } else {
4376        assertEquals(1, results.size());
4377      }
4378      results.clear();
4379      index++;
4380      if (!more) {
4381        break;
4382      }
4383    }
4384  }
4385
4386  @Test
4387  public void testScannerOperationId() throws IOException {
4388    region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES);
4389    Scan scan = new Scan();
4390    RegionScanner scanner = region.getScanner(scan);
4391    assertNull(scanner.getOperationId());
4392    scanner.close();
4393
4394    String operationId = "test_operation_id_0101";
4395    scan = new Scan().setId(operationId);
4396    scanner = region.getScanner(scan);
4397    assertEquals(operationId, scanner.getOperationId());
4398    scanner.close();
4399
4400    HBaseTestingUtil.closeRegionAndWAL(this.region);
4401  }
4402
4403  /**
4404   * Write an HFile block full with Cells whose qualifier that are identical between 0 and
4405   * Short.MAX_VALUE. See HBASE-13329. n
4406   */
4407  @Test
4408  public void testLongQualifier() throws Exception {
4409    byte[] family = Bytes.toBytes("family");
4410    this.region = initHRegion(tableName, method, CONF, family);
4411    byte[] q = new byte[Short.MAX_VALUE + 2];
4412    Arrays.fill(q, 0, q.length - 1, (byte) 42);
4413    for (byte i = 0; i < 10; i++) {
4414      Put p = new Put(Bytes.toBytes("row"));
4415      // qualifiers that differ past Short.MAX_VALUE
4416      q[q.length - 1] = i;
4417      p.addColumn(family, q, q);
4418      region.put(p);
4419    }
4420    region.flush(false);
4421  }
4422
4423  /**
4424   * Flushes the cache in a thread while scanning. The tests verify that the scan is coherent - e.g.
4425   * the returned results are always of the same or later update as the previous results. n * scan /
4426   * compact n * thread join
4427   */
4428  @Test
4429  public void testFlushCacheWhileScanning() throws IOException, InterruptedException {
4430    byte[] family = Bytes.toBytes("family");
4431    int numRows = 1000;
4432    int flushAndScanInterval = 10;
4433    int compactInterval = 10 * flushAndScanInterval;
4434
4435    this.region = initHRegion(tableName, method, CONF, family);
4436    FlushThread flushThread = new FlushThread();
4437    try {
4438      flushThread.start();
4439
4440      Scan scan = new Scan();
4441      scan.addFamily(family);
4442      scan.setFilter(new SingleColumnValueFilter(family, qual1, CompareOperator.EQUAL,
4443        new BinaryComparator(Bytes.toBytes(5L))));
4444
4445      int expectedCount = 0;
4446      List<Cell> res = new ArrayList<>();
4447
4448      boolean toggle = true;
4449      for (long i = 0; i < numRows; i++) {
4450        Put put = new Put(Bytes.toBytes(i));
4451        put.setDurability(Durability.SKIP_WAL);
4452        put.addColumn(family, qual1, Bytes.toBytes(i % 10));
4453        region.put(put);
4454
4455        if (i != 0 && i % compactInterval == 0) {
4456          LOG.debug("iteration = " + i + " ts=" + EnvironmentEdgeManager.currentTime());
4457          region.compact(true);
4458        }
4459
4460        if (i % 10 == 5L) {
4461          expectedCount++;
4462        }
4463
4464        if (i != 0 && i % flushAndScanInterval == 0) {
4465          res.clear();
4466          InternalScanner scanner = region.getScanner(scan);
4467          if (toggle) {
4468            flushThread.flush();
4469          }
4470          while (scanner.next(res))
4471            ;
4472          if (!toggle) {
4473            flushThread.flush();
4474          }
4475          assertEquals(
4476            "toggle=" + toggle + "i=" + i + " ts=" + EnvironmentEdgeManager.currentTime(),
4477            expectedCount, res.size());
4478          toggle = !toggle;
4479        }
4480      }
4481
4482    } finally {
4483      try {
4484        flushThread.done();
4485        flushThread.join();
4486        flushThread.checkNoError();
4487      } catch (InterruptedException ie) {
4488        LOG.warn("Caught exception when joining with flushThread", ie);
4489      }
4490      HBaseTestingUtil.closeRegionAndWAL(this.region);
4491      this.region = null;
4492    }
4493  }
4494
4495  protected class FlushThread extends Thread {
4496    private volatile boolean done;
4497    private Throwable error = null;
4498
4499    FlushThread() {
4500      super("FlushThread");
4501    }
4502
4503    public void done() {
4504      done = true;
4505      synchronized (this) {
4506        interrupt();
4507      }
4508    }
4509
4510    public void checkNoError() {
4511      if (error != null) {
4512        assertNull(error);
4513      }
4514    }
4515
4516    @Override
4517    public void run() {
4518      done = false;
4519      while (!done) {
4520        synchronized (this) {
4521          try {
4522            wait();
4523          } catch (InterruptedException ignored) {
4524            if (done) {
4525              break;
4526            }
4527          }
4528        }
4529        try {
4530          region.flush(true);
4531        } catch (IOException e) {
4532          if (!done) {
4533            LOG.error("Error while flushing cache", e);
4534            error = e;
4535          }
4536          break;
4537        } catch (Throwable t) {
4538          LOG.error("Uncaught exception", t);
4539          throw t;
4540        }
4541      }
4542    }
4543
4544    public void flush() {
4545      synchronized (this) {
4546        notify();
4547      }
4548    }
4549  }
4550
4551  /**
4552   * So can be overridden in subclasses.
4553   */
4554  int getNumQualifiersForTestWritesWhileScanning() {
4555    return 100;
4556  }
4557
4558  /**
4559   * So can be overridden in subclasses.
4560   */
4561  int getTestCountForTestWritesWhileScanning() {
4562    return 100;
4563  }
4564
4565  /**
4566   * Writes very wide records and scans for the latest every time.. Flushes and compacts the region
4567   * every now and then to keep things realistic. n * by flush / scan / compaction n * when joining
4568   * threads
4569   */
4570  @Test
4571  public void testWritesWhileScanning() throws IOException, InterruptedException {
4572    int testCount = getTestCountForTestWritesWhileScanning();
4573    int numRows = 1;
4574    int numFamilies = 10;
4575    int numQualifiers = getNumQualifiersForTestWritesWhileScanning();
4576    int flushInterval = 7;
4577    int compactInterval = 5 * flushInterval;
4578    byte[][] families = new byte[numFamilies][];
4579    for (int i = 0; i < numFamilies; i++) {
4580      families[i] = Bytes.toBytes("family" + i);
4581    }
4582    byte[][] qualifiers = new byte[numQualifiers][];
4583    for (int i = 0; i < numQualifiers; i++) {
4584      qualifiers[i] = Bytes.toBytes("qual" + i);
4585    }
4586
4587    this.region = initHRegion(tableName, method, CONF, families);
4588    FlushThread flushThread = new FlushThread();
4589    PutThread putThread = new PutThread(numRows, families, qualifiers);
4590    try {
4591      putThread.start();
4592      putThread.waitForFirstPut();
4593
4594      flushThread.start();
4595
4596      Scan scan = new Scan().withStartRow(Bytes.toBytes("row0")).withStopRow(Bytes.toBytes("row1"));
4597
4598      int expectedCount = numFamilies * numQualifiers;
4599      List<Cell> res = new ArrayList<>();
4600
4601      long prevTimestamp = 0L;
4602      for (int i = 0; i < testCount; i++) {
4603
4604        if (i != 0 && i % compactInterval == 0) {
4605          region.compact(true);
4606          for (HStore store : region.getStores()) {
4607            store.closeAndArchiveCompactedFiles();
4608          }
4609        }
4610
4611        if (i != 0 && i % flushInterval == 0) {
4612          flushThread.flush();
4613        }
4614
4615        boolean previousEmpty = res.isEmpty();
4616        res.clear();
4617        try (InternalScanner scanner = region.getScanner(scan)) {
4618          boolean moreRows;
4619          do {
4620            moreRows = scanner.next(res);
4621          } while (moreRows);
4622        }
4623        if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
4624          assertEquals("i=" + i, expectedCount, res.size());
4625          long timestamp = res.get(0).getTimestamp();
4626          assertTrue("Timestamps were broke: " + timestamp + " prev: " + prevTimestamp,
4627            timestamp >= prevTimestamp);
4628          prevTimestamp = timestamp;
4629        }
4630      }
4631
4632      putThread.done();
4633
4634      region.flush(true);
4635
4636    } finally {
4637      try {
4638        flushThread.done();
4639        flushThread.join();
4640        flushThread.checkNoError();
4641
4642        putThread.join();
4643        putThread.checkNoError();
4644      } catch (InterruptedException ie) {
4645        LOG.warn("Caught exception when joining with flushThread", ie);
4646      }
4647
4648      try {
4649        HBaseTestingUtil.closeRegionAndWAL(this.region);
4650      } catch (DroppedSnapshotException dse) {
4651        // We could get this on way out because we interrupt the background flusher and it could
4652        // fail anywhere causing a DSE over in the background flusher... only it is not properly
4653        // dealt with so could still be memory hanging out when we get to here -- memory we can't
4654        // flush because the accounting is 'off' since original DSE.
4655      }
4656      this.region = null;
4657    }
4658  }
4659
4660  protected class PutThread extends Thread {
4661    private volatile boolean done;
4662    private volatile int numPutsFinished = 0;
4663
4664    private Throwable error = null;
4665    private int numRows;
4666    private byte[][] families;
4667    private byte[][] qualifiers;
4668
4669    private PutThread(int numRows, byte[][] families, byte[][] qualifiers) {
4670      super("PutThread");
4671      this.numRows = numRows;
4672      this.families = families;
4673      this.qualifiers = qualifiers;
4674    }
4675
4676    /**
4677     * Block calling thread until this instance of PutThread has put at least one row.
4678     */
4679    public void waitForFirstPut() throws InterruptedException {
4680      // wait until put thread actually puts some data
4681      while (isAlive() && numPutsFinished == 0) {
4682        checkNoError();
4683        Thread.sleep(50);
4684      }
4685    }
4686
4687    public void done() {
4688      done = true;
4689      synchronized (this) {
4690        interrupt();
4691      }
4692    }
4693
4694    public void checkNoError() {
4695      if (error != null) {
4696        assertNull(error);
4697      }
4698    }
4699
4700    @Override
4701    public void run() {
4702      done = false;
4703      while (!done) {
4704        try {
4705          for (int r = 0; r < numRows; r++) {
4706            byte[] row = Bytes.toBytes("row" + r);
4707            Put put = new Put(row);
4708            put.setDurability(Durability.SKIP_WAL);
4709            byte[] value = Bytes.toBytes(String.valueOf(numPutsFinished));
4710            for (byte[] family : families) {
4711              for (byte[] qualifier : qualifiers) {
4712                put.addColumn(family, qualifier, numPutsFinished, value);
4713              }
4714            }
4715            region.put(put);
4716            numPutsFinished++;
4717            if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
4718              LOG.debug("put iteration = {}", numPutsFinished);
4719              Delete delete = new Delete(row, (long) numPutsFinished - 30);
4720              region.delete(delete);
4721            }
4722            numPutsFinished++;
4723          }
4724        } catch (InterruptedIOException e) {
4725          // This is fine. It means we are done, or didn't get the lock on time
4726          LOG.info("Interrupted", e);
4727        } catch (IOException e) {
4728          LOG.error("Error while putting records", e);
4729          error = e;
4730          break;
4731        }
4732      }
4733
4734    }
4735
4736  }
4737
4738  /**
4739   * Writes very wide records and gets the latest row every time.. Flushes and compacts the region
4740   * aggressivly to catch issues. n * by flush / scan / compaction n * when joining threads
4741   */
4742  @Test
4743  public void testWritesWhileGetting() throws Exception {
4744    int testCount = 50;
4745    int numRows = 1;
4746    int numFamilies = 10;
4747    int numQualifiers = 100;
4748    int compactInterval = 100;
4749    byte[][] families = new byte[numFamilies][];
4750    for (int i = 0; i < numFamilies; i++) {
4751      families[i] = Bytes.toBytes("family" + i);
4752    }
4753    byte[][] qualifiers = new byte[numQualifiers][];
4754    for (int i = 0; i < numQualifiers; i++) {
4755      qualifiers[i] = Bytes.toBytes("qual" + i);
4756    }
4757
4758    // This test flushes constantly and can cause many files to be created,
4759    // possibly
4760    // extending over the ulimit. Make sure compactions are aggressive in
4761    // reducing
4762    // the number of HFiles created.
4763    Configuration conf = HBaseConfiguration.create(CONF);
4764    conf.setInt("hbase.hstore.compaction.min", 1);
4765    conf.setInt("hbase.hstore.compaction.max", 1000);
4766    this.region = initHRegion(tableName, method, conf, families);
4767    PutThread putThread = null;
4768    MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
4769    try {
4770      putThread = new PutThread(numRows, families, qualifiers);
4771      putThread.start();
4772      putThread.waitForFirstPut();
4773
4774      // Add a thread that flushes as fast as possible
4775      ctx.addThread(new RepeatingTestThread(ctx) {
4776
4777        @Override
4778        public void doAnAction() throws Exception {
4779          region.flush(true);
4780          // Compact regularly to avoid creating too many files and exceeding
4781          // the ulimit.
4782          region.compact(false);
4783          for (HStore store : region.getStores()) {
4784            store.closeAndArchiveCompactedFiles();
4785          }
4786        }
4787      });
4788      ctx.startThreads();
4789
4790      Get get = new Get(Bytes.toBytes("row0"));
4791      Result result = null;
4792
4793      int expectedCount = numFamilies * numQualifiers;
4794
4795      long prevTimestamp = 0L;
4796      for (int i = 0; i < testCount; i++) {
4797        LOG.info("testWritesWhileGetting verify turn " + i);
4798        boolean previousEmpty = result == null || result.isEmpty();
4799        result = region.get(get);
4800        if (!result.isEmpty() || !previousEmpty || i > compactInterval) {
4801          assertEquals("i=" + i, expectedCount, result.size());
4802          // TODO this was removed, now what dangit?!
4803          // search looking for the qualifier in question?
4804          long timestamp = 0;
4805          for (Cell kv : result.rawCells()) {
4806            if (
4807              CellUtil.matchingFamily(kv, families[0])
4808                && CellUtil.matchingQualifier(kv, qualifiers[0])
4809            ) {
4810              timestamp = kv.getTimestamp();
4811            }
4812          }
4813          assertTrue(timestamp >= prevTimestamp);
4814          prevTimestamp = timestamp;
4815          Cell previousKV = null;
4816
4817          for (Cell kv : result.rawCells()) {
4818            byte[] thisValue = CellUtil.cloneValue(kv);
4819            if (previousKV != null) {
4820              if (Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue) != 0) {
4821                LOG.warn("These two KV should have the same value." + " Previous KV:" + previousKV
4822                  + "(memStoreTS:" + previousKV.getSequenceId() + ")" + ", New KV: " + kv
4823                  + "(memStoreTS:" + kv.getSequenceId() + ")");
4824                assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(previousKV), thisValue));
4825              }
4826            }
4827            previousKV = kv;
4828          }
4829        }
4830      }
4831    } finally {
4832      if (putThread != null) putThread.done();
4833
4834      region.flush(true);
4835
4836      if (putThread != null) {
4837        putThread.join();
4838        putThread.checkNoError();
4839      }
4840
4841      ctx.stop();
4842      HBaseTestingUtil.closeRegionAndWAL(this.region);
4843      this.region = null;
4844    }
4845  }
4846
4847  @Test
4848  public void testHolesInMeta() throws Exception {
4849    byte[] family = Bytes.toBytes("family");
4850    this.region =
4851      initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method, CONF, false, family);
4852    byte[] rowNotServed = Bytes.toBytes("a");
4853    Get g = new Get(rowNotServed);
4854    try {
4855      region.get(g);
4856      fail();
4857    } catch (WrongRegionException x) {
4858      // OK
4859    }
4860    byte[] row = Bytes.toBytes("y");
4861    g = new Get(row);
4862    region.get(g);
4863  }
4864
4865  @Test
4866  public void testIndexesScanWithOneDeletedRow() throws IOException {
4867    byte[] family = Bytes.toBytes("family");
4868
4869    // Setting up region
4870    this.region = initHRegion(tableName, method, CONF, family);
4871    Put put = new Put(Bytes.toBytes(1L));
4872    put.addColumn(family, qual1, 1L, Bytes.toBytes(1L));
4873    region.put(put);
4874
4875    region.flush(true);
4876
4877    Delete delete = new Delete(Bytes.toBytes(1L), 1L);
4878    region.delete(delete);
4879
4880    put = new Put(Bytes.toBytes(2L));
4881    put.addColumn(family, qual1, 2L, Bytes.toBytes(2L));
4882    region.put(put);
4883
4884    Scan idxScan = new Scan();
4885    idxScan.addFamily(family);
4886    idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,
4887      Arrays.<Filter> asList(
4888        new SingleColumnValueFilter(family, qual1, CompareOperator.GREATER_OR_EQUAL,
4889          new BinaryComparator(Bytes.toBytes(0L))),
4890        new SingleColumnValueFilter(family, qual1, CompareOperator.LESS_OR_EQUAL,
4891          new BinaryComparator(Bytes.toBytes(3L))))));
4892    InternalScanner scanner = region.getScanner(idxScan);
4893    List<Cell> res = new ArrayList<>();
4894
4895    while (scanner.next(res)) {
4896      // Ignore res value.
4897    }
4898    assertEquals(1L, res.size());
4899  }
4900
4901  // ////////////////////////////////////////////////////////////////////////////
4902  // Bloom filter test
4903  // ////////////////////////////////////////////////////////////////////////////
4904  @Test
4905  public void testBloomFilterSize() throws IOException {
4906    byte[] fam1 = Bytes.toBytes("fam1");
4907    byte[] qf1 = Bytes.toBytes("col");
4908    byte[] val1 = Bytes.toBytes("value1");
4909    // Create Table
4910    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
4911      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam1)
4912        .setMaxVersions(Integer.MAX_VALUE).setBloomFilterType(BloomType.ROWCOL).build())
4913      .build();
4914    RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
4915    this.region = TEST_UTIL.createLocalHRegion(info, tableDescriptor);
4916    int num_unique_rows = 10;
4917    int duplicate_multiplier = 2;
4918    int num_storefiles = 4;
4919
4920    int version = 0;
4921    for (int f = 0; f < num_storefiles; f++) {
4922      for (int i = 0; i < duplicate_multiplier; i++) {
4923        for (int j = 0; j < num_unique_rows; j++) {
4924          Put put = new Put(Bytes.toBytes("row" + j));
4925          put.setDurability(Durability.SKIP_WAL);
4926          long ts = version++;
4927          put.addColumn(fam1, qf1, ts, val1);
4928          region.put(put);
4929        }
4930      }
4931      region.flush(true);
4932    }
4933    // before compaction
4934    HStore store = region.getStore(fam1);
4935    Collection<HStoreFile> storeFiles = store.getStorefiles();
4936    for (HStoreFile storefile : storeFiles) {
4937      StoreFileReader reader = storefile.getReader();
4938      reader.loadFileInfo();
4939      reader.loadBloomfilter();
4940      assertEquals(num_unique_rows * duplicate_multiplier, reader.getEntries());
4941      assertEquals(num_unique_rows, reader.getFilterEntries());
4942    }
4943
4944    region.compact(true);
4945
4946    // after compaction
4947    storeFiles = store.getStorefiles();
4948    for (HStoreFile storefile : storeFiles) {
4949      StoreFileReader reader = storefile.getReader();
4950      reader.loadFileInfo();
4951      reader.loadBloomfilter();
4952      assertEquals(num_unique_rows * duplicate_multiplier * num_storefiles, reader.getEntries());
4953      assertEquals(num_unique_rows, reader.getFilterEntries());
4954    }
4955  }
4956
4957  @Test
4958  public void testAllColumnsWithBloomFilter() throws IOException {
4959    byte[] TABLE = Bytes.toBytes(name.getMethodName());
4960    byte[] FAMILY = Bytes.toBytes("family");
4961
4962    // Create table
4963    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(TableName.valueOf(TABLE))
4964      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY)
4965        .setMaxVersions(Integer.MAX_VALUE).setBloomFilterType(BloomType.ROWCOL).build())
4966      .build();
4967    RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
4968    this.region = TEST_UTIL.createLocalHRegion(info, tableDescriptor);
4969    // For row:0, col:0: insert versions 1 through 5.
4970    byte[] row = Bytes.toBytes("row:" + 0);
4971    byte[] column = Bytes.toBytes("column:" + 0);
4972    Put put = new Put(row);
4973    put.setDurability(Durability.SKIP_WAL);
4974    for (long idx = 1; idx <= 4; idx++) {
4975      put.addColumn(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx));
4976    }
4977    region.put(put);
4978
4979    // Flush
4980    region.flush(true);
4981
4982    // Get rows
4983    Get get = new Get(row);
4984    get.readAllVersions();
4985    Cell[] kvs = region.get(get).rawCells();
4986
4987    // Check if rows are correct
4988    assertEquals(4, kvs.length);
4989    checkOneCell(kvs[0], FAMILY, 0, 0, 4);
4990    checkOneCell(kvs[1], FAMILY, 0, 0, 3);
4991    checkOneCell(kvs[2], FAMILY, 0, 0, 2);
4992    checkOneCell(kvs[3], FAMILY, 0, 0, 1);
4993  }
4994
4995  /**
4996   * Testcase to cover bug-fix for HBASE-2823 Ensures correct delete when issuing delete row on
4997   * columns with bloom filter set to row+col (BloomType.ROWCOL)
4998   */
4999  @Test
5000  public void testDeleteRowWithBloomFilter() throws IOException {
5001    byte[] familyName = Bytes.toBytes("familyName");
5002
5003    // Create Table
5004    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
5005      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(familyName)
5006        .setMaxVersions(Integer.MAX_VALUE).setBloomFilterType(BloomType.ROWCOL).build())
5007      .build();
5008    RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
5009    this.region = TEST_UTIL.createLocalHRegion(info, tableDescriptor);
5010    // Insert some data
5011    byte[] row = Bytes.toBytes("row1");
5012    byte[] col = Bytes.toBytes("col1");
5013
5014    Put put = new Put(row);
5015    put.addColumn(familyName, col, 1, Bytes.toBytes("SomeRandomValue"));
5016    region.put(put);
5017    region.flush(true);
5018
5019    Delete del = new Delete(row);
5020    region.delete(del);
5021    region.flush(true);
5022
5023    // Get remaining rows (should have none)
5024    Get get = new Get(row);
5025    get.addColumn(familyName, col);
5026
5027    Cell[] keyValues = region.get(get).rawCells();
5028    assertEquals(0, keyValues.length);
5029  }
5030
5031  @Test
5032  public void testgetHDFSBlocksDistribution() throws Exception {
5033    HBaseTestingUtil htu = new HBaseTestingUtil();
5034    // Why do we set the block size in this test? If we set it smaller than the kvs, then we'll
5035    // break up the file in to more pieces that can be distributed across the three nodes and we
5036    // won't be able to have the condition this test asserts; that at least one node has
5037    // a copy of all replicas -- if small block size, then blocks are spread evenly across the
5038    // the three nodes. hfilev3 with tags seems to put us over the block size. St.Ack.
5039    // final int DEFAULT_BLOCK_SIZE = 1024;
5040    // htu.getConfiguration().setLong("dfs.blocksize", DEFAULT_BLOCK_SIZE);
5041    htu.getConfiguration().setInt("dfs.replication", 2);
5042
5043    // set up a cluster with 3 nodes
5044    SingleProcessHBaseCluster cluster = null;
5045    String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
5046    int regionServersCount = 3;
5047
5048    try {
5049      StartTestingClusterOption option = StartTestingClusterOption.builder()
5050        .numRegionServers(regionServersCount).dataNodeHosts(dataNodeHosts).build();
5051      cluster = htu.startMiniCluster(option);
5052      byte[][] families = { fam1, fam2 };
5053      Table ht = htu.createTable(tableName, families);
5054
5055      // Setting up region
5056      byte row[] = Bytes.toBytes("row1");
5057      byte col[] = Bytes.toBytes("col1");
5058
5059      Put put = new Put(row);
5060      put.addColumn(fam1, col, 1, Bytes.toBytes("test1"));
5061      put.addColumn(fam2, col, 1, Bytes.toBytes("test2"));
5062      ht.put(put);
5063
5064      HRegion firstRegion = htu.getHBaseCluster().getRegions(tableName).get(0);
5065      firstRegion.flush(true);
5066      HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution();
5067
5068      // Given the default replication factor is 2 and we have 2 HFiles,
5069      // we will have total of 4 replica of blocks on 3 datanodes; thus there
5070      // must be at least one host that have replica for 2 HFiles. That host's
5071      // weight will be equal to the unique block weight.
5072      long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight();
5073      StringBuilder sb = new StringBuilder();
5074      for (String host : blocksDistribution1.getTopHosts()) {
5075        if (sb.length() > 0) sb.append(", ");
5076        sb.append(host);
5077        sb.append("=");
5078        sb.append(blocksDistribution1.getWeight(host));
5079      }
5080
5081      String topHost = blocksDistribution1.getTopHosts().get(0);
5082      long topHostWeight = blocksDistribution1.getWeight(topHost);
5083      String msg = "uniqueBlocksWeight=" + uniqueBlocksWeight1 + ", topHostWeight=" + topHostWeight
5084        + ", topHost=" + topHost + "; " + sb.toString();
5085      LOG.info(msg);
5086      assertTrue(msg, uniqueBlocksWeight1 == topHostWeight);
5087
5088      // use the static method to compute the value, it should be the same.
5089      // static method is used by load balancer or other components
5090      HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution(
5091        htu.getConfiguration(), firstRegion.getTableDescriptor(), firstRegion.getRegionInfo());
5092      long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight();
5093
5094      assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2);
5095
5096      ht.close();
5097    } finally {
5098      if (cluster != null) {
5099        htu.shutdownMiniCluster();
5100      }
5101    }
5102  }
5103
5104  /**
5105   * Testcase to check state of region initialization task set to ABORTED or not if any exceptions
5106   * during initialization n
5107   */
5108  @Test
5109  public void testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization() throws Exception {
5110    RegionInfo info;
5111    try {
5112      FileSystem fs = Mockito.mock(FileSystem.class);
5113      Mockito.when(fs.exists((Path) Mockito.anyObject())).thenThrow(new IOException());
5114      TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(tableName);
5115      ColumnFamilyDescriptor columnFamilyDescriptor =
5116        ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf")).build();
5117      tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
5118      info = RegionInfoBuilder.newBuilder(tableName).build();
5119      Path path = new Path(dir + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization");
5120      region = HRegion.newHRegion(path, null, fs, CONF, info, tableDescriptorBuilder.build(), null);
5121      // region initialization throws IOException and set task state to ABORTED.
5122      region.initialize();
5123      fail("Region initialization should fail due to IOException");
5124    } catch (IOException io) {
5125      List<MonitoredTask> tasks = TaskMonitor.get().getTasks();
5126      for (MonitoredTask monitoredTask : tasks) {
5127        if (
5128          !(monitoredTask instanceof MonitoredRPCHandler)
5129            && monitoredTask.getDescription().contains(region.toString())
5130        ) {
5131          assertTrue("Region state should be ABORTED.",
5132            monitoredTask.getState().equals(MonitoredTask.State.ABORTED));
5133          break;
5134        }
5135      }
5136    }
5137  }
5138
5139  /**
5140   * Verifies that the .regioninfo file is written on region creation and that is recreated if
5141   * missing during region opening.
5142   */
5143  @Test
5144  public void testRegionInfoFileCreation() throws IOException {
5145    Path rootDir = new Path(dir + "testRegionInfoFileCreation");
5146
5147    TableDescriptorBuilder tableDescriptorBuilder =
5148      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()));
5149    ColumnFamilyDescriptor columnFamilyDescriptor =
5150      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf")).build();
5151    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
5152    TableDescriptor tableDescriptor = tableDescriptorBuilder.build();
5153
5154    RegionInfo hri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
5155
5156    // Create a region and skip the initialization (like CreateTableHandler)
5157    region = HBaseTestingUtil.createRegionAndWAL(hri, rootDir, CONF, tableDescriptor, false);
5158    Path regionDir = region.getRegionFileSystem().getRegionDir();
5159    FileSystem fs = region.getRegionFileSystem().getFileSystem();
5160    HBaseTestingUtil.closeRegionAndWAL(region);
5161
5162    Path regionInfoFile = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
5163
5164    // Verify that the .regioninfo file is present
5165    assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
5166      fs.exists(regionInfoFile));
5167
5168    // Try to open the region
5169    region = HRegion.openHRegion(rootDir, hri, tableDescriptor, null, CONF);
5170    assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
5171    HBaseTestingUtil.closeRegionAndWAL(region);
5172
5173    // Verify that the .regioninfo file is still there
5174    assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
5175      fs.exists(regionInfoFile));
5176
5177    // Remove the .regioninfo file and verify is recreated on region open
5178    fs.delete(regionInfoFile, true);
5179    assertFalse(HRegionFileSystem.REGION_INFO_FILE + " should be removed from the region dir",
5180      fs.exists(regionInfoFile));
5181
5182    region = HRegion.openHRegion(rootDir, hri, tableDescriptor, null, CONF);
5183    // region = TEST_UTIL.openHRegion(hri, htd);
5184    assertEquals(regionDir, region.getRegionFileSystem().getRegionDir());
5185    HBaseTestingUtil.closeRegionAndWAL(region);
5186
5187    // Verify that the .regioninfo file is still there
5188    assertTrue(HRegionFileSystem.REGION_INFO_FILE + " should be present in the region dir",
5189      fs.exists(new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE)));
5190
5191    region = null;
5192  }
5193
5194  /**
5195   * TestCase for increment
5196   */
5197  private static class Incrementer implements Runnable {
5198    private HRegion region;
5199    private final static byte[] incRow = Bytes.toBytes("incRow");
5200    private final static byte[] family = Bytes.toBytes("family");
5201    private final static byte[] qualifier = Bytes.toBytes("qualifier");
5202    private final static long ONE = 1L;
5203    private int incCounter;
5204
5205    public Incrementer(HRegion region, int incCounter) {
5206      this.region = region;
5207      this.incCounter = incCounter;
5208    }
5209
5210    @Override
5211    public void run() {
5212      int count = 0;
5213      while (count < incCounter) {
5214        Increment inc = new Increment(incRow);
5215        inc.addColumn(family, qualifier, ONE);
5216        count++;
5217        try {
5218          region.increment(inc);
5219        } catch (IOException e) {
5220          LOG.info("Count=" + count + ", " + e);
5221          break;
5222        }
5223      }
5224    }
5225  }
5226
5227  /**
5228   * Test case to check increment function with memstore flushing n
5229   */
5230  @Test
5231  public void testParallelIncrementWithMemStoreFlush() throws Exception {
5232    byte[] family = Incrementer.family;
5233    this.region = initHRegion(tableName, method, CONF, family);
5234    final HRegion region = this.region;
5235    final AtomicBoolean incrementDone = new AtomicBoolean(false);
5236    Runnable flusher = new Runnable() {
5237      @Override
5238      public void run() {
5239        while (!incrementDone.get()) {
5240          try {
5241            region.flush(true);
5242          } catch (Exception e) {
5243            e.printStackTrace();
5244          }
5245        }
5246      }
5247    };
5248
5249    // after all increment finished, the row will increment to 20*100 = 2000
5250    int threadNum = 20;
5251    int incCounter = 100;
5252    long expected = (long) threadNum * incCounter;
5253    Thread[] incrementers = new Thread[threadNum];
5254    Thread flushThread = new Thread(flusher);
5255    for (int i = 0; i < threadNum; i++) {
5256      incrementers[i] = new Thread(new Incrementer(this.region, incCounter));
5257      incrementers[i].start();
5258    }
5259    flushThread.start();
5260    for (int i = 0; i < threadNum; i++) {
5261      incrementers[i].join();
5262    }
5263
5264    incrementDone.set(true);
5265    flushThread.join();
5266
5267    Get get = new Get(Incrementer.incRow);
5268    get.addColumn(Incrementer.family, Incrementer.qualifier);
5269    get.readVersions(1);
5270    Result res = this.region.get(get);
5271    List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
5272
5273    // we just got the latest version
5274    assertEquals(1, kvs.size());
5275    Cell kv = kvs.get(0);
5276    assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
5277  }
5278
5279  /**
5280   * TestCase for append
5281   */
5282  private static class Appender implements Runnable {
5283    private HRegion region;
5284    private final static byte[] appendRow = Bytes.toBytes("appendRow");
5285    private final static byte[] family = Bytes.toBytes("family");
5286    private final static byte[] qualifier = Bytes.toBytes("qualifier");
5287    private final static byte[] CHAR = Bytes.toBytes("a");
5288    private int appendCounter;
5289
5290    public Appender(HRegion region, int appendCounter) {
5291      this.region = region;
5292      this.appendCounter = appendCounter;
5293    }
5294
5295    @Override
5296    public void run() {
5297      int count = 0;
5298      while (count < appendCounter) {
5299        Append app = new Append(appendRow);
5300        app.addColumn(family, qualifier, CHAR);
5301        count++;
5302        try {
5303          region.append(app);
5304        } catch (IOException e) {
5305          LOG.info("Count=" + count + ", max=" + appendCounter + ", " + e);
5306          break;
5307        }
5308      }
5309    }
5310  }
5311
5312  /**
5313   * Test case to check append function with memstore flushing n
5314   */
5315  @Test
5316  public void testParallelAppendWithMemStoreFlush() throws Exception {
5317    byte[] family = Appender.family;
5318    this.region = initHRegion(tableName, method, CONF, family);
5319    final HRegion region = this.region;
5320    final AtomicBoolean appendDone = new AtomicBoolean(false);
5321    Runnable flusher = new Runnable() {
5322      @Override
5323      public void run() {
5324        while (!appendDone.get()) {
5325          try {
5326            region.flush(true);
5327          } catch (Exception e) {
5328            e.printStackTrace();
5329          }
5330        }
5331      }
5332    };
5333
5334    // After all append finished, the value will append to threadNum *
5335    // appendCounter Appender.CHAR
5336    int threadNum = 20;
5337    int appendCounter = 100;
5338    byte[] expected = new byte[threadNum * appendCounter];
5339    for (int i = 0; i < threadNum * appendCounter; i++) {
5340      System.arraycopy(Appender.CHAR, 0, expected, i, 1);
5341    }
5342    Thread[] appenders = new Thread[threadNum];
5343    Thread flushThread = new Thread(flusher);
5344    for (int i = 0; i < threadNum; i++) {
5345      appenders[i] = new Thread(new Appender(this.region, appendCounter));
5346      appenders[i].start();
5347    }
5348    flushThread.start();
5349    for (int i = 0; i < threadNum; i++) {
5350      appenders[i].join();
5351    }
5352
5353    appendDone.set(true);
5354    flushThread.join();
5355
5356    Get get = new Get(Appender.appendRow);
5357    get.addColumn(Appender.family, Appender.qualifier);
5358    get.readVersions(1);
5359    Result res = this.region.get(get);
5360    List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
5361
5362    // we just got the latest version
5363    assertEquals(1, kvs.size());
5364    Cell kv = kvs.get(0);
5365    byte[] appendResult = new byte[kv.getValueLength()];
5366    System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
5367    assertArrayEquals(expected, appendResult);
5368  }
5369
5370  /**
5371   * Test case to check put function with memstore flushing for same row, same ts n
5372   */
5373  @Test
5374  public void testPutWithMemStoreFlush() throws Exception {
5375    byte[] family = Bytes.toBytes("family");
5376    byte[] qualifier = Bytes.toBytes("qualifier");
5377    byte[] row = Bytes.toBytes("putRow");
5378    byte[] value = null;
5379    this.region = initHRegion(tableName, method, CONF, family);
5380    Put put = null;
5381    Get get = null;
5382    List<Cell> kvs = null;
5383    Result res = null;
5384
5385    put = new Put(row);
5386    value = Bytes.toBytes("value0");
5387    put.addColumn(family, qualifier, 1234567L, value);
5388    region.put(put);
5389    get = new Get(row);
5390    get.addColumn(family, qualifier);
5391    get.readAllVersions();
5392    res = this.region.get(get);
5393    kvs = res.getColumnCells(family, qualifier);
5394    assertEquals(1, kvs.size());
5395    assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
5396
5397    region.flush(true);
5398    get = new Get(row);
5399    get.addColumn(family, qualifier);
5400    get.readAllVersions();
5401    res = this.region.get(get);
5402    kvs = res.getColumnCells(family, qualifier);
5403    assertEquals(1, kvs.size());
5404    assertArrayEquals(Bytes.toBytes("value0"), CellUtil.cloneValue(kvs.get(0)));
5405
5406    put = new Put(row);
5407    value = Bytes.toBytes("value1");
5408    put.addColumn(family, qualifier, 1234567L, value);
5409    region.put(put);
5410    get = new Get(row);
5411    get.addColumn(family, qualifier);
5412    get.readAllVersions();
5413    res = this.region.get(get);
5414    kvs = res.getColumnCells(family, qualifier);
5415    assertEquals(1, kvs.size());
5416    assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
5417
5418    region.flush(true);
5419    get = new Get(row);
5420    get.addColumn(family, qualifier);
5421    get.readAllVersions();
5422    res = this.region.get(get);
5423    kvs = res.getColumnCells(family, qualifier);
5424    assertEquals(1, kvs.size());
5425    assertArrayEquals(Bytes.toBytes("value1"), CellUtil.cloneValue(kvs.get(0)));
5426  }
5427
5428  @Test
5429  public void testDurability() throws Exception {
5430    // there are 5 x 5 cases:
5431    // table durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT) x mutation
5432    // durability(SYNC,FSYNC,ASYC,SKIP,USE_DEFAULT)
5433
5434    // expected cases for append and sync wal
5435    durabilityTest(method, Durability.SYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
5436    durabilityTest(method, Durability.SYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
5437    durabilityTest(method, Durability.SYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
5438
5439    durabilityTest(method, Durability.FSYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
5440    durabilityTest(method, Durability.FSYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
5441    durabilityTest(method, Durability.FSYNC_WAL, Durability.USE_DEFAULT, 0, true, true, false);
5442
5443    durabilityTest(method, Durability.ASYNC_WAL, Durability.SYNC_WAL, 0, true, true, false);
5444    durabilityTest(method, Durability.ASYNC_WAL, Durability.FSYNC_WAL, 0, true, true, false);
5445
5446    durabilityTest(method, Durability.SKIP_WAL, Durability.SYNC_WAL, 0, true, true, false);
5447    durabilityTest(method, Durability.SKIP_WAL, Durability.FSYNC_WAL, 0, true, true, false);
5448
5449    durabilityTest(method, Durability.USE_DEFAULT, Durability.SYNC_WAL, 0, true, true, false);
5450    durabilityTest(method, Durability.USE_DEFAULT, Durability.FSYNC_WAL, 0, true, true, false);
5451    durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false);
5452
5453    // expected cases for async wal
5454    durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
5455    durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
5456    durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false);
5457    durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 0, true, false, false);
5458    durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false);
5459    durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false);
5460
5461    durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
5462    durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
5463    durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
5464    durabilityTest(method, Durability.SKIP_WAL, Durability.ASYNC_WAL, 5000, true, false, true);
5465    durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 5000, true, false, true);
5466    durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 5000, true, false, true);
5467
5468    // expect skip wal cases
5469    durabilityTest(method, Durability.SYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
5470    durabilityTest(method, Durability.FSYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
5471    durabilityTest(method, Durability.ASYNC_WAL, Durability.SKIP_WAL, 0, false, false, false);
5472    durabilityTest(method, Durability.SKIP_WAL, Durability.SKIP_WAL, 0, false, false, false);
5473    durabilityTest(method, Durability.USE_DEFAULT, Durability.SKIP_WAL, 0, false, false, false);
5474    durabilityTest(method, Durability.SKIP_WAL, Durability.USE_DEFAULT, 0, false, false, false);
5475
5476  }
5477
5478  private void durabilityTest(String method, Durability tableDurability,
5479    Durability mutationDurability, long timeout, boolean expectAppend, final boolean expectSync,
5480    final boolean expectSyncFromLogSyncer) throws Exception {
5481    Configuration conf = HBaseConfiguration.create(CONF);
5482    method = method + "_" + tableDurability.name() + "_" + mutationDurability.name();
5483    byte[] family = Bytes.toBytes("family");
5484    Path logDir = new Path(new Path(dir + method), "log");
5485    final Configuration walConf = new Configuration(conf);
5486    CommonFSUtils.setRootDir(walConf, logDir);
5487    // XXX: The spied AsyncFSWAL can not work properly because of a Mockito defect that can not
5488    // deal with classes which have a field of an inner class. See discussions in HBASE-15536.
5489    walConf.set(WALFactory.WAL_PROVIDER, "filesystem");
5490    final WALFactory wals = new WALFactory(walConf, HBaseTestingUtil.getRandomUUID().toString());
5491    final WAL wal = spy(wals.getWAL(RegionInfoBuilder.newBuilder(tableName).build()));
5492    this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, CONF,
5493      false, tableDurability, wal, new byte[][] { family });
5494
5495    Put put = new Put(Bytes.toBytes("r1"));
5496    put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
5497    put.setDurability(mutationDurability);
5498    region.put(put);
5499
5500    // verify append called or not
5501    verify(wal, expectAppend ? times(1) : never()).appendData((RegionInfo) any(),
5502      (WALKeyImpl) any(), (WALEdit) any());
5503
5504    // verify sync called or not
5505    if (expectSync || expectSyncFromLogSyncer) {
5506      TEST_UTIL.waitFor(timeout, new Waiter.Predicate<Exception>() {
5507        @Override
5508        public boolean evaluate() throws Exception {
5509          try {
5510            if (expectSync) {
5511              verify(wal, times(1)).sync(anyLong()); // Hregion calls this one
5512            } else if (expectSyncFromLogSyncer) {
5513              verify(wal, times(1)).sync(); // wal syncer calls this one
5514            }
5515          } catch (Throwable ignore) {
5516          }
5517          return true;
5518        }
5519      });
5520    } else {
5521      // verify(wal, never()).sync(anyLong());
5522      verify(wal, never()).sync();
5523    }
5524
5525    HBaseTestingUtil.closeRegionAndWAL(this.region);
5526    wals.close();
5527    this.region = null;
5528  }
5529
5530  @Test
5531  public void testRegionReplicaSecondary() throws IOException {
5532    // create a primary region, load some data and flush
5533    // create a secondary region, and do a get against that
5534    Path rootDir = new Path(dir + name.getMethodName());
5535    CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
5536
5537    byte[][] families =
5538      new byte[][] { Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3") };
5539    byte[] cq = Bytes.toBytes("cq");
5540    TableDescriptorBuilder builder =
5541      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()));
5542    for (byte[] family : families) {
5543      builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family));
5544    }
5545    TableDescriptor tableDescriptor = builder.build();
5546    long time = EnvironmentEdgeManager.currentTime();
5547    RegionInfo primaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName())
5548      .setRegionId(time).setReplicaId(0).build();
5549    RegionInfo secondaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName())
5550      .setRegionId(time).setReplicaId(1).build();
5551
5552    HRegion primaryRegion = null, secondaryRegion = null;
5553
5554    try {
5555      primaryRegion = HBaseTestingUtil.createRegionAndWAL(primaryHri, rootDir,
5556        TEST_UTIL.getConfiguration(), tableDescriptor);
5557
5558      // load some data
5559      putData(primaryRegion, 0, 1000, cq, families);
5560
5561      // flush region
5562      primaryRegion.flush(true);
5563
5564      // open secondary region
5565      secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, tableDescriptor, null, CONF);
5566
5567      verifyData(secondaryRegion, 0, 1000, cq, families);
5568    } finally {
5569      if (primaryRegion != null) {
5570        HBaseTestingUtil.closeRegionAndWAL(primaryRegion);
5571      }
5572      if (secondaryRegion != null) {
5573        HBaseTestingUtil.closeRegionAndWAL(secondaryRegion);
5574      }
5575    }
5576  }
5577
5578  @Test
5579  public void testRegionReplicaSecondaryIsReadOnly() throws IOException {
5580    // create a primary region, load some data and flush
5581    // create a secondary region, and do a put against that
5582    Path rootDir = new Path(dir + name.getMethodName());
5583    CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
5584
5585    byte[][] families =
5586      new byte[][] { Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3") };
5587    byte[] cq = Bytes.toBytes("cq");
5588    TableDescriptorBuilder builder =
5589      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()));
5590    for (byte[] family : families) {
5591      builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family));
5592    }
5593    TableDescriptor tableDescriptor = builder.build();
5594    long time = EnvironmentEdgeManager.currentTime();
5595    RegionInfo primaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName())
5596      .setRegionId(time).setReplicaId(0).build();
5597    RegionInfo secondaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName())
5598      .setRegionId(time).setReplicaId(1).build();
5599
5600    HRegion primaryRegion = null, secondaryRegion = null;
5601
5602    try {
5603      primaryRegion = HBaseTestingUtil.createRegionAndWAL(primaryHri, rootDir,
5604        TEST_UTIL.getConfiguration(), tableDescriptor);
5605
5606      // load some data
5607      putData(primaryRegion, 0, 1000, cq, families);
5608
5609      // flush region
5610      primaryRegion.flush(true);
5611
5612      // open secondary region
5613      secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, tableDescriptor, null, CONF);
5614
5615      try {
5616        putData(secondaryRegion, 0, 1000, cq, families);
5617        fail("Should have thrown exception");
5618      } catch (IOException ex) {
5619        // expected
5620      }
5621    } finally {
5622      if (primaryRegion != null) {
5623        HBaseTestingUtil.closeRegionAndWAL(primaryRegion);
5624      }
5625      if (secondaryRegion != null) {
5626        HBaseTestingUtil.closeRegionAndWAL(secondaryRegion);
5627      }
5628    }
5629  }
5630
5631  static WALFactory createWALFactory(Configuration conf, Path rootDir) throws IOException {
5632    Configuration confForWAL = new Configuration(conf);
5633    confForWAL.set(HConstants.HBASE_DIR, rootDir.toString());
5634    return new WALFactory(confForWAL, "hregion-" + RandomStringUtils.randomNumeric(8));
5635  }
5636
5637  @Test
5638  public void testCompactionFromPrimary() throws IOException {
5639    Path rootDir = new Path(dir + name.getMethodName());
5640    CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
5641
5642    byte[][] families =
5643      new byte[][] { Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3") };
5644    byte[] cq = Bytes.toBytes("cq");
5645    TableDescriptorBuilder builder =
5646      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()));
5647    for (byte[] family : families) {
5648      builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family));
5649    }
5650    TableDescriptor tableDescriptor = builder.build();
5651    long time = EnvironmentEdgeManager.currentTime();
5652    RegionInfo primaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName())
5653      .setRegionId(time).setReplicaId(0).build();
5654    RegionInfo secondaryHri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName())
5655      .setRegionId(time).setReplicaId(1).build();
5656
5657    HRegion primaryRegion = null, secondaryRegion = null;
5658
5659    try {
5660      primaryRegion = HBaseTestingUtil.createRegionAndWAL(primaryHri, rootDir,
5661        TEST_UTIL.getConfiguration(), tableDescriptor);
5662
5663      // load some data
5664      putData(primaryRegion, 0, 1000, cq, families);
5665
5666      // flush region
5667      primaryRegion.flush(true);
5668
5669      // open secondary region
5670      secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, tableDescriptor, null, CONF);
5671
5672      // move the file of the primary region to the archive, simulating a compaction
5673      Collection<HStoreFile> storeFiles = primaryRegion.getStore(families[0]).getStorefiles();
5674      primaryRegion.getRegionFileSystem().removeStoreFiles(Bytes.toString(families[0]), storeFiles);
5675      Collection<StoreFileInfo> storeFileInfos =
5676        primaryRegion.getRegionFileSystem().getStoreFiles(Bytes.toString(families[0]));
5677      Assert.assertTrue(storeFileInfos == null || storeFileInfos.isEmpty());
5678
5679      verifyData(secondaryRegion, 0, 1000, cq, families);
5680    } finally {
5681      if (primaryRegion != null) {
5682        HBaseTestingUtil.closeRegionAndWAL(primaryRegion);
5683      }
5684      if (secondaryRegion != null) {
5685        HBaseTestingUtil.closeRegionAndWAL(secondaryRegion);
5686      }
5687    }
5688  }
5689
5690  private void putData(int startRow, int numRows, byte[] qf, byte[]... families)
5691    throws IOException {
5692    putData(this.region, startRow, numRows, qf, families);
5693  }
5694
5695  private void putData(HRegion region, int startRow, int numRows, byte[] qf, byte[]... families)
5696    throws IOException {
5697    putData(region, Durability.SKIP_WAL, startRow, numRows, qf, families);
5698  }
5699
5700  static void putData(HRegion region, Durability durability, int startRow, int numRows, byte[] qf,
5701    byte[]... families) throws IOException {
5702    for (int i = startRow; i < startRow + numRows; i++) {
5703      Put put = new Put(Bytes.toBytes("" + i));
5704      put.setDurability(durability);
5705      for (byte[] family : families) {
5706        put.addColumn(family, qf, null);
5707      }
5708      region.put(put);
5709      LOG.info(put.toString());
5710    }
5711  }
5712
5713  static void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
5714    throws IOException {
5715    for (int i = startRow; i < startRow + numRows; i++) {
5716      byte[] row = Bytes.toBytes("" + i);
5717      Get get = new Get(row);
5718      for (byte[] family : families) {
5719        get.addColumn(family, qf);
5720      }
5721      Result result = newReg.get(get);
5722      Cell[] raw = result.rawCells();
5723      assertEquals(families.length, result.size());
5724      for (int j = 0; j < families.length; j++) {
5725        assertTrue(CellUtil.matchingRows(raw[j], row));
5726        assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
5727        assertTrue(CellUtil.matchingQualifier(raw[j], qf));
5728      }
5729    }
5730  }
5731
5732  static void assertGet(final HRegion r, final byte[] family, final byte[] k) throws IOException {
5733    // Now I have k, get values out and assert they are as expected.
5734    Get get = new Get(k).addFamily(family).readAllVersions();
5735    Cell[] results = r.get(get).rawCells();
5736    for (int j = 0; j < results.length; j++) {
5737      byte[] tmp = CellUtil.cloneValue(results[j]);
5738      // Row should be equal to value every time.
5739      assertTrue(Bytes.equals(k, tmp));
5740    }
5741  }
5742
5743  /*
5744   * Assert first value in the passed region is <code>firstValue</code>. n * n * n * n
5745   */
5746  protected void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue)
5747    throws IOException {
5748    byte[][] families = { fs };
5749    Scan scan = new Scan();
5750    for (int i = 0; i < families.length; i++)
5751      scan.addFamily(families[i]);
5752    InternalScanner s = r.getScanner(scan);
5753    try {
5754      List<Cell> curVals = new ArrayList<>();
5755      boolean first = true;
5756      OUTER_LOOP: while (s.next(curVals)) {
5757        for (Cell kv : curVals) {
5758          byte[] val = CellUtil.cloneValue(kv);
5759          byte[] curval = val;
5760          if (first) {
5761            first = false;
5762            assertTrue(Bytes.compareTo(curval, firstValue) == 0);
5763          } else {
5764            // Not asserting anything. Might as well break.
5765            break OUTER_LOOP;
5766          }
5767        }
5768      }
5769    } finally {
5770      s.close();
5771    }
5772  }
5773
5774  /**
5775   * Test that we get the expected flush results back
5776   */
5777  @Test
5778  public void testFlushResult() throws IOException {
5779    byte[] family = Bytes.toBytes("family");
5780
5781    this.region = initHRegion(tableName, method, family);
5782
5783    // empty memstore, flush doesn't run
5784    HRegion.FlushResult fr = region.flush(true);
5785    assertFalse(fr.isFlushSucceeded());
5786    assertFalse(fr.isCompactionNeeded());
5787
5788    // Flush enough files to get up to the threshold, doesn't need compactions
5789    for (int i = 0; i < 2; i++) {
5790      Put put = new Put(tableName.toBytes()).addColumn(family, family, tableName.toBytes());
5791      region.put(put);
5792      fr = region.flush(true);
5793      assertTrue(fr.isFlushSucceeded());
5794      assertFalse(fr.isCompactionNeeded());
5795    }
5796
5797    // Two flushes after the threshold, compactions are needed
5798    for (int i = 0; i < 2; i++) {
5799      Put put = new Put(tableName.toBytes()).addColumn(family, family, tableName.toBytes());
5800      region.put(put);
5801      fr = region.flush(true);
5802      assertTrue(fr.isFlushSucceeded());
5803      assertTrue(fr.isCompactionNeeded());
5804    }
5805  }
5806
5807  protected Configuration initSplit() {
5808    // Always compact if there is more than one store file.
5809    CONF.setInt("hbase.hstore.compactionThreshold", 2);
5810
5811    CONF.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 10 * 1000);
5812
5813    // Increase the amount of time between client retries
5814    CONF.setLong("hbase.client.pause", 15 * 1000);
5815
5816    // This size should make it so we always split using the addContent
5817    // below. After adding all data, the first region is 1.3M
5818    CONF.setLong(HConstants.HREGION_MAX_FILESIZE, 1024 * 128);
5819    return CONF;
5820  }
5821
5822  /**
5823   * @return A region on which you must call {@link HBaseTestingUtil#closeRegionAndWAL(HRegion)}
5824   *         when done.
5825   */
5826  protected HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
5827    byte[]... families) throws IOException {
5828    return initHRegion(tableName, callingMethod, conf, false, families);
5829  }
5830
5831  /**
5832   * @return A region on which you must call {@link HBaseTestingUtil#closeRegionAndWAL(HRegion)}
5833   *         when done.
5834   */
5835  protected HRegion initHRegion(TableName tableName, String callingMethod, Configuration conf,
5836    boolean isReadOnly, byte[]... families) throws IOException {
5837    return initHRegion(tableName, null, null, callingMethod, conf, isReadOnly, families);
5838  }
5839
5840  protected HRegion initHRegion(TableName tableName, byte[] startKey, byte[] stopKey,
5841    String callingMethod, Configuration conf, boolean isReadOnly, byte[]... families)
5842    throws IOException {
5843    Path logDir = TEST_UTIL.getDataTestDirOnTestFS(callingMethod + ".log");
5844    RegionInfo hri =
5845      RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey).setEndKey(stopKey).build();
5846    final WAL wal = HBaseTestingUtil.createWal(conf, logDir, hri);
5847    return initHRegion(tableName, startKey, stopKey, conf, isReadOnly, Durability.SYNC_WAL, wal,
5848      families);
5849  }
5850
5851  /**
5852   * @return A region on which you must call {@link HBaseTestingUtil#closeRegionAndWAL(HRegion)}
5853   *         when done.
5854   */
5855  public HRegion initHRegion(TableName tableName, byte[] startKey, byte[] stopKey,
5856    Configuration conf, boolean isReadOnly, Durability durability, WAL wal, byte[]... families)
5857    throws IOException {
5858    ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null,
5859      MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);
5860    return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, conf, isReadOnly, durability,
5861      wal, families);
5862  }
5863
5864  /**
5865   * Assert that the passed in Cell has expected contents for the specified row, column & timestamp.
5866   */
5867  private void checkOneCell(Cell kv, byte[] cf, int rowIdx, int colIdx, long ts) {
5868    String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
5869    assertEquals("Row mismatch which checking: " + ctx, "row:" + rowIdx,
5870      Bytes.toString(CellUtil.cloneRow(kv)));
5871    assertEquals("ColumnFamily mismatch while checking: " + ctx, Bytes.toString(cf),
5872      Bytes.toString(CellUtil.cloneFamily(kv)));
5873    assertEquals("Column qualifier mismatch while checking: " + ctx, "column:" + colIdx,
5874      Bytes.toString(CellUtil.cloneQualifier(kv)));
5875    assertEquals("Timestamp mismatch while checking: " + ctx, ts, kv.getTimestamp());
5876    assertEquals("Value mismatch while checking: " + ctx, "value-version-" + ts,
5877      Bytes.toString(CellUtil.cloneValue(kv)));
5878  }
5879
5880  @Test
5881  public void testReverseScanner_FromMemStore_SingleCF_Normal() throws IOException {
5882    byte[] rowC = Bytes.toBytes("rowC");
5883    byte[] rowA = Bytes.toBytes("rowA");
5884    byte[] rowB = Bytes.toBytes("rowB");
5885    byte[] cf = Bytes.toBytes("CF");
5886    byte[][] families = { cf };
5887    byte[] col = Bytes.toBytes("C");
5888    long ts = 1;
5889    this.region = initHRegion(tableName, method, families);
5890    KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
5891    KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, null);
5892    KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
5893    KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
5894    Put put = null;
5895    put = new Put(rowC);
5896    put.add(kv1);
5897    put.add(kv11);
5898    region.put(put);
5899    put = new Put(rowA);
5900    put.add(kv2);
5901    region.put(put);
5902    put = new Put(rowB);
5903    put.add(kv3);
5904    region.put(put);
5905
5906    Scan scan = new Scan().withStartRow(rowC);
5907    scan.readVersions(5);
5908    scan.setReversed(true);
5909    InternalScanner scanner = region.getScanner(scan);
5910    List<Cell> currRow = new ArrayList<>();
5911    boolean hasNext = scanner.next(currRow);
5912    assertEquals(2, currRow.size());
5913    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
5914      currRow.get(0).getRowLength(), rowC, 0, rowC.length));
5915    assertTrue(hasNext);
5916    currRow.clear();
5917    hasNext = scanner.next(currRow);
5918    assertEquals(1, currRow.size());
5919    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
5920      currRow.get(0).getRowLength(), rowB, 0, rowB.length));
5921    assertTrue(hasNext);
5922    currRow.clear();
5923    hasNext = scanner.next(currRow);
5924    assertEquals(1, currRow.size());
5925    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
5926      currRow.get(0).getRowLength(), rowA, 0, rowA.length));
5927    assertFalse(hasNext);
5928    scanner.close();
5929  }
5930
5931  @Test
5932  public void testReverseScanner_FromMemStore_SingleCF_LargerKey() throws IOException {
5933    byte[] rowC = Bytes.toBytes("rowC");
5934    byte[] rowA = Bytes.toBytes("rowA");
5935    byte[] rowB = Bytes.toBytes("rowB");
5936    byte[] rowD = Bytes.toBytes("rowD");
5937    byte[] cf = Bytes.toBytes("CF");
5938    byte[][] families = { cf };
5939    byte[] col = Bytes.toBytes("C");
5940    long ts = 1;
5941    this.region = initHRegion(tableName, method, families);
5942    KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
5943    KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, null);
5944    KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
5945    KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
5946    Put put = null;
5947    put = new Put(rowC);
5948    put.add(kv1);
5949    put.add(kv11);
5950    region.put(put);
5951    put = new Put(rowA);
5952    put.add(kv2);
5953    region.put(put);
5954    put = new Put(rowB);
5955    put.add(kv3);
5956    region.put(put);
5957
5958    Scan scan = new Scan().withStartRow(rowD);
5959    List<Cell> currRow = new ArrayList<>();
5960    scan.setReversed(true);
5961    scan.readVersions(5);
5962    InternalScanner scanner = region.getScanner(scan);
5963    boolean hasNext = scanner.next(currRow);
5964    assertEquals(2, currRow.size());
5965    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
5966      currRow.get(0).getRowLength(), rowC, 0, rowC.length));
5967    assertTrue(hasNext);
5968    currRow.clear();
5969    hasNext = scanner.next(currRow);
5970    assertEquals(1, currRow.size());
5971    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
5972      currRow.get(0).getRowLength(), rowB, 0, rowB.length));
5973    assertTrue(hasNext);
5974    currRow.clear();
5975    hasNext = scanner.next(currRow);
5976    assertEquals(1, currRow.size());
5977    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
5978      currRow.get(0).getRowLength(), rowA, 0, rowA.length));
5979    assertFalse(hasNext);
5980    scanner.close();
5981  }
5982
5983  @Test
5984  public void testReverseScanner_FromMemStore_SingleCF_FullScan() throws IOException {
5985    byte[] rowC = Bytes.toBytes("rowC");
5986    byte[] rowA = Bytes.toBytes("rowA");
5987    byte[] rowB = Bytes.toBytes("rowB");
5988    byte[] cf = Bytes.toBytes("CF");
5989    byte[][] families = { cf };
5990    byte[] col = Bytes.toBytes("C");
5991    long ts = 1;
5992    this.region = initHRegion(tableName, method, families);
5993    KeyValue kv1 = new KeyValue(rowC, cf, col, ts, KeyValue.Type.Put, null);
5994    KeyValue kv11 = new KeyValue(rowC, cf, col, ts + 1, KeyValue.Type.Put, null);
5995    KeyValue kv2 = new KeyValue(rowA, cf, col, ts, KeyValue.Type.Put, null);
5996    KeyValue kv3 = new KeyValue(rowB, cf, col, ts, KeyValue.Type.Put, null);
5997    Put put = null;
5998    put = new Put(rowC);
5999    put.add(kv1);
6000    put.add(kv11);
6001    region.put(put);
6002    put = new Put(rowA);
6003    put.add(kv2);
6004    region.put(put);
6005    put = new Put(rowB);
6006    put.add(kv3);
6007    region.put(put);
6008    Scan scan = new Scan();
6009    List<Cell> currRow = new ArrayList<>();
6010    scan.setReversed(true);
6011    InternalScanner scanner = region.getScanner(scan);
6012    boolean hasNext = scanner.next(currRow);
6013    assertEquals(1, currRow.size());
6014    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6015      currRow.get(0).getRowLength(), rowC, 0, rowC.length));
6016    assertTrue(hasNext);
6017    currRow.clear();
6018    hasNext = scanner.next(currRow);
6019    assertEquals(1, currRow.size());
6020    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6021      currRow.get(0).getRowLength(), rowB, 0, rowB.length));
6022    assertTrue(hasNext);
6023    currRow.clear();
6024    hasNext = scanner.next(currRow);
6025    assertEquals(1, currRow.size());
6026    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6027      currRow.get(0).getRowLength(), rowA, 0, rowA.length));
6028    assertFalse(hasNext);
6029    scanner.close();
6030  }
6031
6032  @Test
6033  public void testReverseScanner_moreRowsMayExistAfter() throws IOException {
6034    // case for "INCLUDE_AND_SEEK_NEXT_ROW & SEEK_NEXT_ROW" endless loop
6035    byte[] rowA = Bytes.toBytes("rowA");
6036    byte[] rowB = Bytes.toBytes("rowB");
6037    byte[] rowC = Bytes.toBytes("rowC");
6038    byte[] rowD = Bytes.toBytes("rowD");
6039    byte[] rowE = Bytes.toBytes("rowE");
6040    byte[] cf = Bytes.toBytes("CF");
6041    byte[][] families = { cf };
6042    byte[] col1 = Bytes.toBytes("col1");
6043    byte[] col2 = Bytes.toBytes("col2");
6044    long ts = 1;
6045    this.region = initHRegion(tableName, method, families);
6046    KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
6047    KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
6048    KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
6049    KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
6050    KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
6051    KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
6052    Put put = null;
6053    put = new Put(rowA);
6054    put.add(kv1);
6055    region.put(put);
6056    put = new Put(rowB);
6057    put.add(kv2);
6058    region.put(put);
6059    put = new Put(rowC);
6060    put.add(kv3);
6061    region.put(put);
6062    put = new Put(rowD);
6063    put.add(kv4_1);
6064    region.put(put);
6065    put = new Put(rowD);
6066    put.add(kv4_2);
6067    region.put(put);
6068    put = new Put(rowE);
6069    put.add(kv5);
6070    region.put(put);
6071    region.flush(true);
6072    Scan scan = new Scan().withStartRow(rowD).withStopRow(rowA);
6073    scan.addColumn(families[0], col1);
6074    scan.setReversed(true);
6075    List<Cell> currRow = new ArrayList<>();
6076    InternalScanner scanner = region.getScanner(scan);
6077    boolean hasNext = scanner.next(currRow);
6078    assertEquals(1, currRow.size());
6079    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6080      currRow.get(0).getRowLength(), rowD, 0, rowD.length));
6081    assertTrue(hasNext);
6082    currRow.clear();
6083    hasNext = scanner.next(currRow);
6084    assertEquals(1, currRow.size());
6085    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6086      currRow.get(0).getRowLength(), rowC, 0, rowC.length));
6087    assertTrue(hasNext);
6088    currRow.clear();
6089    hasNext = scanner.next(currRow);
6090    assertEquals(1, currRow.size());
6091    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6092      currRow.get(0).getRowLength(), rowB, 0, rowB.length));
6093    assertFalse(hasNext);
6094    scanner.close();
6095
6096    scan = new Scan().withStartRow(rowD).withStopRow(rowA);
6097    scan.addColumn(families[0], col2);
6098    scan.setReversed(true);
6099    currRow.clear();
6100    scanner = region.getScanner(scan);
6101    hasNext = scanner.next(currRow);
6102    assertEquals(1, currRow.size());
6103    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6104      currRow.get(0).getRowLength(), rowD, 0, rowD.length));
6105    scanner.close();
6106  }
6107
6108  @Test
6109  public void testReverseScanner_smaller_blocksize() throws IOException {
6110    // case to ensure no conflict with HFile index optimization
6111    byte[] rowA = Bytes.toBytes("rowA");
6112    byte[] rowB = Bytes.toBytes("rowB");
6113    byte[] rowC = Bytes.toBytes("rowC");
6114    byte[] rowD = Bytes.toBytes("rowD");
6115    byte[] rowE = Bytes.toBytes("rowE");
6116    byte[] cf = Bytes.toBytes("CF");
6117    byte[][] families = { cf };
6118    byte[] col1 = Bytes.toBytes("col1");
6119    byte[] col2 = Bytes.toBytes("col2");
6120    long ts = 1;
6121    Configuration conf = new Configuration(CONF);
6122    conf.setInt("test.block.size", 1);
6123    this.region = initHRegion(tableName, method, conf, families);
6124    KeyValue kv1 = new KeyValue(rowA, cf, col1, ts, KeyValue.Type.Put, null);
6125    KeyValue kv2 = new KeyValue(rowB, cf, col1, ts, KeyValue.Type.Put, null);
6126    KeyValue kv3 = new KeyValue(rowC, cf, col1, ts, KeyValue.Type.Put, null);
6127    KeyValue kv4_1 = new KeyValue(rowD, cf, col1, ts, KeyValue.Type.Put, null);
6128    KeyValue kv4_2 = new KeyValue(rowD, cf, col2, ts, KeyValue.Type.Put, null);
6129    KeyValue kv5 = new KeyValue(rowE, cf, col1, ts, KeyValue.Type.Put, null);
6130    Put put = null;
6131    put = new Put(rowA);
6132    put.add(kv1);
6133    region.put(put);
6134    put = new Put(rowB);
6135    put.add(kv2);
6136    region.put(put);
6137    put = new Put(rowC);
6138    put.add(kv3);
6139    region.put(put);
6140    put = new Put(rowD);
6141    put.add(kv4_1);
6142    region.put(put);
6143    put = new Put(rowD);
6144    put.add(kv4_2);
6145    region.put(put);
6146    put = new Put(rowE);
6147    put.add(kv5);
6148    region.put(put);
6149    region.flush(true);
6150    Scan scan = new Scan().withStartRow(rowD).withStopRow(rowA);
6151    scan.addColumn(families[0], col1);
6152    scan.setReversed(true);
6153    List<Cell> currRow = new ArrayList<>();
6154    InternalScanner scanner = region.getScanner(scan);
6155    boolean hasNext = scanner.next(currRow);
6156    assertEquals(1, currRow.size());
6157    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6158      currRow.get(0).getRowLength(), rowD, 0, rowD.length));
6159    assertTrue(hasNext);
6160    currRow.clear();
6161    hasNext = scanner.next(currRow);
6162    assertEquals(1, currRow.size());
6163    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6164      currRow.get(0).getRowLength(), rowC, 0, rowC.length));
6165    assertTrue(hasNext);
6166    currRow.clear();
6167    hasNext = scanner.next(currRow);
6168    assertEquals(1, currRow.size());
6169    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6170      currRow.get(0).getRowLength(), rowB, 0, rowB.length));
6171    assertFalse(hasNext);
6172    scanner.close();
6173
6174    scan = new Scan().withStartRow(rowD).withStopRow(rowA);
6175    scan.addColumn(families[0], col2);
6176    scan.setReversed(true);
6177    currRow.clear();
6178    scanner = region.getScanner(scan);
6179    hasNext = scanner.next(currRow);
6180    assertEquals(1, currRow.size());
6181    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6182      currRow.get(0).getRowLength(), rowD, 0, rowD.length));
6183    scanner.close();
6184  }
6185
6186  @Test
6187  public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs1() throws IOException {
6188    byte[] row0 = Bytes.toBytes("row0"); // 1 kv
6189    byte[] row1 = Bytes.toBytes("row1"); // 2 kv
6190    byte[] row2 = Bytes.toBytes("row2"); // 4 kv
6191    byte[] row3 = Bytes.toBytes("row3"); // 2 kv
6192    byte[] row4 = Bytes.toBytes("row4"); // 5 kv
6193    byte[] row5 = Bytes.toBytes("row5"); // 2 kv
6194    byte[] cf1 = Bytes.toBytes("CF1");
6195    byte[] cf2 = Bytes.toBytes("CF2");
6196    byte[] cf3 = Bytes.toBytes("CF3");
6197    byte[][] families = { cf1, cf2, cf3 };
6198    byte[] col = Bytes.toBytes("C");
6199    long ts = 1;
6200    Configuration conf = new Configuration(CONF);
6201    // disable compactions in this test.
6202    conf.setInt("hbase.hstore.compactionThreshold", 10000);
6203    this.region = initHRegion(tableName, method, conf, families);
6204    // kv naming style: kv(row number) totalKvCountInThisRow seq no
6205    KeyValue kv0_1_1 = new KeyValue(row0, cf1, col, ts, KeyValue.Type.Put, null);
6206    KeyValue kv1_2_1 = new KeyValue(row1, cf2, col, ts, KeyValue.Type.Put, null);
6207    KeyValue kv1_2_2 = new KeyValue(row1, cf1, col, ts + 1, KeyValue.Type.Put, null);
6208    KeyValue kv2_4_1 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
6209    KeyValue kv2_4_2 = new KeyValue(row2, cf1, col, ts, KeyValue.Type.Put, null);
6210    KeyValue kv2_4_3 = new KeyValue(row2, cf3, col, ts, KeyValue.Type.Put, null);
6211    KeyValue kv2_4_4 = new KeyValue(row2, cf1, col, ts + 4, KeyValue.Type.Put, null);
6212    KeyValue kv3_2_1 = new KeyValue(row3, cf2, col, ts, KeyValue.Type.Put, null);
6213    KeyValue kv3_2_2 = new KeyValue(row3, cf1, col, ts + 4, KeyValue.Type.Put, null);
6214    KeyValue kv4_5_1 = new KeyValue(row4, cf1, col, ts, KeyValue.Type.Put, null);
6215    KeyValue kv4_5_2 = new KeyValue(row4, cf3, col, ts, KeyValue.Type.Put, null);
6216    KeyValue kv4_5_3 = new KeyValue(row4, cf3, col, ts + 5, KeyValue.Type.Put, null);
6217    KeyValue kv4_5_4 = new KeyValue(row4, cf2, col, ts, KeyValue.Type.Put, null);
6218    KeyValue kv4_5_5 = new KeyValue(row4, cf1, col, ts + 3, KeyValue.Type.Put, null);
6219    KeyValue kv5_2_1 = new KeyValue(row5, cf2, col, ts, KeyValue.Type.Put, null);
6220    KeyValue kv5_2_2 = new KeyValue(row5, cf3, col, ts, KeyValue.Type.Put, null);
6221    // hfiles(cf1/cf2) :"row1"(1 kv) / "row2"(1 kv) / "row4"(2 kv)
6222    Put put = null;
6223    put = new Put(row1);
6224    put.add(kv1_2_1);
6225    region.put(put);
6226    put = new Put(row2);
6227    put.add(kv2_4_1);
6228    region.put(put);
6229    put = new Put(row4);
6230    put.add(kv4_5_4);
6231    put.add(kv4_5_5);
6232    region.put(put);
6233    region.flush(true);
6234    // hfiles(cf1/cf3) : "row1" (1 kvs) / "row2" (1 kv) / "row4" (2 kv)
6235    put = new Put(row4);
6236    put.add(kv4_5_1);
6237    put.add(kv4_5_3);
6238    region.put(put);
6239    put = new Put(row1);
6240    put.add(kv1_2_2);
6241    region.put(put);
6242    put = new Put(row2);
6243    put.add(kv2_4_4);
6244    region.put(put);
6245    region.flush(true);
6246    // hfiles(cf1/cf3) : "row2"(2 kv) / "row3"(1 kvs) / "row4" (1 kv)
6247    put = new Put(row4);
6248    put.add(kv4_5_2);
6249    region.put(put);
6250    put = new Put(row2);
6251    put.add(kv2_4_2);
6252    put.add(kv2_4_3);
6253    region.put(put);
6254    put = new Put(row3);
6255    put.add(kv3_2_2);
6256    region.put(put);
6257    region.flush(true);
6258    // memstore(cf1/cf2/cf3) : "row0" (1 kvs) / "row3" ( 1 kv) / "row5" (max)
6259    // ( 2 kv)
6260    put = new Put(row0);
6261    put.add(kv0_1_1);
6262    region.put(put);
6263    put = new Put(row3);
6264    put.add(kv3_2_1);
6265    region.put(put);
6266    put = new Put(row5);
6267    put.add(kv5_2_1);
6268    put.add(kv5_2_2);
6269    region.put(put);
6270    // scan range = ["row4", min), skip the max "row5"
6271    Scan scan = new Scan().withStartRow(row4);
6272    scan.readVersions(5);
6273    scan.setBatch(3);
6274    scan.setReversed(true);
6275    InternalScanner scanner = region.getScanner(scan);
6276    List<Cell> currRow = new ArrayList<>();
6277    boolean hasNext = false;
6278    // 1. scan out "row4" (5 kvs), "row5" can't be scanned out since not
6279    // included in scan range
6280    // "row4" takes 2 next() calls since batch=3
6281    hasNext = scanner.next(currRow);
6282    assertEquals(3, currRow.size());
6283    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6284      currRow.get(0).getRowLength(), row4, 0, row4.length));
6285    assertTrue(hasNext);
6286    currRow.clear();
6287    hasNext = scanner.next(currRow);
6288    assertEquals(2, currRow.size());
6289    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6290      currRow.get(0).getRowLength(), row4, 0, row4.length));
6291    assertTrue(hasNext);
6292    // 2. scan out "row3" (2 kv)
6293    currRow.clear();
6294    hasNext = scanner.next(currRow);
6295    assertEquals(2, currRow.size());
6296    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6297      currRow.get(0).getRowLength(), row3, 0, row3.length));
6298    assertTrue(hasNext);
6299    // 3. scan out "row2" (4 kvs)
6300    // "row2" takes 2 next() calls since batch=3
6301    currRow.clear();
6302    hasNext = scanner.next(currRow);
6303    assertEquals(3, currRow.size());
6304    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6305      currRow.get(0).getRowLength(), row2, 0, row2.length));
6306    assertTrue(hasNext);
6307    currRow.clear();
6308    hasNext = scanner.next(currRow);
6309    assertEquals(1, currRow.size());
6310    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6311      currRow.get(0).getRowLength(), row2, 0, row2.length));
6312    assertTrue(hasNext);
6313    // 4. scan out "row1" (2 kv)
6314    currRow.clear();
6315    hasNext = scanner.next(currRow);
6316    assertEquals(2, currRow.size());
6317    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6318      currRow.get(0).getRowLength(), row1, 0, row1.length));
6319    assertTrue(hasNext);
6320    // 5. scan out "row0" (1 kv)
6321    currRow.clear();
6322    hasNext = scanner.next(currRow);
6323    assertEquals(1, currRow.size());
6324    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6325      currRow.get(0).getRowLength(), row0, 0, row0.length));
6326    assertFalse(hasNext);
6327
6328    scanner.close();
6329  }
6330
6331  @Test
6332  public void testReverseScanner_FromMemStoreAndHFiles_MultiCFs2() throws IOException {
6333    byte[] row1 = Bytes.toBytes("row1");
6334    byte[] row2 = Bytes.toBytes("row2");
6335    byte[] row3 = Bytes.toBytes("row3");
6336    byte[] row4 = Bytes.toBytes("row4");
6337    byte[] cf1 = Bytes.toBytes("CF1");
6338    byte[] cf2 = Bytes.toBytes("CF2");
6339    byte[] cf3 = Bytes.toBytes("CF3");
6340    byte[] cf4 = Bytes.toBytes("CF4");
6341    byte[][] families = { cf1, cf2, cf3, cf4 };
6342    byte[] col = Bytes.toBytes("C");
6343    long ts = 1;
6344    Configuration conf = new Configuration(CONF);
6345    // disable compactions in this test.
6346    conf.setInt("hbase.hstore.compactionThreshold", 10000);
6347    this.region = initHRegion(tableName, method, conf, families);
6348    KeyValue kv1 = new KeyValue(row1, cf1, col, ts, KeyValue.Type.Put, null);
6349    KeyValue kv2 = new KeyValue(row2, cf2, col, ts, KeyValue.Type.Put, null);
6350    KeyValue kv3 = new KeyValue(row3, cf3, col, ts, KeyValue.Type.Put, null);
6351    KeyValue kv4 = new KeyValue(row4, cf4, col, ts, KeyValue.Type.Put, null);
6352    // storefile1
6353    Put put = new Put(row1);
6354    put.add(kv1);
6355    region.put(put);
6356    region.flush(true);
6357    // storefile2
6358    put = new Put(row2);
6359    put.add(kv2);
6360    region.put(put);
6361    region.flush(true);
6362    // storefile3
6363    put = new Put(row3);
6364    put.add(kv3);
6365    region.put(put);
6366    region.flush(true);
6367    // memstore
6368    put = new Put(row4);
6369    put.add(kv4);
6370    region.put(put);
6371    // scan range = ["row4", min)
6372    Scan scan = new Scan().withStartRow(row4);
6373    scan.setReversed(true);
6374    scan.setBatch(10);
6375    InternalScanner scanner = region.getScanner(scan);
6376    List<Cell> currRow = new ArrayList<>();
6377    boolean hasNext = scanner.next(currRow);
6378    assertEquals(1, currRow.size());
6379    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6380      currRow.get(0).getRowLength(), row4, 0, row4.length));
6381    assertTrue(hasNext);
6382    currRow.clear();
6383    hasNext = scanner.next(currRow);
6384    assertEquals(1, currRow.size());
6385    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6386      currRow.get(0).getRowLength(), row3, 0, row3.length));
6387    assertTrue(hasNext);
6388    currRow.clear();
6389    hasNext = scanner.next(currRow);
6390    assertEquals(1, currRow.size());
6391    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6392      currRow.get(0).getRowLength(), row2, 0, row2.length));
6393    assertTrue(hasNext);
6394    currRow.clear();
6395    hasNext = scanner.next(currRow);
6396    assertEquals(1, currRow.size());
6397    assertTrue(Bytes.equals(currRow.get(0).getRowArray(), currRow.get(0).getRowOffset(),
6398      currRow.get(0).getRowLength(), row1, 0, row1.length));
6399    assertFalse(hasNext);
6400  }
6401
6402  /**
6403   * Test for HBASE-14497: Reverse Scan threw StackOverflow caused by readPt checking
6404   */
6405  @Test
6406  public void testReverseScanner_StackOverflow() throws IOException {
6407    byte[] cf1 = Bytes.toBytes("CF1");
6408    byte[][] families = { cf1 };
6409    byte[] col = Bytes.toBytes("C");
6410    Configuration conf = new Configuration(CONF);
6411    this.region = initHRegion(tableName, method, conf, families);
6412    // setup with one storefile and one memstore, to create scanner and get an earlier readPt
6413    Put put = new Put(Bytes.toBytes("19998"));
6414    put.addColumn(cf1, col, Bytes.toBytes("val"));
6415    region.put(put);
6416    region.flushcache(true, true, FlushLifeCycleTracker.DUMMY);
6417    Put put2 = new Put(Bytes.toBytes("19997"));
6418    put2.addColumn(cf1, col, Bytes.toBytes("val"));
6419    region.put(put2);
6420
6421    Scan scan = new Scan().withStartRow(Bytes.toBytes("19998"));
6422    scan.setReversed(true);
6423    InternalScanner scanner = region.getScanner(scan);
6424
6425    // create one storefile contains many rows will be skipped
6426    // to check StoreFileScanner.seekToPreviousRow
6427    for (int i = 10000; i < 20000; i++) {
6428      Put p = new Put(Bytes.toBytes("" + i));
6429      p.addColumn(cf1, col, Bytes.toBytes("" + i));
6430      region.put(p);
6431    }
6432    region.flushcache(true, true, FlushLifeCycleTracker.DUMMY);
6433
6434    // create one memstore contains many rows will be skipped
6435    // to check MemStoreScanner.seekToPreviousRow
6436    for (int i = 10000; i < 20000; i++) {
6437      Put p = new Put(Bytes.toBytes("" + i));
6438      p.addColumn(cf1, col, Bytes.toBytes("" + i));
6439      region.put(p);
6440    }
6441
6442    List<Cell> currRow = new ArrayList<>();
6443    boolean hasNext;
6444    do {
6445      hasNext = scanner.next(currRow);
6446    } while (hasNext);
6447    assertEquals(2, currRow.size());
6448    assertEquals("19998", Bytes.toString(currRow.get(0).getRowArray(),
6449      currRow.get(0).getRowOffset(), currRow.get(0).getRowLength()));
6450    assertEquals("19997", Bytes.toString(currRow.get(1).getRowArray(),
6451      currRow.get(1).getRowOffset(), currRow.get(1).getRowLength()));
6452  }
6453
6454  @Test
6455  public void testReverseScanShouldNotScanMemstoreIfReadPtLesser() throws Exception {
6456    byte[] cf1 = Bytes.toBytes("CF1");
6457    byte[][] families = { cf1 };
6458    byte[] col = Bytes.toBytes("C");
6459    this.region = initHRegion(tableName, method, CONF, families);
6460    // setup with one storefile and one memstore, to create scanner and get an earlier readPt
6461    Put put = new Put(Bytes.toBytes("19996"));
6462    put.addColumn(cf1, col, Bytes.toBytes("val"));
6463    region.put(put);
6464    Put put2 = new Put(Bytes.toBytes("19995"));
6465    put2.addColumn(cf1, col, Bytes.toBytes("val"));
6466    region.put(put2);
6467    // create a reverse scan
6468    Scan scan = new Scan().withStartRow(Bytes.toBytes("19996"));
6469    scan.setReversed(true);
6470    RegionScannerImpl scanner = region.getScanner(scan);
6471
6472    // flush the cache. This will reset the store scanner
6473    region.flushcache(true, true, FlushLifeCycleTracker.DUMMY);
6474
6475    // create one memstore contains many rows will be skipped
6476    // to check MemStoreScanner.seekToPreviousRow
6477    for (int i = 10000; i < 20000; i++) {
6478      Put p = new Put(Bytes.toBytes("" + i));
6479      p.addColumn(cf1, col, Bytes.toBytes("" + i));
6480      region.put(p);
6481    }
6482    List<Cell> currRow = new ArrayList<>();
6483    boolean hasNext;
6484    boolean assertDone = false;
6485    do {
6486      hasNext = scanner.next(currRow);
6487      // With HBASE-15871, after the scanner is reset the memstore scanner should not be
6488      // added here
6489      if (!assertDone) {
6490        StoreScanner current = (StoreScanner) (scanner.storeHeap).getCurrentForTesting();
6491        List<KeyValueScanner> scanners = current.getAllScannersForTesting();
6492        assertEquals("There should be only one scanner the store file scanner", 1, scanners.size());
6493        assertDone = true;
6494      }
6495    } while (hasNext);
6496    assertEquals(2, currRow.size());
6497    assertEquals("19996", Bytes.toString(currRow.get(0).getRowArray(),
6498      currRow.get(0).getRowOffset(), currRow.get(0).getRowLength()));
6499    assertEquals("19995", Bytes.toString(currRow.get(1).getRowArray(),
6500      currRow.get(1).getRowOffset(), currRow.get(1).getRowLength()));
6501  }
6502
6503  @Test
6504  public void testReverseScanWhenPutCellsAfterOpenReverseScan() throws Exception {
6505    byte[] cf1 = Bytes.toBytes("CF1");
6506    byte[][] families = { cf1 };
6507    byte[] col = Bytes.toBytes("C");
6508
6509    this.region = initHRegion(tableName, method, CONF, families);
6510
6511    Put put = new Put(Bytes.toBytes("199996"));
6512    put.addColumn(cf1, col, Bytes.toBytes("val"));
6513    region.put(put);
6514    Put put2 = new Put(Bytes.toBytes("199995"));
6515    put2.addColumn(cf1, col, Bytes.toBytes("val"));
6516    region.put(put2);
6517
6518    // Create a reverse scan
6519    Scan scan = new Scan().withStartRow(Bytes.toBytes("199996"));
6520    scan.setReversed(true);
6521    RegionScannerImpl scanner = region.getScanner(scan);
6522
6523    // Put a lot of cells that have sequenceIDs grater than the readPt of the reverse scan
6524    for (int i = 100000; i < 200000; i++) {
6525      Put p = new Put(Bytes.toBytes("" + i));
6526      p.addColumn(cf1, col, Bytes.toBytes("" + i));
6527      region.put(p);
6528    }
6529    List<Cell> currRow = new ArrayList<>();
6530    boolean hasNext;
6531    do {
6532      hasNext = scanner.next(currRow);
6533    } while (hasNext);
6534
6535    assertEquals(2, currRow.size());
6536    assertEquals("199996", Bytes.toString(currRow.get(0).getRowArray(),
6537      currRow.get(0).getRowOffset(), currRow.get(0).getRowLength()));
6538    assertEquals("199995", Bytes.toString(currRow.get(1).getRowArray(),
6539      currRow.get(1).getRowOffset(), currRow.get(1).getRowLength()));
6540  }
6541
6542  @Test
6543  public void testWriteRequestsCounter() throws IOException {
6544    byte[] fam = Bytes.toBytes("info");
6545    byte[][] families = { fam };
6546    this.region = initHRegion(tableName, method, CONF, families);
6547
6548    Assert.assertEquals(0L, region.getWriteRequestsCount());
6549
6550    Put put = new Put(row);
6551    put.addColumn(fam, fam, fam);
6552
6553    Assert.assertEquals(0L, region.getWriteRequestsCount());
6554    region.put(put);
6555    Assert.assertEquals(1L, region.getWriteRequestsCount());
6556    region.put(put);
6557    Assert.assertEquals(2L, region.getWriteRequestsCount());
6558    region.put(put);
6559    Assert.assertEquals(3L, region.getWriteRequestsCount());
6560
6561    region.delete(new Delete(row));
6562    Assert.assertEquals(4L, region.getWriteRequestsCount());
6563  }
6564
6565  @Test
6566  public void testOpenRegionWrittenToWAL() throws Exception {
6567    final ServerName serverName = ServerName.valueOf(name.getMethodName(), 100, 42);
6568    final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName));
6569
6570    TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
6571      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1))
6572      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam2)).build();
6573    RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
6574
6575    // open the region w/o rss and wal and flush some files
6576    region = HBaseTestingUtil.createRegionAndWAL(hri, TEST_UTIL.getDataTestDir(),
6577      TEST_UTIL.getConfiguration(), htd);
6578    assertNotNull(region);
6579
6580    // create a file in fam1 for the region before opening in OpenRegionHandler
6581    region.put(new Put(Bytes.toBytes("a")).addColumn(fam1, fam1, fam1));
6582    region.flush(true);
6583    HBaseTestingUtil.closeRegionAndWAL(region);
6584
6585    ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class);
6586
6587    // capture append() calls
6588    WAL wal = mockWAL();
6589    when(rss.getWAL(any(RegionInfo.class))).thenReturn(wal);
6590
6591    region =
6592      HRegion.openHRegion(hri, htd, rss.getWAL(hri), TEST_UTIL.getConfiguration(), rss, null);
6593
6594    verify(wal, times(1)).appendMarker(any(RegionInfo.class), any(WALKeyImpl.class),
6595      editCaptor.capture());
6596
6597    WALEdit edit = editCaptor.getValue();
6598    assertNotNull(edit);
6599    assertNotNull(edit.getCells());
6600    assertEquals(1, edit.getCells().size());
6601    RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0));
6602    assertNotNull(desc);
6603
6604    LOG.info("RegionEventDescriptor from WAL: " + desc);
6605
6606    assertEquals(RegionEventDescriptor.EventType.REGION_OPEN, desc.getEventType());
6607    assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getTableName().toBytes()));
6608    assertTrue(
6609      Bytes.equals(desc.getEncodedRegionName().toByteArray(), hri.getEncodedNameAsBytes()));
6610    assertTrue(desc.getLogSequenceNumber() > 0);
6611    assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer()));
6612    assertEquals(2, desc.getStoresCount());
6613
6614    StoreDescriptor store = desc.getStores(0);
6615    assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1));
6616    assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1));
6617    assertEquals(1, store.getStoreFileCount()); // 1store file
6618    assertFalse(store.getStoreFile(0).contains("/")); // ensure path is relative
6619
6620    store = desc.getStores(1);
6621    assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2));
6622    assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2));
6623    assertEquals(0, store.getStoreFileCount()); // no store files
6624  }
6625
6626  // Helper for test testOpenRegionWrittenToWALForLogReplay
6627  static class HRegionWithSeqId extends HRegion {
6628    public HRegionWithSeqId(final Path tableDir, final WAL wal, final FileSystem fs,
6629      final Configuration confParam, final RegionInfo regionInfo, final TableDescriptor htd,
6630      final RegionServerServices rsServices) {
6631      super(tableDir, wal, fs, confParam, regionInfo, htd, rsServices);
6632    }
6633
6634    @Override
6635    protected long getNextSequenceId(WAL wal) throws IOException {
6636      return 42;
6637    }
6638  }
6639
6640  @Test
6641  public void testFlushedFileWithNoTags() throws Exception {
6642    final TableName tableName = TableName.valueOf(name.getMethodName());
6643    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
6644      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)).build();
6645    RegionInfo info = RegionInfoBuilder.newBuilder(tableName).build();
6646    Path path = TEST_UTIL.getDataTestDir(getClass().getSimpleName());
6647    region = HBaseTestingUtil.createRegionAndWAL(info, path, TEST_UTIL.getConfiguration(),
6648      tableDescriptor);
6649    Put put = new Put(Bytes.toBytes("a-b-0-0"));
6650    put.addColumn(fam1, qual1, Bytes.toBytes("c1-value"));
6651    region.put(put);
6652    region.flush(true);
6653    HStore store = region.getStore(fam1);
6654    Collection<HStoreFile> storefiles = store.getStorefiles();
6655    for (HStoreFile sf : storefiles) {
6656      assertFalse("Tags should not be present ",
6657        sf.getReader().getHFileReader().getFileContext().isIncludesTags());
6658    }
6659  }
6660
6661  /**
6662   * Utility method to setup a WAL mock.
6663   * <p/>
6664   * Needs to do the bit where we close latch on the WALKeyImpl on append else test hangs.
6665   * @return a mock WAL
6666   */
6667  private WAL mockWAL() throws IOException {
6668    WAL wal = mock(WAL.class);
6669    when(wal.appendData(any(RegionInfo.class), any(WALKeyImpl.class), any(WALEdit.class)))
6670      .thenAnswer(new Answer<Long>() {
6671        @Override
6672        public Long answer(InvocationOnMock invocation) throws Throwable {
6673          WALKeyImpl key = invocation.getArgument(1);
6674          MultiVersionConcurrencyControl.WriteEntry we = key.getMvcc().begin();
6675          key.setWriteEntry(we);
6676          return 1L;
6677        }
6678      });
6679    when(wal.appendMarker(any(RegionInfo.class), any(WALKeyImpl.class), any(WALEdit.class)))
6680      .thenAnswer(new Answer<Long>() {
6681        @Override
6682        public Long answer(InvocationOnMock invocation) throws Throwable {
6683          WALKeyImpl key = invocation.getArgument(1);
6684          MultiVersionConcurrencyControl.WriteEntry we = key.getMvcc().begin();
6685          key.setWriteEntry(we);
6686          return 1L;
6687        }
6688      });
6689    return wal;
6690  }
6691
6692  @Test
6693  public void testCloseRegionWrittenToWAL() throws Exception {
6694    Path rootDir = new Path(dir + name.getMethodName());
6695    CommonFSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);
6696
6697    final ServerName serverName = ServerName.valueOf("testCloseRegionWrittenToWAL", 100, 42);
6698    final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName));
6699
6700    TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
6701      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1))
6702      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam2)).build();
6703    RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
6704
6705    ArgumentCaptor<WALEdit> editCaptor = ArgumentCaptor.forClass(WALEdit.class);
6706
6707    // capture append() calls
6708    WAL wal = mockWAL();
6709    when(rss.getWAL(any(RegionInfo.class))).thenReturn(wal);
6710
6711    // create and then open a region first so that it can be closed later
6712    region =
6713      HRegion.createHRegion(hri, rootDir, TEST_UTIL.getConfiguration(), htd, rss.getWAL(hri));
6714    region =
6715      HRegion.openHRegion(hri, htd, rss.getWAL(hri), TEST_UTIL.getConfiguration(), rss, null);
6716
6717    // close the region
6718    region.close(false);
6719
6720    // 2 times, one for region open, the other close region
6721    verify(wal, times(2)).appendMarker(any(RegionInfo.class), (WALKeyImpl) any(WALKeyImpl.class),
6722      editCaptor.capture());
6723
6724    WALEdit edit = editCaptor.getAllValues().get(1);
6725    assertNotNull(edit);
6726    assertNotNull(edit.getCells());
6727    assertEquals(1, edit.getCells().size());
6728    RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0));
6729    assertNotNull(desc);
6730
6731    LOG.info("RegionEventDescriptor from WAL: " + desc);
6732
6733    assertEquals(RegionEventDescriptor.EventType.REGION_CLOSE, desc.getEventType());
6734    assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getTableName().toBytes()));
6735    assertTrue(
6736      Bytes.equals(desc.getEncodedRegionName().toByteArray(), hri.getEncodedNameAsBytes()));
6737    assertTrue(desc.getLogSequenceNumber() > 0);
6738    assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer()));
6739    assertEquals(2, desc.getStoresCount());
6740
6741    StoreDescriptor store = desc.getStores(0);
6742    assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1));
6743    assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1));
6744    assertEquals(0, store.getStoreFileCount()); // no store files
6745
6746    store = desc.getStores(1);
6747    assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2));
6748    assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2));
6749    assertEquals(0, store.getStoreFileCount()); // no store files
6750  }
6751
6752  /**
6753   * Test RegionTooBusyException thrown when region is busy
6754   */
6755  @Test
6756  public void testRegionTooBusy() throws IOException {
6757    byte[] family = Bytes.toBytes("family");
6758    long defaultBusyWaitDuration =
6759      CONF.getLong("hbase.busy.wait.duration", HRegion.DEFAULT_BUSY_WAIT_DURATION);
6760    CONF.setLong("hbase.busy.wait.duration", 1000);
6761    region = initHRegion(tableName, method, CONF, family);
6762    final AtomicBoolean stopped = new AtomicBoolean(true);
6763    Thread t = new Thread(new Runnable() {
6764      @Override
6765      public void run() {
6766        try {
6767          region.lock.writeLock().lock();
6768          stopped.set(false);
6769          while (!stopped.get()) {
6770            Thread.sleep(100);
6771          }
6772        } catch (InterruptedException ie) {
6773        } finally {
6774          region.lock.writeLock().unlock();
6775        }
6776      }
6777    });
6778    t.start();
6779    Get get = new Get(row);
6780    try {
6781      while (stopped.get()) {
6782        Thread.sleep(100);
6783      }
6784      region.get(get);
6785      fail("Should throw RegionTooBusyException");
6786    } catch (InterruptedException ie) {
6787      fail("test interrupted");
6788    } catch (RegionTooBusyException e) {
6789      // Good, expected
6790    } finally {
6791      stopped.set(true);
6792      try {
6793        t.join();
6794      } catch (Throwable e) {
6795      }
6796
6797      HBaseTestingUtil.closeRegionAndWAL(region);
6798      region = null;
6799      CONF.setLong("hbase.busy.wait.duration", defaultBusyWaitDuration);
6800    }
6801  }
6802
6803  @Test
6804  public void testCellTTLs() throws IOException {
6805    IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge();
6806    EnvironmentEdgeManager.injectEdge(edge);
6807
6808    final byte[] row = Bytes.toBytes("testRow");
6809    final byte[] q1 = Bytes.toBytes("q1");
6810    final byte[] q2 = Bytes.toBytes("q2");
6811    final byte[] q3 = Bytes.toBytes("q3");
6812    final byte[] q4 = Bytes.toBytes("q4");
6813
6814    // 10 seconds
6815    TableDescriptor tableDescriptor =
6816      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
6817        .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam1).setTimeToLive(10).build())
6818        .build();
6819
6820    Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
6821    conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS);
6822
6823    region = HBaseTestingUtil.createRegionAndWAL(
6824      RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build(),
6825      TEST_UTIL.getDataTestDir(), conf, tableDescriptor);
6826    assertNotNull(region);
6827    long now = EnvironmentEdgeManager.currentTime();
6828    // Add a cell that will expire in 5 seconds via cell TTL
6829    region.put(new Put(row).add(new KeyValue(row, fam1, q1, now, HConstants.EMPTY_BYTE_ARRAY,
6830      new ArrayBackedTag[] {
6831        // TTL tags specify ts in milliseconds
6832        new ArrayBackedTag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) })));
6833    // Add a cell that will expire after 10 seconds via family setting
6834    region.put(new Put(row).addColumn(fam1, q2, now, HConstants.EMPTY_BYTE_ARRAY));
6835    // Add a cell that will expire in 15 seconds via cell TTL
6836    region.put(new Put(row).add(new KeyValue(row, fam1, q3, now + 10000 - 1,
6837      HConstants.EMPTY_BYTE_ARRAY, new ArrayBackedTag[] {
6838        // TTL tags specify ts in milliseconds
6839        new ArrayBackedTag(TagType.TTL_TAG_TYPE, Bytes.toBytes(5000L)) })));
6840    // Add a cell that will expire in 20 seconds via family setting
6841    region.put(new Put(row).addColumn(fam1, q4, now + 10000 - 1, HConstants.EMPTY_BYTE_ARRAY));
6842
6843    // Flush so we are sure store scanning gets this right
6844    region.flush(true);
6845
6846    // A query at time T+0 should return all cells
6847    Result r = region.get(new Get(row));
6848    assertNotNull(r.getValue(fam1, q1));
6849    assertNotNull(r.getValue(fam1, q2));
6850    assertNotNull(r.getValue(fam1, q3));
6851    assertNotNull(r.getValue(fam1, q4));
6852
6853    // Increment time to T+5 seconds
6854    edge.incrementTime(5000);
6855
6856    r = region.get(new Get(row));
6857    assertNull(r.getValue(fam1, q1));
6858    assertNotNull(r.getValue(fam1, q2));
6859    assertNotNull(r.getValue(fam1, q3));
6860    assertNotNull(r.getValue(fam1, q4));
6861
6862    // Increment time to T+10 seconds
6863    edge.incrementTime(5000);
6864
6865    r = region.get(new Get(row));
6866    assertNull(r.getValue(fam1, q1));
6867    assertNull(r.getValue(fam1, q2));
6868    assertNotNull(r.getValue(fam1, q3));
6869    assertNotNull(r.getValue(fam1, q4));
6870
6871    // Increment time to T+15 seconds
6872    edge.incrementTime(5000);
6873
6874    r = region.get(new Get(row));
6875    assertNull(r.getValue(fam1, q1));
6876    assertNull(r.getValue(fam1, q2));
6877    assertNull(r.getValue(fam1, q3));
6878    assertNotNull(r.getValue(fam1, q4));
6879
6880    // Increment time to T+20 seconds
6881    edge.incrementTime(10000);
6882
6883    r = region.get(new Get(row));
6884    assertNull(r.getValue(fam1, q1));
6885    assertNull(r.getValue(fam1, q2));
6886    assertNull(r.getValue(fam1, q3));
6887    assertNull(r.getValue(fam1, q4));
6888
6889    // Fun with disappearing increments
6890
6891    // Start at 1
6892    region.put(new Put(row).addColumn(fam1, q1, Bytes.toBytes(1L)));
6893    r = region.get(new Get(row));
6894    byte[] val = r.getValue(fam1, q1);
6895    assertNotNull(val);
6896    assertEquals(1L, Bytes.toLong(val));
6897
6898    // Increment with a TTL of 5 seconds
6899    Increment incr = new Increment(row).addColumn(fam1, q1, 1L);
6900    incr.setTTL(5000);
6901    region.increment(incr); // 2
6902
6903    // New value should be 2
6904    r = region.get(new Get(row));
6905    val = r.getValue(fam1, q1);
6906    assertNotNull(val);
6907    assertEquals(2L, Bytes.toLong(val));
6908
6909    // Increment time to T+25 seconds
6910    edge.incrementTime(5000);
6911
6912    // Value should be back to 1
6913    r = region.get(new Get(row));
6914    val = r.getValue(fam1, q1);
6915    assertNotNull(val);
6916    assertEquals(1L, Bytes.toLong(val));
6917
6918    // Increment time to T+30 seconds
6919    edge.incrementTime(5000);
6920
6921    // Original value written at T+20 should be gone now via family TTL
6922    r = region.get(new Get(row));
6923    assertNull(r.getValue(fam1, q1));
6924  }
6925
6926  @Test
6927  public void testIncrementTimestampsAreMonotonic() throws IOException {
6928    region = initHRegion(tableName, method, CONF, fam1);
6929    ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
6930    EnvironmentEdgeManager.injectEdge(edge);
6931
6932    edge.setValue(10);
6933    Increment inc = new Increment(row);
6934    inc.setDurability(Durability.SKIP_WAL);
6935    inc.addColumn(fam1, qual1, 1L);
6936    region.increment(inc);
6937
6938    Result result = region.get(new Get(row));
6939    Cell c = result.getColumnLatestCell(fam1, qual1);
6940    assertNotNull(c);
6941    assertEquals(10L, c.getTimestamp());
6942
6943    edge.setValue(1); // clock goes back
6944    region.increment(inc);
6945    result = region.get(new Get(row));
6946    c = result.getColumnLatestCell(fam1, qual1);
6947    assertEquals(11L, c.getTimestamp());
6948    assertEquals(2L, Bytes.toLong(c.getValueArray(), c.getValueOffset(), c.getValueLength()));
6949  }
6950
6951  @Test
6952  public void testAppendTimestampsAreMonotonic() throws IOException {
6953    region = initHRegion(tableName, method, CONF, fam1);
6954    ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
6955    EnvironmentEdgeManager.injectEdge(edge);
6956
6957    edge.setValue(10);
6958    Append a = new Append(row);
6959    a.setDurability(Durability.SKIP_WAL);
6960    a.addColumn(fam1, qual1, qual1);
6961    region.append(a);
6962
6963    Result result = region.get(new Get(row));
6964    Cell c = result.getColumnLatestCell(fam1, qual1);
6965    assertNotNull(c);
6966    assertEquals(10L, c.getTimestamp());
6967
6968    edge.setValue(1); // clock goes back
6969    region.append(a);
6970    result = region.get(new Get(row));
6971    c = result.getColumnLatestCell(fam1, qual1);
6972    assertEquals(11L, c.getTimestamp());
6973
6974    byte[] expected = new byte[qual1.length * 2];
6975    System.arraycopy(qual1, 0, expected, 0, qual1.length);
6976    System.arraycopy(qual1, 0, expected, qual1.length, qual1.length);
6977
6978    assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(), expected, 0,
6979      expected.length));
6980  }
6981
6982  @Test
6983  public void testCheckAndMutateTimestampsAreMonotonic() throws IOException {
6984    region = initHRegion(tableName, method, CONF, fam1);
6985    ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
6986    EnvironmentEdgeManager.injectEdge(edge);
6987
6988    edge.setValue(10);
6989    Put p = new Put(row);
6990    p.setDurability(Durability.SKIP_WAL);
6991    p.addColumn(fam1, qual1, qual1);
6992    region.put(p);
6993
6994    Result result = region.get(new Get(row));
6995    Cell c = result.getColumnLatestCell(fam1, qual1);
6996    assertNotNull(c);
6997    assertEquals(10L, c.getTimestamp());
6998
6999    edge.setValue(1); // clock goes back
7000    p = new Put(row);
7001    p.setDurability(Durability.SKIP_WAL);
7002    p.addColumn(fam1, qual1, qual2);
7003    region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(qual1), p);
7004    result = region.get(new Get(row));
7005    c = result.getColumnLatestCell(fam1, qual1);
7006    assertEquals(10L, c.getTimestamp());
7007
7008    assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(), qual2, 0,
7009      qual2.length));
7010  }
7011
7012  @Test
7013  public void testBatchMutateWithWrongRegionException() throws Exception {
7014    final byte[] a = Bytes.toBytes("a");
7015    final byte[] b = Bytes.toBytes("b");
7016    final byte[] c = Bytes.toBytes("c"); // exclusive
7017
7018    int prevLockTimeout = CONF.getInt("hbase.rowlock.wait.duration", 30000);
7019    CONF.setInt("hbase.rowlock.wait.duration", 1000);
7020    region = initHRegion(tableName, a, c, method, CONF, false, fam1);
7021
7022    Mutation[] mutations = new Mutation[] {
7023      new Put(a).add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a)
7024        .setFamily(fam1).setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()),
7025      // this is outside the region boundary
7026      new Put(c).add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(c)
7027        .setFamily(fam1).setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Type.Put).build()),
7028      new Put(b)
7029        .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1)
7030          .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()) };
7031
7032    OperationStatus[] status = region.batchMutate(mutations);
7033    assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode());
7034    assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, status[1].getOperationStatusCode());
7035    assertEquals(OperationStatusCode.SUCCESS, status[2].getOperationStatusCode());
7036
7037    // test with a row lock held for a long time
7038    final CountDownLatch obtainedRowLock = new CountDownLatch(1);
7039    ExecutorService exec = Executors.newFixedThreadPool(2);
7040    Future<Void> f1 = exec.submit(new Callable<Void>() {
7041      @Override
7042      public Void call() throws Exception {
7043        LOG.info("Acquiring row lock");
7044        RowLock rl = region.getRowLock(b);
7045        obtainedRowLock.countDown();
7046        LOG.info("Waiting for 5 seconds before releasing lock");
7047        Threads.sleep(5000);
7048        LOG.info("Releasing row lock");
7049        rl.release();
7050        return null;
7051      }
7052    });
7053    obtainedRowLock.await(30, TimeUnit.SECONDS);
7054
7055    Future<Void> f2 = exec.submit(new Callable<Void>() {
7056      @Override
7057      public Void call() throws Exception {
7058        Mutation[] mutations = new Mutation[] {
7059          new Put(a)
7060            .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a).setFamily(fam1)
7061              .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()),
7062          new Put(b)
7063            .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1)
7064              .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), };
7065
7066        // this will wait for the row lock, and it will eventually succeed
7067        OperationStatus[] status = region.batchMutate(mutations);
7068        assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode());
7069        assertEquals(OperationStatusCode.SUCCESS, status[1].getOperationStatusCode());
7070        return null;
7071      }
7072    });
7073
7074    f1.get();
7075    f2.get();
7076
7077    CONF.setInt("hbase.rowlock.wait.duration", prevLockTimeout);
7078  }
7079
7080  @Test
7081  public void testBatchMutateWithZeroRowLockWait() throws Exception {
7082    final byte[] a = Bytes.toBytes("a");
7083    final byte[] b = Bytes.toBytes("b");
7084    final byte[] c = Bytes.toBytes("c"); // exclusive
7085
7086    Configuration conf = new Configuration(CONF);
7087    conf.setInt("hbase.rowlock.wait.duration", 0);
7088    final RegionInfo hri =
7089      RegionInfoBuilder.newBuilder(tableName).setStartKey(a).setEndKey(c).build();
7090    final TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName)
7091      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)).build();
7092    region = HRegion.createHRegion(hri, TEST_UTIL.getDataTestDir(), conf, htd,
7093      HBaseTestingUtil.createWal(conf, TEST_UTIL.getDataTestDirOnTestFS(method + ".log"), hri));
7094
7095    Mutation[] mutations = new Mutation[] {
7096      new Put(a).add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a)
7097        .setFamily(fam1).setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()),
7098      new Put(b)
7099        .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1)
7100          .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()) };
7101
7102    OperationStatus[] status = region.batchMutate(mutations);
7103    assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode());
7104    assertEquals(OperationStatusCode.SUCCESS, status[1].getOperationStatusCode());
7105
7106    // test with a row lock held for a long time
7107    final CountDownLatch obtainedRowLock = new CountDownLatch(1);
7108    ExecutorService exec = Executors.newFixedThreadPool(2);
7109    Future<Void> f1 = exec.submit(new Callable<Void>() {
7110      @Override
7111      public Void call() throws Exception {
7112        LOG.info("Acquiring row lock");
7113        RowLock rl = region.getRowLock(b);
7114        obtainedRowLock.countDown();
7115        LOG.info("Waiting for 5 seconds before releasing lock");
7116        Threads.sleep(5000);
7117        LOG.info("Releasing row lock");
7118        rl.release();
7119        return null;
7120      }
7121    });
7122    obtainedRowLock.await(30, TimeUnit.SECONDS);
7123
7124    Future<Void> f2 = exec.submit(new Callable<Void>() {
7125      @Override
7126      public Void call() throws Exception {
7127        Mutation[] mutations = new Mutation[] {
7128          new Put(a)
7129            .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(a).setFamily(fam1)
7130              .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()),
7131          new Put(b)
7132            .add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(b).setFamily(fam1)
7133              .setTimestamp(HConstants.LATEST_TIMESTAMP).setType(Cell.Type.Put).build()), };
7134        // when handling row b we are going to spin on the failure to get the row lock
7135        // until the lock above is released, but we will still succeed so long as that
7136        // takes less time then the test time out.
7137        OperationStatus[] status = region.batchMutate(mutations);
7138        assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode());
7139        assertEquals(OperationStatusCode.SUCCESS, status[1].getOperationStatusCode());
7140        return null;
7141      }
7142    });
7143
7144    f1.get();
7145    f2.get();
7146  }
7147
7148  @Test
7149  public void testCheckAndRowMutateTimestampsAreMonotonic() throws IOException {
7150    region = initHRegion(tableName, method, CONF, fam1);
7151    ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
7152    EnvironmentEdgeManager.injectEdge(edge);
7153
7154    edge.setValue(10);
7155    Put p = new Put(row);
7156    p.setDurability(Durability.SKIP_WAL);
7157    p.addColumn(fam1, qual1, qual1);
7158    region.put(p);
7159
7160    Result result = region.get(new Get(row));
7161    Cell c = result.getColumnLatestCell(fam1, qual1);
7162    assertNotNull(c);
7163    assertEquals(10L, c.getTimestamp());
7164
7165    edge.setValue(1); // clock goes back
7166    p = new Put(row);
7167    p.setDurability(Durability.SKIP_WAL);
7168    p.addColumn(fam1, qual1, qual2);
7169    RowMutations rm = new RowMutations(row);
7170    rm.add(p);
7171    assertTrue(region.checkAndRowMutate(row, fam1, qual1, CompareOperator.EQUAL,
7172      new BinaryComparator(qual1), rm));
7173    result = region.get(new Get(row));
7174    c = result.getColumnLatestCell(fam1, qual1);
7175    assertEquals(10L, c.getTimestamp());
7176    LOG.info(
7177      "c value " + Bytes.toStringBinary(c.getValueArray(), c.getValueOffset(), c.getValueLength()));
7178
7179    assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(), qual2, 0,
7180      qual2.length));
7181  }
7182
7183  HRegion initHRegion(TableName tableName, String callingMethod, byte[]... families)
7184    throws IOException {
7185    return initHRegion(tableName, callingMethod, HBaseConfiguration.create(), families);
7186  }
7187
7188  /**
7189   * HBASE-16429 Make sure no stuck if roll writer when ring buffer is filled with appends
7190   * @throws IOException if IO error occurred during test
7191   */
7192  @Test
7193  public void testWritesWhileRollWriter() throws IOException {
7194    int testCount = 10;
7195    int numRows = 1024;
7196    int numFamilies = 2;
7197    int numQualifiers = 2;
7198    final byte[][] families = new byte[numFamilies][];
7199    for (int i = 0; i < numFamilies; i++) {
7200      families[i] = Bytes.toBytes("family" + i);
7201    }
7202    final byte[][] qualifiers = new byte[numQualifiers][];
7203    for (int i = 0; i < numQualifiers; i++) {
7204      qualifiers[i] = Bytes.toBytes("qual" + i);
7205    }
7206
7207    CONF.setInt("hbase.regionserver.wal.disruptor.event.count", 2);
7208    this.region = initHRegion(tableName, method, CONF, families);
7209    try {
7210      List<Thread> threads = new ArrayList<>();
7211      for (int i = 0; i < numRows; i++) {
7212        final int count = i;
7213        Thread t = new Thread(new Runnable() {
7214
7215          @Override
7216          public void run() {
7217            byte[] row = Bytes.toBytes("row" + count);
7218            Put put = new Put(row);
7219            put.setDurability(Durability.SYNC_WAL);
7220            byte[] value = Bytes.toBytes(String.valueOf(count));
7221            for (byte[] family : families) {
7222              for (byte[] qualifier : qualifiers) {
7223                put.addColumn(family, qualifier, count, value);
7224              }
7225            }
7226            try {
7227              region.put(put);
7228            } catch (IOException e) {
7229              throw new RuntimeException(e);
7230            }
7231          }
7232        });
7233        threads.add(t);
7234      }
7235      for (Thread t : threads) {
7236        t.start();
7237      }
7238
7239      for (int i = 0; i < testCount; i++) {
7240        region.getWAL().rollWriter();
7241        Thread.yield();
7242      }
7243    } finally {
7244      try {
7245        HBaseTestingUtil.closeRegionAndWAL(this.region);
7246        CONF.setInt("hbase.regionserver.wal.disruptor.event.count", 16 * 1024);
7247      } catch (DroppedSnapshotException dse) {
7248        // We could get this on way out because we interrupt the background flusher and it could
7249        // fail anywhere causing a DSE over in the background flusher... only it is not properly
7250        // dealt with so could still be memory hanging out when we get to here -- memory we can't
7251        // flush because the accounting is 'off' since original DSE.
7252      }
7253      this.region = null;
7254    }
7255  }
7256
7257  @Test
7258  public void testMutateRow() throws Exception {
7259    final byte[] row = Bytes.toBytes("row");
7260    final byte[] q1 = Bytes.toBytes("q1");
7261    final byte[] q2 = Bytes.toBytes("q2");
7262    final byte[] q3 = Bytes.toBytes("q3");
7263    final byte[] q4 = Bytes.toBytes("q4");
7264    final String v1 = "v1";
7265
7266    region = initHRegion(tableName, method, CONF, fam1);
7267
7268    // Initial values
7269    region
7270      .batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q2, Bytes.toBytes("toBeDeleted")),
7271        new Put(row).addColumn(fam1, q3, Bytes.toBytes(5L)),
7272        new Put(row).addColumn(fam1, q4, Bytes.toBytes("a")), });
7273
7274    // Do mutateRow
7275    Result result = region.mutateRow(
7276      new RowMutations(row).add(Arrays.asList(new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1)),
7277        new Delete(row).addColumns(fam1, q2), new Increment(row).addColumn(fam1, q3, 1),
7278        new Append(row).addColumn(fam1, q4, Bytes.toBytes("b")))));
7279
7280    assertNotNull(result);
7281    assertEquals(6L, Bytes.toLong(result.getValue(fam1, q3)));
7282    assertEquals("ab", Bytes.toString(result.getValue(fam1, q4)));
7283
7284    // Verify the value
7285    result = region.get(new Get(row));
7286    assertEquals(v1, Bytes.toString(result.getValue(fam1, q1)));
7287    assertNull(result.getValue(fam1, q2));
7288    assertEquals(6L, Bytes.toLong(result.getValue(fam1, q3)));
7289    assertEquals("ab", Bytes.toString(result.getValue(fam1, q4)));
7290  }
7291
7292  @Test
7293  public void testMutateRowInParallel() throws Exception {
7294    final int numReaderThreads = 100;
7295    final CountDownLatch latch = new CountDownLatch(numReaderThreads);
7296
7297    final byte[] row = Bytes.toBytes("row");
7298    final byte[] q1 = Bytes.toBytes("q1");
7299    final byte[] q2 = Bytes.toBytes("q2");
7300    final byte[] q3 = Bytes.toBytes("q3");
7301    final byte[] q4 = Bytes.toBytes("q4");
7302    final String v1 = "v1";
7303    final String v2 = "v2";
7304
7305    // We need to ensure the timestamp of the delete operation is more than the previous one
7306    final AtomicLong deleteTimestamp = new AtomicLong();
7307
7308    region = initHRegion(tableName, method, CONF, fam1);
7309
7310    // Initial values
7311    region.batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1))
7312      .addColumn(fam1, q2, deleteTimestamp.getAndIncrement(), Bytes.toBytes(v2))
7313      .addColumn(fam1, q3, Bytes.toBytes(1L)).addColumn(fam1, q4, Bytes.toBytes("a")) });
7314
7315    final AtomicReference<AssertionError> assertionError = new AtomicReference<>();
7316
7317    // Writer thread
7318    Thread writerThread = new Thread(() -> {
7319      try {
7320        while (true) {
7321          // If all the reader threads finish, then stop the writer thread
7322          if (latch.await(0, TimeUnit.MILLISECONDS)) {
7323            return;
7324          }
7325
7326          // Execute the mutations. This should be done atomically
7327          region.mutateRow(new RowMutations(row)
7328            .add(Arrays.asList(new Put(row).addColumn(fam1, q1, Bytes.toBytes(v2)),
7329              new Delete(row).addColumns(fam1, q2, deleteTimestamp.getAndIncrement()),
7330              new Increment(row).addColumn(fam1, q3, 1L),
7331              new Append(row).addColumn(fam1, q4, Bytes.toBytes("b")))));
7332
7333          // We need to ensure the timestamps of the Increment/Append operations are more than the
7334          // previous ones
7335          Result result = region.get(new Get(row).addColumn(fam1, q3).addColumn(fam1, q4));
7336          long tsIncrement = result.getColumnLatestCell(fam1, q3).getTimestamp();
7337          long tsAppend = result.getColumnLatestCell(fam1, q4).getTimestamp();
7338
7339          // Put the initial values
7340          region.batchMutate(new Mutation[] { new Put(row).addColumn(fam1, q1, Bytes.toBytes(v1))
7341            .addColumn(fam1, q2, deleteTimestamp.getAndIncrement(), Bytes.toBytes(v2))
7342            .addColumn(fam1, q3, tsIncrement + 1, Bytes.toBytes(1L))
7343            .addColumn(fam1, q4, tsAppend + 1, Bytes.toBytes("a")) });
7344        }
7345      } catch (Exception e) {
7346        assertionError.set(new AssertionError(e));
7347      }
7348    });
7349    writerThread.start();
7350
7351    // Reader threads
7352    for (int i = 0; i < numReaderThreads; i++) {
7353      new Thread(() -> {
7354        try {
7355          for (int j = 0; j < 10000; j++) {
7356            // Verify the values
7357            Result result = region.get(new Get(row));
7358
7359            // The values should be equals to either the initial values or the values after
7360            // executing the mutations
7361            String q1Value = Bytes.toString(result.getValue(fam1, q1));
7362            if (v1.equals(q1Value)) {
7363              assertEquals(v2, Bytes.toString(result.getValue(fam1, q2)));
7364              assertEquals(1L, Bytes.toLong(result.getValue(fam1, q3)));
7365              assertEquals("a", Bytes.toString(result.getValue(fam1, q4)));
7366            } else if (v2.equals(q1Value)) {
7367              assertNull(Bytes.toString(result.getValue(fam1, q2)));
7368              assertEquals(2L, Bytes.toLong(result.getValue(fam1, q3)));
7369              assertEquals("ab", Bytes.toString(result.getValue(fam1, q4)));
7370            } else {
7371              fail("the qualifier " + Bytes.toString(q1) + " should be " + v1 + " or " + v2
7372                + ", but " + q1Value);
7373            }
7374          }
7375        } catch (Exception e) {
7376          assertionError.set(new AssertionError(e));
7377        } catch (AssertionError e) {
7378          assertionError.set(e);
7379        }
7380
7381        latch.countDown();
7382      }).start();
7383    }
7384
7385    writerThread.join();
7386
7387    if (assertionError.get() != null) {
7388      throw assertionError.get();
7389    }
7390  }
7391
7392  @Test
7393  public void testMutateRow_WriteRequestCount() throws Exception {
7394    byte[] row1 = Bytes.toBytes("row1");
7395    byte[] fam1 = Bytes.toBytes("fam1");
7396    byte[] qf1 = Bytes.toBytes("qualifier");
7397    byte[] val1 = Bytes.toBytes("value1");
7398
7399    RowMutations rm = new RowMutations(row1);
7400    Put put = new Put(row1);
7401    put.addColumn(fam1, qf1, val1);
7402    rm.add(put);
7403
7404    this.region = initHRegion(tableName, method, CONF, fam1);
7405    long wrcBeforeMutate = this.region.writeRequestsCount.longValue();
7406    this.region.mutateRow(rm);
7407    long wrcAfterMutate = this.region.writeRequestsCount.longValue();
7408    Assert.assertEquals(wrcBeforeMutate + rm.getMutations().size(), wrcAfterMutate);
7409  }
7410
7411  @Test
7412  public void testBulkLoadReplicationEnabled() throws IOException {
7413    TEST_UTIL.getConfiguration().setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true);
7414    final ServerName serverName = ServerName.valueOf(name.getMethodName(), 100, 42);
7415    final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName));
7416
7417    TableDescriptor tableDescriptor =
7418      TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
7419        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(fam1)).build();
7420    RegionInfo hri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
7421    region = HRegion.openHRegion(hri, tableDescriptor, rss.getWAL(hri),
7422      TEST_UTIL.getConfiguration(), rss, null);
7423
7424    assertTrue(region.conf.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, false));
7425    String plugins = region.conf.get(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, "");
7426    String replicationCoprocessorClass = ReplicationObserver.class.getCanonicalName();
7427    assertTrue(plugins.contains(replicationCoprocessorClass));
7428    assertTrue(region.getCoprocessorHost().getCoprocessors()
7429      .contains(ReplicationObserver.class.getSimpleName()));
7430  }
7431
7432  /**
7433   * The same as HRegion class, the only difference is that instantiateHStore will create a
7434   * different HStore - HStoreForTesting. [HBASE-8518]
7435   */
7436  public static class HRegionForTesting extends HRegion {
7437
7438    public HRegionForTesting(final Path tableDir, final WAL wal, final FileSystem fs,
7439      final Configuration confParam, final RegionInfo regionInfo, final TableDescriptor htd,
7440      final RegionServerServices rsServices) {
7441      this(new HRegionFileSystem(confParam, fs, tableDir, regionInfo), wal, confParam, htd,
7442        rsServices);
7443    }
7444
7445    public HRegionForTesting(HRegionFileSystem fs, WAL wal, Configuration confParam,
7446      TableDescriptor htd, RegionServerServices rsServices) {
7447      super(fs, wal, confParam, htd, rsServices);
7448    }
7449
7450    /**
7451     * Create HStore instance.
7452     * @return If Mob is enabled, return HMobStore, otherwise return HStoreForTesting.
7453     */
7454    @Override
7455    protected HStore instantiateHStore(final ColumnFamilyDescriptor family, boolean warmup)
7456      throws IOException {
7457      if (family.isMobEnabled()) {
7458        if (HFile.getFormatVersion(this.conf) < HFile.MIN_FORMAT_VERSION_WITH_TAGS) {
7459          throw new IOException("A minimum HFile version of " + HFile.MIN_FORMAT_VERSION_WITH_TAGS
7460            + " is required for MOB feature. Consider setting " + HFile.FORMAT_VERSION_KEY
7461            + " accordingly.");
7462        }
7463        return new HMobStore(this, family, this.conf, warmup);
7464      }
7465      return new HStoreForTesting(this, family, this.conf, warmup);
7466    }
7467  }
7468
7469  /**
7470   * HStoreForTesting is merely the same as HStore, the difference is in the doCompaction method of
7471   * HStoreForTesting there is a checkpoint "hbase.hstore.compaction.complete" which doesn't let
7472   * hstore compaction complete. In the former edition, this config is set in HStore class inside
7473   * compact method, though this is just for testing, otherwise it doesn't do any help. In
7474   * HBASE-8518, we try to get rid of all "hbase.hstore.compaction.complete" config (except for
7475   * testing code).
7476   */
7477  public static class HStoreForTesting extends HStore {
7478
7479    protected HStoreForTesting(final HRegion region, final ColumnFamilyDescriptor family,
7480      final Configuration confParam, boolean warmup) throws IOException {
7481      super(region, family, confParam, warmup);
7482    }
7483
7484    @Override
7485    protected List<HStoreFile> doCompaction(CompactionRequestImpl cr,
7486      Collection<HStoreFile> filesToCompact, User user, long compactionStartTime,
7487      List<Path> newFiles) throws IOException {
7488      // let compaction incomplete.
7489      if (!this.conf.getBoolean("hbase.hstore.compaction.complete", true)) {
7490        LOG.warn("hbase.hstore.compaction.complete is set to false");
7491        List<HStoreFile> sfs = new ArrayList<>(newFiles.size());
7492        final boolean evictOnClose =
7493          getCacheConfig() != null ? getCacheConfig().shouldEvictOnClose() : true;
7494        for (Path newFile : newFiles) {
7495          // Create storefile around what we wrote with a reader on it.
7496          HStoreFile sf = storeEngine.createStoreFileAndReader(newFile);
7497          sf.closeStoreFile(evictOnClose);
7498          sfs.add(sf);
7499        }
7500        return sfs;
7501      }
7502      return super.doCompaction(cr, filesToCompact, user, compactionStartTime, newFiles);
7503    }
7504  }
7505
7506  @Test
7507  public void testCloseNoInterrupt() throws Exception {
7508    byte[] cf1 = Bytes.toBytes("CF1");
7509    byte[][] families = { cf1 };
7510    final int SLEEP_TIME = 10 * 1000;
7511
7512    Configuration conf = new Configuration(CONF);
7513    // Disable close thread interrupt and server abort behavior
7514    conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, false);
7515    conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, 1000);
7516    region = initHRegion(tableName, method, conf, families);
7517
7518    final CountDownLatch latch = new CountDownLatch(1);
7519    final AtomicBoolean holderInterrupted = new AtomicBoolean();
7520    Thread holder = new Thread(new Runnable() {
7521      @Override
7522      public void run() {
7523        try {
7524          LOG.info("Starting region operation holder");
7525          region.startRegionOperation(Operation.SCAN);
7526          latch.countDown();
7527          try {
7528            Thread.sleep(SLEEP_TIME);
7529          } catch (InterruptedException e) {
7530            LOG.info("Interrupted");
7531            holderInterrupted.set(true);
7532          }
7533        } catch (Exception e) {
7534          throw new RuntimeException(e);
7535        } finally {
7536          try {
7537            region.closeRegionOperation();
7538          } catch (IOException e) {
7539          }
7540          LOG.info("Stopped region operation holder");
7541        }
7542      }
7543    });
7544
7545    holder.start();
7546    latch.await();
7547    region.close();
7548    region = null;
7549    holder.join();
7550
7551    assertFalse("Region lock holder should not have been interrupted", holderInterrupted.get());
7552  }
7553
7554  @Test
7555  public void testCloseInterrupt() throws Exception {
7556    byte[] cf1 = Bytes.toBytes("CF1");
7557    byte[][] families = { cf1 };
7558    final int SLEEP_TIME = 10 * 1000;
7559
7560    Configuration conf = new Configuration(CONF);
7561    // Enable close thread interrupt and server abort behavior
7562    conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, true);
7563    // Speed up the unit test, no need to wait default 10 seconds.
7564    conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, 1000);
7565    region = initHRegion(tableName, method, conf, families);
7566
7567    final CountDownLatch latch = new CountDownLatch(1);
7568    final AtomicBoolean holderInterrupted = new AtomicBoolean();
7569    Thread holder = new Thread(new Runnable() {
7570      @Override
7571      public void run() {
7572        try {
7573          LOG.info("Starting region operation holder");
7574          region.startRegionOperation(Operation.SCAN);
7575          latch.countDown();
7576          try {
7577            Thread.sleep(SLEEP_TIME);
7578          } catch (InterruptedException e) {
7579            LOG.info("Interrupted");
7580            holderInterrupted.set(true);
7581          }
7582        } catch (Exception e) {
7583          throw new RuntimeException(e);
7584        } finally {
7585          try {
7586            region.closeRegionOperation();
7587          } catch (IOException e) {
7588          }
7589          LOG.info("Stopped region operation holder");
7590        }
7591      }
7592    });
7593
7594    holder.start();
7595    latch.await();
7596    region.close();
7597    region = null;
7598    holder.join();
7599
7600    assertTrue("Region lock holder was not interrupted", holderInterrupted.get());
7601  }
7602
7603  @Test
7604  public void testCloseAbort() throws Exception {
7605    byte[] cf1 = Bytes.toBytes("CF1");
7606    byte[][] families = { cf1 };
7607    final int SLEEP_TIME = 10 * 1000;
7608
7609    Configuration conf = new Configuration(CONF);
7610    // Enable close thread interrupt and server abort behavior.
7611    conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, true);
7612    // Set the abort interval to a fraction of sleep time so we are guaranteed to be aborted.
7613    conf.setInt(HRegion.CLOSE_WAIT_TIME, SLEEP_TIME / 2);
7614    // Set the wait interval to a fraction of sleep time so we are guaranteed to be interrupted.
7615    conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, SLEEP_TIME / 4);
7616    region = initHRegion(tableName, method, conf, families);
7617    RegionServerServices rsServices = mock(RegionServerServices.class);
7618    when(rsServices.getServerName()).thenReturn(ServerName.valueOf("localhost", 1000, 1000));
7619    region.rsServices = rsServices;
7620
7621    final CountDownLatch latch = new CountDownLatch(1);
7622    Thread holder = new Thread(new Runnable() {
7623      @Override
7624      public void run() {
7625        try {
7626          LOG.info("Starting region operation holder");
7627          region.startRegionOperation(Operation.SCAN);
7628          latch.countDown();
7629          // Hold the lock for SLEEP_TIME seconds no matter how many times we are interrupted.
7630          int timeRemaining = SLEEP_TIME;
7631          while (timeRemaining > 0) {
7632            long start = EnvironmentEdgeManager.currentTime();
7633            try {
7634              Thread.sleep(timeRemaining);
7635            } catch (InterruptedException e) {
7636              LOG.info("Interrupted");
7637            }
7638            long end = EnvironmentEdgeManager.currentTime();
7639            timeRemaining -= end - start;
7640            if (timeRemaining < 0) {
7641              timeRemaining = 0;
7642            }
7643            if (timeRemaining > 0) {
7644              LOG.info("Sleeping again, remaining time " + timeRemaining + " ms");
7645            }
7646          }
7647        } catch (Exception e) {
7648          throw new RuntimeException(e);
7649        } finally {
7650          try {
7651            region.closeRegionOperation();
7652          } catch (IOException e) {
7653          }
7654          LOG.info("Stopped region operation holder");
7655        }
7656      }
7657    });
7658
7659    holder.start();
7660    latch.await();
7661    try {
7662      region.close();
7663    } catch (IOException e) {
7664      LOG.info("Caught expected exception", e);
7665    }
7666    region = null;
7667    holder.join();
7668
7669    // Verify the region tried to abort the server
7670    verify(rsServices, atLeast(1)).abort(anyString(), any());
7671  }
7672
7673  @Test
7674  public void testInterruptProtection() throws Exception {
7675    byte[] cf1 = Bytes.toBytes("CF1");
7676    byte[][] families = { cf1 };
7677    final int SLEEP_TIME = 10 * 1000;
7678
7679    Configuration conf = new Configuration(CONF);
7680    // Enable close thread interrupt and server abort behavior.
7681    conf.setBoolean(HRegion.CLOSE_WAIT_ABORT, true);
7682    conf.setInt(HRegion.CLOSE_WAIT_INTERVAL, 1000);
7683    region = initHRegion(tableName, method, conf, families);
7684
7685    final CountDownLatch latch = new CountDownLatch(1);
7686    final AtomicBoolean holderInterrupted = new AtomicBoolean();
7687    Thread holder = new Thread(new Runnable() {
7688      @Override
7689      public void run() {
7690        try {
7691          LOG.info("Starting region operation holder");
7692          region.startRegionOperation(Operation.SCAN);
7693          LOG.info("Protecting against interrupts");
7694          region.disableInterrupts();
7695          try {
7696            latch.countDown();
7697            try {
7698              Thread.sleep(SLEEP_TIME);
7699            } catch (InterruptedException e) {
7700              LOG.info("Interrupted");
7701              holderInterrupted.set(true);
7702            }
7703          } finally {
7704            region.enableInterrupts();
7705          }
7706        } catch (Exception e) {
7707          throw new RuntimeException(e);
7708        } finally {
7709          try {
7710            region.closeRegionOperation();
7711          } catch (IOException e) {
7712          }
7713          LOG.info("Stopped region operation holder");
7714        }
7715      }
7716    });
7717
7718    holder.start();
7719    latch.await();
7720    region.close();
7721    region = null;
7722    holder.join();
7723
7724    assertFalse("Region lock holder should not have been interrupted", holderInterrupted.get());
7725  }
7726
7727  @Test
7728  public void testRegionOnCoprocessorsChange() throws IOException {
7729    byte[] cf1 = Bytes.toBytes("CF1");
7730    byte[][] families = { cf1 };
7731
7732    Configuration conf = new Configuration(CONF);
7733    region = initHRegion(tableName, method, conf, families);
7734    assertNull(region.getCoprocessorHost());
7735
7736    // set and verify the system coprocessors for region and user region
7737    Configuration newConf = new Configuration(conf);
7738    newConf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, MetaTableMetrics.class.getName());
7739    newConf.set(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
7740      NoOpRegionCoprocessor.class.getName());
7741    // trigger configuration change
7742    region.onConfigurationChange(newConf);
7743    assertTrue(region.getCoprocessorHost() != null);
7744    Set<String> coprocessors = region.getCoprocessorHost().getCoprocessors();
7745    assertTrue(coprocessors.size() == 2);
7746    assertTrue(region.getCoprocessorHost().getCoprocessors()
7747      .contains(MetaTableMetrics.class.getSimpleName()));
7748    assertTrue(region.getCoprocessorHost().getCoprocessors()
7749      .contains(NoOpRegionCoprocessor.class.getSimpleName()));
7750
7751    // remove region coprocessor and keep only user region coprocessor
7752    newConf.unset(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY);
7753    region.onConfigurationChange(newConf);
7754    assertTrue(region.getCoprocessorHost() != null);
7755    coprocessors = region.getCoprocessorHost().getCoprocessors();
7756    assertTrue(coprocessors.size() == 1);
7757    assertTrue(region.getCoprocessorHost().getCoprocessors()
7758      .contains(NoOpRegionCoprocessor.class.getSimpleName()));
7759  }
7760
7761  @Test
7762  public void testRegionOnCoprocessorsWithoutChange() throws IOException {
7763    byte[] cf1 = Bytes.toBytes("CF1");
7764    byte[][] families = { cf1 };
7765
7766    Configuration conf = new Configuration(CONF);
7767    conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
7768      MetaTableMetrics.class.getCanonicalName());
7769    region = initHRegion(tableName, method, conf, families);
7770    // region service is null in unit test, we need to load the coprocessor once
7771    region.setCoprocessorHost(new RegionCoprocessorHost(region, null, conf));
7772    RegionCoprocessor regionCoprocessor =
7773      region.getCoprocessorHost().findCoprocessor(MetaTableMetrics.class.getName());
7774
7775    // simulate when other configuration may have changed and onConfigurationChange execute once
7776    region.onConfigurationChange(conf);
7777    RegionCoprocessor regionCoprocessorAfterOnConfigurationChange =
7778      region.getCoprocessorHost().findCoprocessor(MetaTableMetrics.class.getName());
7779    assertEquals(regionCoprocessor, regionCoprocessorAfterOnConfigurationChange);
7780  }
7781
7782  public static class NoOpRegionCoprocessor implements RegionCoprocessor, RegionObserver {
7783    // a empty region coprocessor class
7784  }
7785}