001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.coprocessor;
019
020import static org.junit.Assert.assertArrayEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import java.lang.reflect.Method;
027import java.util.ArrayList;
028import java.util.Arrays;
029import java.util.List;
030import java.util.Optional;
031import org.apache.hadoop.conf.Configuration;
032import org.apache.hadoop.fs.FileSystem;
033import org.apache.hadoop.fs.Path;
034import org.apache.hadoop.hbase.Cell;
035import org.apache.hadoop.hbase.CellUtil;
036import org.apache.hadoop.hbase.CompareOperator;
037import org.apache.hadoop.hbase.Coprocessor;
038import org.apache.hadoop.hbase.HBaseClassTestRule;
039import org.apache.hadoop.hbase.HBaseTestingUtility;
040import org.apache.hadoop.hbase.HColumnDescriptor;
041import org.apache.hadoop.hbase.HTableDescriptor;
042import org.apache.hadoop.hbase.KeyValue;
043import org.apache.hadoop.hbase.MiniHBaseCluster;
044import org.apache.hadoop.hbase.ServerName;
045import org.apache.hadoop.hbase.TableName;
046import org.apache.hadoop.hbase.client.Admin;
047import org.apache.hadoop.hbase.client.Append;
048import org.apache.hadoop.hbase.client.CheckAndMutate;
049import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
050import org.apache.hadoop.hbase.client.Delete;
051import org.apache.hadoop.hbase.client.Durability;
052import org.apache.hadoop.hbase.client.Get;
053import org.apache.hadoop.hbase.client.Increment;
054import org.apache.hadoop.hbase.client.Mutation;
055import org.apache.hadoop.hbase.client.Put;
056import org.apache.hadoop.hbase.client.RegionInfo;
057import org.apache.hadoop.hbase.client.RegionLocator;
058import org.apache.hadoop.hbase.client.Result;
059import org.apache.hadoop.hbase.client.ResultScanner;
060import org.apache.hadoop.hbase.client.RowMutations;
061import org.apache.hadoop.hbase.client.Scan;
062import org.apache.hadoop.hbase.client.Table;
063import org.apache.hadoop.hbase.client.TableDescriptor;
064import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
065import org.apache.hadoop.hbase.filter.FilterAllFilter;
066import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
067import org.apache.hadoop.hbase.io.hfile.CacheConfig;
068import org.apache.hadoop.hbase.io.hfile.HFile;
069import org.apache.hadoop.hbase.io.hfile.HFileContext;
070import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
071import org.apache.hadoop.hbase.regionserver.FlushLifeCycleTracker;
072import org.apache.hadoop.hbase.regionserver.HRegion;
073import org.apache.hadoop.hbase.regionserver.InternalScanner;
074import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
075import org.apache.hadoop.hbase.regionserver.ScanType;
076import org.apache.hadoop.hbase.regionserver.ScannerContext;
077import org.apache.hadoop.hbase.regionserver.Store;
078import org.apache.hadoop.hbase.regionserver.StoreFile;
079import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
080import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
081import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
082import org.apache.hadoop.hbase.testclassification.CoprocessorTests;
083import org.apache.hadoop.hbase.testclassification.LargeTests;
084import org.apache.hadoop.hbase.tool.LoadIncrementalHFiles;
085import org.apache.hadoop.hbase.util.Bytes;
086import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
087import org.apache.hadoop.hbase.util.JVMClusterUtil;
088import org.apache.hadoop.hbase.util.Threads;
089import org.apache.hadoop.hbase.wal.WALEdit;
090import org.apache.hadoop.hbase.wal.WALKey;
091import org.apache.hadoop.hbase.wal.WALKeyImpl;
092import org.junit.AfterClass;
093import org.junit.Assert;
094import org.junit.BeforeClass;
095import org.junit.ClassRule;
096import org.junit.Rule;
097import org.junit.Test;
098import org.junit.experimental.categories.Category;
099import org.junit.rules.TestName;
100import org.mockito.Mockito;
101import org.slf4j.Logger;
102import org.slf4j.LoggerFactory;
103
104import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
105
106@Category({ CoprocessorTests.class, LargeTests.class })
107public class TestRegionObserverInterface {
108
109  @ClassRule
110  public static final HBaseClassTestRule CLASS_RULE =
111    HBaseClassTestRule.forClass(TestRegionObserverInterface.class);
112
113  private static final Logger LOG = LoggerFactory.getLogger(TestRegionObserverInterface.class);
114
115  public static final TableName TEST_TABLE = TableName.valueOf("TestTable");
116  public static final byte[] FAMILY = Bytes.toBytes("f");
117  public final static byte[] A = Bytes.toBytes("a");
118  public final static byte[] B = Bytes.toBytes("b");
119  public final static byte[] C = Bytes.toBytes("c");
120  public final static byte[] ROW = Bytes.toBytes("testrow");
121
122  private static HBaseTestingUtility util = new HBaseTestingUtility();
123  private static MiniHBaseCluster cluster = null;
124
125  @Rule
126  public TestName name = new TestName();
127
128  @BeforeClass
129  public static void setupBeforeClass() throws Exception {
130    // set configure to indicate which cp should be loaded
131    Configuration conf = util.getConfiguration();
132    conf.setBoolean("hbase.master.distributed.log.replay", true);
133    conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
134      SimpleRegionObserver.class.getName());
135
136    util.startMiniCluster();
137    cluster = util.getMiniHBaseCluster();
138  }
139
140  @AfterClass
141  public static void tearDownAfterClass() throws Exception {
142    util.shutdownMiniCluster();
143  }
144
145  @Test
146  public void testRegionObserver() throws IOException {
147    final TableName tableName =
148      TableName.valueOf(TEST_TABLE.getNameAsString() + "." + name.getMethodName());
149    // recreate table every time in order to reset the status of the
150    // coprocessor.
151    Table table = util.createTable(tableName, new byte[][] { A, B, C });
152    try {
153      verifyMethodResult(SimpleRegionObserver.class,
154        new String[] { "hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", "hadDelete",
155          "hadPostStartRegionOperation", "hadPostCloseRegionOperation",
156          "hadPostBatchMutateIndispensably" },
157        tableName, new Boolean[] { false, false, false, false, false, false, false, false });
158
159      Put put = new Put(ROW);
160      put.addColumn(A, A, A);
161      put.addColumn(B, B, B);
162      put.addColumn(C, C, C);
163      table.put(put);
164
165      verifyMethodResult(SimpleRegionObserver.class,
166        new String[] { "hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", "hadPreBatchMutate",
167          "hadPostBatchMutate", "hadDelete", "hadPostStartRegionOperation",
168          "hadPostCloseRegionOperation", "hadPostBatchMutateIndispensably" },
169        TEST_TABLE,
170        new Boolean[] { false, false, true, true, true, true, false, true, true, true });
171
172      verifyMethodResult(SimpleRegionObserver.class,
173        new String[] { "getCtPreOpen", "getCtPostOpen", "getCtPreClose", "getCtPostClose" },
174        tableName, new Integer[] { 1, 1, 0, 0 });
175
176      Get get = new Get(ROW);
177      get.addColumn(A, A);
178      get.addColumn(B, B);
179      get.addColumn(C, C);
180      table.get(get);
181
182      verifyMethodResult(SimpleRegionObserver.class,
183        new String[] { "hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", "hadDelete",
184          "hadPrePreparedDeleteTS" },
185        tableName, new Boolean[] { true, true, true, true, false, false });
186
187      Delete delete = new Delete(ROW);
188      delete.addColumn(A, A);
189      delete.addColumn(B, B);
190      delete.addColumn(C, C);
191      table.delete(delete);
192
193      verifyMethodResult(SimpleRegionObserver.class,
194        new String[] { "hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", "hadPreBatchMutate",
195          "hadPostBatchMutate", "hadDelete", "hadPrePreparedDeleteTS" },
196        tableName, new Boolean[] { true, true, true, true, true, true, true, true });
197    } finally {
198      util.deleteTable(tableName);
199      table.close();
200    }
201    verifyMethodResult(SimpleRegionObserver.class,
202      new String[] { "getCtPreOpen", "getCtPostOpen", "getCtPreClose", "getCtPostClose" },
203      tableName, new Integer[] { 1, 1, 1, 1 });
204  }
205
206  @Test
207  public void testRowMutation() throws IOException {
208    final TableName tableName =
209      TableName.valueOf(TEST_TABLE.getNameAsString() + "." + name.getMethodName());
210    Table table = util.createTable(tableName, new byte[][] { A, B, C });
211    try {
212      verifyMethodResult(SimpleRegionObserver.class,
213        new String[] { "hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", "hadDeleted" },
214        tableName, new Boolean[] { false, false, false, false, false });
215      Put put = new Put(ROW);
216      put.addColumn(A, A, A);
217      put.addColumn(B, B, B);
218      put.addColumn(C, C, C);
219
220      Delete delete = new Delete(ROW);
221      delete.addColumn(A, A);
222      delete.addColumn(B, B);
223      delete.addColumn(C, C);
224
225      RowMutations arm = new RowMutations(ROW);
226      arm.add(put);
227      arm.add(delete);
228      table.mutateRow(arm);
229
230      verifyMethodResult(SimpleRegionObserver.class,
231        new String[] { "hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", "hadDeleted" },
232        tableName, new Boolean[] { false, false, true, true, true });
233    } finally {
234      util.deleteTable(tableName);
235      table.close();
236    }
237  }
238
239  @Test
240  public void testIncrementHook() throws IOException {
241    final TableName tableName =
242      TableName.valueOf(TEST_TABLE.getNameAsString() + "." + name.getMethodName());
243    Table table = util.createTable(tableName, new byte[][] { A, B, C });
244    try {
245      Increment inc = new Increment(Bytes.toBytes(0));
246      inc.addColumn(A, A, 1);
247
248      verifyMethodResult(SimpleRegionObserver.class,
249        new String[] { "hadPreIncrement", "hadPostIncrement", "hadPreIncrementAfterRowLock",
250          "hadPreBatchMutate", "hadPostBatchMutate", "hadPostBatchMutateIndispensably" },
251        tableName, new Boolean[] { false, false, false, false, false, false });
252
253      table.increment(inc);
254
255      verifyMethodResult(SimpleRegionObserver.class,
256        new String[] { "hadPreIncrement", "hadPostIncrement", "hadPreIncrementAfterRowLock",
257          "hadPreBatchMutate", "hadPostBatchMutate", "hadPostBatchMutateIndispensably" },
258        tableName, new Boolean[] { true, true, true, true, true, true });
259    } finally {
260      util.deleteTable(tableName);
261      table.close();
262    }
263  }
264
265  @Test
266  public void testCheckAndPutHooks() throws IOException {
267    final TableName tableName =
268      TableName.valueOf(TEST_TABLE.getNameAsString() + "." + name.getMethodName());
269    try (Table table = util.createTable(tableName, new byte[][] { A, B, C })) {
270      Put p = new Put(Bytes.toBytes(0));
271      p.addColumn(A, A, A);
272      table.put(p);
273      p = new Put(Bytes.toBytes(0));
274      p.addColumn(A, A, A);
275      verifyMethodResult(SimpleRegionObserver.class,
276        new String[] { "getPreCheckAndPut", "getPreCheckAndPutAfterRowLock", "getPostCheckAndPut",
277          "getPreCheckAndPutWithFilter", "getPreCheckAndPutWithFilterAfterRowLock",
278          "getPostCheckAndPutWithFilter", "getPreCheckAndMutate",
279          "getPreCheckAndMutateAfterRowLock", "getPostCheckAndMutate" },
280        tableName, new Integer[] { 0, 0, 0, 0, 0, 0, 0, 0, 0 });
281
282      table.checkAndMutate(Bytes.toBytes(0), A).qualifier(A).ifEquals(A).thenPut(p);
283      verifyMethodResult(SimpleRegionObserver.class,
284        new String[] { "getPreCheckAndPut", "getPreCheckAndPutAfterRowLock", "getPostCheckAndPut",
285          "getPreCheckAndPutWithFilter", "getPreCheckAndPutWithFilterAfterRowLock",
286          "getPostCheckAndPutWithFilter", "getPreCheckAndMutate",
287          "getPreCheckAndMutateAfterRowLock", "getPostCheckAndMutate" },
288        tableName, new Integer[] { 1, 1, 1, 0, 0, 0, 1, 1, 1 });
289
290      table.checkAndMutate(Bytes.toBytes(0),
291        new SingleColumnValueFilter(A, A, CompareOperator.EQUAL, A)).thenPut(p);
292      verifyMethodResult(SimpleRegionObserver.class,
293        new String[] { "getPreCheckAndPut", "getPreCheckAndPutAfterRowLock", "getPostCheckAndPut",
294          "getPreCheckAndPutWithFilter", "getPreCheckAndPutWithFilterAfterRowLock",
295          "getPostCheckAndPutWithFilter", "getPreCheckAndMutate",
296          "getPreCheckAndMutateAfterRowLock", "getPostCheckAndMutate" },
297        tableName, new Integer[] { 1, 1, 1, 1, 1, 1, 2, 2, 2 });
298    } finally {
299      util.deleteTable(tableName);
300    }
301  }
302
303  @Test
304  public void testCheckAndDeleteHooks() throws IOException {
305    final TableName tableName =
306      TableName.valueOf(TEST_TABLE.getNameAsString() + "." + name.getMethodName());
307    Table table = util.createTable(tableName, new byte[][] { A, B, C });
308    try {
309      Put p = new Put(Bytes.toBytes(0));
310      p.addColumn(A, A, A);
311      table.put(p);
312      Delete d = new Delete(Bytes.toBytes(0));
313      table.delete(d);
314      verifyMethodResult(SimpleRegionObserver.class,
315        new String[] { "getPreCheckAndDelete", "getPreCheckAndDeleteAfterRowLock",
316          "getPostCheckAndDelete", "getPreCheckAndDeleteWithFilter",
317          "getPreCheckAndDeleteWithFilterAfterRowLock", "getPostCheckAndDeleteWithFilter",
318          "getPreCheckAndMutate", "getPreCheckAndMutateAfterRowLock", "getPostCheckAndMutate" },
319        tableName, new Integer[] { 0, 0, 0, 0, 0, 0, 0, 0, 0 });
320
321      table.checkAndMutate(Bytes.toBytes(0), A).qualifier(A).ifEquals(A).thenDelete(d);
322      verifyMethodResult(SimpleRegionObserver.class,
323        new String[] { "getPreCheckAndDelete", "getPreCheckAndDeleteAfterRowLock",
324          "getPostCheckAndDelete", "getPreCheckAndDeleteWithFilter",
325          "getPreCheckAndDeleteWithFilterAfterRowLock", "getPostCheckAndDeleteWithFilter",
326          "getPreCheckAndMutate", "getPreCheckAndMutateAfterRowLock", "getPostCheckAndMutate" },
327        tableName, new Integer[] { 1, 1, 1, 0, 0, 0, 1, 1, 1 });
328
329      table.checkAndMutate(Bytes.toBytes(0),
330        new SingleColumnValueFilter(A, A, CompareOperator.EQUAL, A)).thenDelete(d);
331      verifyMethodResult(SimpleRegionObserver.class,
332        new String[] { "getPreCheckAndDelete", "getPreCheckAndDeleteAfterRowLock",
333          "getPostCheckAndDelete", "getPreCheckAndDeleteWithFilter",
334          "getPreCheckAndDeleteWithFilterAfterRowLock", "getPostCheckAndDeleteWithFilter",
335          "getPreCheckAndMutate", "getPreCheckAndMutateAfterRowLock", "getPostCheckAndMutate" },
336        tableName, new Integer[] { 1, 1, 1, 1, 1, 1, 2, 2, 2 });
337    } finally {
338      util.deleteTable(tableName);
339      table.close();
340    }
341  }
342
343  @Test
344  public void testCheckAndIncrementHooks() throws Exception {
345    final TableName tableName =
346      TableName.valueOf(TEST_TABLE.getNameAsString() + "." + name.getMethodName());
347    Table table = util.createTable(tableName, new byte[][] { A, B, C });
348    try {
349      byte[] row = Bytes.toBytes(0);
350
351      verifyMethodResult(
352        SimpleRegionObserver.class, new String[] { "getPreCheckAndMutate",
353          "getPreCheckAndMutateAfterRowLock", "getPostCheckAndMutate" },
354        tableName, new Integer[] { 0, 0, 0 });
355
356      table.checkAndMutate(CheckAndMutate.newBuilder(row).ifNotExists(A, A)
357        .build(new Increment(row).addColumn(A, A, 1)));
358      verifyMethodResult(
359        SimpleRegionObserver.class, new String[] { "getPreCheckAndMutate",
360          "getPreCheckAndMutateAfterRowLock", "getPostCheckAndMutate" },
361        tableName, new Integer[] { 1, 1, 1 });
362
363      table.checkAndMutate(CheckAndMutate.newBuilder(row).ifEquals(A, A, Bytes.toBytes(1L))
364        .build(new Increment(row).addColumn(A, A, 1)));
365      verifyMethodResult(
366        SimpleRegionObserver.class, new String[] { "getPreCheckAndMutate",
367          "getPreCheckAndMutateAfterRowLock", "getPostCheckAndMutate" },
368        tableName, new Integer[] { 2, 2, 2 });
369    } finally {
370      util.deleteTable(tableName);
371      table.close();
372    }
373  }
374
375  @Test
376  public void testCheckAndAppendHooks() throws Exception {
377    final TableName tableName =
378      TableName.valueOf(TEST_TABLE.getNameAsString() + "." + name.getMethodName());
379    Table table = util.createTable(tableName, new byte[][] { A, B, C });
380    try {
381      byte[] row = Bytes.toBytes(0);
382
383      verifyMethodResult(
384        SimpleRegionObserver.class, new String[] { "getPreCheckAndMutate",
385          "getPreCheckAndMutateAfterRowLock", "getPostCheckAndMutate" },
386        tableName, new Integer[] { 0, 0, 0 });
387
388      table.checkAndMutate(
389        CheckAndMutate.newBuilder(row).ifNotExists(A, A).build(new Append(row).addColumn(A, A, A)));
390      verifyMethodResult(
391        SimpleRegionObserver.class, new String[] { "getPreCheckAndMutate",
392          "getPreCheckAndMutateAfterRowLock", "getPostCheckAndMutate" },
393        tableName, new Integer[] { 1, 1, 1 });
394
395      table.checkAndMutate(
396        CheckAndMutate.newBuilder(row).ifEquals(A, A, A).build(new Append(row).addColumn(A, A, A)));
397      verifyMethodResult(
398        SimpleRegionObserver.class, new String[] { "getPreCheckAndMutate",
399          "getPreCheckAndMutateAfterRowLock", "getPostCheckAndMutate" },
400        tableName, new Integer[] { 2, 2, 2 });
401    } finally {
402      util.deleteTable(tableName);
403      table.close();
404    }
405  }
406
407  @Test
408  public void testCheckAndRowMutationsHooks() throws Exception {
409    final TableName tableName =
410      TableName.valueOf(TEST_TABLE.getNameAsString() + "." + name.getMethodName());
411    Table table = util.createTable(tableName, new byte[][] { A, B, C });
412    try {
413      byte[] row = Bytes.toBytes(0);
414
415      Put p = new Put(row).addColumn(A, A, A);
416      table.put(p);
417      verifyMethodResult(
418        SimpleRegionObserver.class, new String[] { "getPreCheckAndMutate",
419          "getPreCheckAndMutateAfterRowLock", "getPostCheckAndMutate" },
420        tableName, new Integer[] { 0, 0, 0 });
421
422      table
423        .checkAndMutate(CheckAndMutate.newBuilder(row).ifEquals(A, A, A).build(new RowMutations(row)
424          .add((Mutation) new Put(row).addColumn(B, B, B)).add((Mutation) new Delete(row))));
425      verifyMethodResult(
426        SimpleRegionObserver.class, new String[] { "getPreCheckAndMutate",
427          "getPreCheckAndMutateAfterRowLock", "getPostCheckAndMutate" },
428        tableName, new Integer[] { 1, 1, 1 });
429
430      Object[] result = new Object[2];
431      table.batch(
432        Arrays.asList(p,
433          CheckAndMutate.newBuilder(row).ifEquals(A, A, A).build(new RowMutations(row)
434            .add((Mutation) new Put(row).addColumn(B, B, B)).add((Mutation) new Delete(row)))),
435        result);
436      verifyMethodResult(
437        SimpleRegionObserver.class, new String[] { "getPreCheckAndMutate",
438          "getPreCheckAndMutateAfterRowLock", "getPostCheckAndMutate" },
439        tableName, new Integer[] { 2, 2, 2 });
440    } finally {
441      util.deleteTable(tableName);
442      table.close();
443    }
444  }
445
446  @Test
447  public void testAppendHook() throws IOException {
448    final TableName tableName =
449      TableName.valueOf(TEST_TABLE.getNameAsString() + "." + name.getMethodName());
450    Table table = util.createTable(tableName, new byte[][] { A, B, C });
451    try {
452      Append app = new Append(Bytes.toBytes(0));
453      app.addColumn(A, A, A);
454
455      verifyMethodResult(SimpleRegionObserver.class,
456        new String[] { "hadPreAppend", "hadPostAppend", "hadPreAppendAfterRowLock",
457          "hadPreBatchMutate", "hadPostBatchMutate", "hadPostBatchMutateIndispensably" },
458        tableName, new Boolean[] { false, false, false, false, false, false });
459
460      table.append(app);
461
462      verifyMethodResult(SimpleRegionObserver.class,
463        new String[] { "hadPreAppend", "hadPostAppend", "hadPreAppendAfterRowLock",
464          "hadPreBatchMutate", "hadPostBatchMutate", "hadPostBatchMutateIndispensably" },
465        tableName, new Boolean[] { true, true, true, true, true, true });
466    } finally {
467      util.deleteTable(tableName);
468      table.close();
469    }
470  }
471
472  @Test
473  // HBase-3583
474  public void testHBase3583() throws IOException {
475    final TableName tableName = TableName.valueOf(name.getMethodName());
476    util.createTable(tableName, new byte[][] { A, B, C });
477    util.waitUntilAllRegionsAssigned(tableName);
478
479    verifyMethodResult(SimpleRegionObserver.class,
480      new String[] { "hadPreGet", "hadPostGet", "wasScannerNextCalled", "wasScannerCloseCalled" },
481      tableName, new Boolean[] { false, false, false, false });
482
483    Table table = util.getConnection().getTable(tableName);
484    Put put = new Put(ROW);
485    put.addColumn(A, A, A);
486    table.put(put);
487
488    Get get = new Get(ROW);
489    get.addColumn(A, A);
490    table.get(get);
491
492    // verify that scannerNext and scannerClose upcalls won't be invoked
493    // when we perform get().
494    verifyMethodResult(SimpleRegionObserver.class,
495      new String[] { "hadPreGet", "hadPostGet", "wasScannerNextCalled", "wasScannerCloseCalled" },
496      tableName, new Boolean[] { true, true, false, false });
497
498    Scan s = new Scan();
499    ResultScanner scanner = table.getScanner(s);
500    try {
501      for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
502      }
503    } finally {
504      scanner.close();
505    }
506
507    // now scanner hooks should be invoked.
508    verifyMethodResult(SimpleRegionObserver.class,
509      new String[] { "wasScannerNextCalled", "wasScannerCloseCalled" }, tableName,
510      new Boolean[] { true, true });
511    util.deleteTable(tableName);
512    table.close();
513  }
514
515  @Test
516  public void testHBASE14489() throws IOException {
517    final TableName tableName = TableName.valueOf(name.getMethodName());
518    Table table = util.createTable(tableName, new byte[][] { A });
519    Put put = new Put(ROW);
520    put.addColumn(A, A, A);
521    table.put(put);
522
523    Scan s = new Scan();
524    s.setFilter(new FilterAllFilter());
525    ResultScanner scanner = table.getScanner(s);
526    try {
527      for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
528      }
529    } finally {
530      scanner.close();
531    }
532    verifyMethodResult(SimpleRegionObserver.class, new String[] { "wasScannerFilterRowCalled" },
533      tableName, new Boolean[] { true });
534    util.deleteTable(tableName);
535    table.close();
536
537  }
538
539  @Test
540  // HBase-3758
541  public void testHBase3758() throws IOException {
542    final TableName tableName = TableName.valueOf(name.getMethodName());
543    util.createTable(tableName, new byte[][] { A, B, C });
544
545    verifyMethodResult(SimpleRegionObserver.class,
546      new String[] { "hadDeleted", "wasScannerOpenCalled" }, tableName,
547      new Boolean[] { false, false });
548
549    Table table = util.getConnection().getTable(tableName);
550    Put put = new Put(ROW);
551    put.addColumn(A, A, A);
552    table.put(put);
553
554    Delete delete = new Delete(ROW);
555    table.delete(delete);
556
557    verifyMethodResult(SimpleRegionObserver.class,
558      new String[] { "hadDeleted", "wasScannerOpenCalled" }, tableName,
559      new Boolean[] { true, false });
560
561    Scan s = new Scan();
562    ResultScanner scanner = table.getScanner(s);
563    try {
564      for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
565      }
566    } finally {
567      scanner.close();
568    }
569
570    // now scanner hooks should be invoked.
571    verifyMethodResult(SimpleRegionObserver.class, new String[] { "wasScannerOpenCalled" },
572      tableName, new Boolean[] { true });
573    util.deleteTable(tableName);
574    table.close();
575  }
576
577  /* Overrides compaction to only output rows with keys that are even numbers */
578  public static class EvenOnlyCompactor implements RegionCoprocessor, RegionObserver {
579    long lastCompaction;
580    long lastFlush;
581
582    @Override
583    public Optional<RegionObserver> getRegionObserver() {
584      return Optional.of(this);
585    }
586
587    @Override
588    public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store,
589      InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker,
590      CompactionRequest request) {
591      return new InternalScanner() {
592
593        @Override
594        public boolean next(List<Cell> results, ScannerContext scannerContext) throws IOException {
595          List<Cell> internalResults = new ArrayList<>();
596          boolean hasMore;
597          do {
598            hasMore = scanner.next(internalResults, scannerContext);
599            if (!internalResults.isEmpty()) {
600              long row = Bytes.toLong(CellUtil.cloneValue(internalResults.get(0)));
601              if (row % 2 == 0) {
602                // return this row
603                break;
604              }
605              // clear and continue
606              internalResults.clear();
607            }
608          } while (hasMore);
609
610          if (!internalResults.isEmpty()) {
611            results.addAll(internalResults);
612          }
613          return hasMore;
614        }
615
616        @Override
617        public void close() throws IOException {
618          scanner.close();
619        }
620      };
621    }
622
623    @Override
624    public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store,
625      StoreFile resultFile, CompactionLifeCycleTracker tracker, CompactionRequest request) {
626      lastCompaction = EnvironmentEdgeManager.currentTime();
627    }
628
629    @Override
630    public void postFlush(ObserverContext<RegionCoprocessorEnvironment> e,
631      FlushLifeCycleTracker tracker) {
632      lastFlush = EnvironmentEdgeManager.currentTime();
633    }
634  }
635
636  /**
637   * Tests overriding compaction handling via coprocessor hooks
638   */
639  @Test
640  public void testCompactionOverride() throws Exception {
641    final TableName compactTable = TableName.valueOf(name.getMethodName());
642    Admin admin = util.getAdmin();
643    if (admin.tableExists(compactTable)) {
644      admin.disableTable(compactTable);
645      admin.deleteTable(compactTable);
646    }
647
648    HTableDescriptor htd = new HTableDescriptor(compactTable);
649    htd.addFamily(new HColumnDescriptor(A));
650    htd.addCoprocessor(EvenOnlyCompactor.class.getName());
651    admin.createTable(htd);
652
653    Table table = util.getConnection().getTable(compactTable);
654    for (long i = 1; i <= 10; i++) {
655      byte[] iBytes = Bytes.toBytes(i);
656      Put put = new Put(iBytes);
657      put.setDurability(Durability.SKIP_WAL);
658      put.addColumn(A, A, iBytes);
659      table.put(put);
660    }
661
662    HRegion firstRegion = cluster.getRegions(compactTable).get(0);
663    Coprocessor cp = firstRegion.getCoprocessorHost().findCoprocessor(EvenOnlyCompactor.class);
664    assertNotNull("EvenOnlyCompactor coprocessor should be loaded", cp);
665    EvenOnlyCompactor compactor = (EvenOnlyCompactor) cp;
666
667    // force a compaction
668    long ts = EnvironmentEdgeManager.currentTime();
669    admin.flush(compactTable);
670    // wait for flush
671    for (int i = 0; i < 10; i++) {
672      if (compactor.lastFlush >= ts) {
673        break;
674      }
675      Thread.sleep(1000);
676    }
677    assertTrue("Flush didn't complete", compactor.lastFlush >= ts);
678    LOG.debug("Flush complete");
679
680    ts = compactor.lastFlush;
681    admin.majorCompact(compactTable);
682    // wait for compaction
683    for (int i = 0; i < 30; i++) {
684      if (compactor.lastCompaction >= ts) {
685        break;
686      }
687      Thread.sleep(1000);
688    }
689    LOG.debug("Last compaction was at " + compactor.lastCompaction);
690    assertTrue("Compaction didn't complete", compactor.lastCompaction >= ts);
691
692    // only even rows should remain
693    ResultScanner scanner = table.getScanner(new Scan());
694    try {
695      for (long i = 2; i <= 10; i += 2) {
696        Result r = scanner.next();
697        assertNotNull(r);
698        assertFalse(r.isEmpty());
699        byte[] iBytes = Bytes.toBytes(i);
700        assertArrayEquals("Row should be " + i, r.getRow(), iBytes);
701        assertArrayEquals("Value should be " + i, r.getValue(A, A), iBytes);
702      }
703    } finally {
704      scanner.close();
705    }
706    table.close();
707  }
708
709  @Test
710  public void bulkLoadHFileTest() throws Exception {
711    final String testName =
712      TestRegionObserverInterface.class.getName() + "." + name.getMethodName();
713    final TableName tableName =
714      TableName.valueOf(TEST_TABLE.getNameAsString() + "." + name.getMethodName());
715    Configuration conf = util.getConfiguration();
716    Table table = util.createTable(tableName, new byte[][] { A, B, C });
717    try (RegionLocator locator = util.getConnection().getRegionLocator(tableName)) {
718      verifyMethodResult(SimpleRegionObserver.class,
719        new String[] { "hadPreBulkLoadHFile", "hadPostBulkLoadHFile" }, tableName,
720        new Boolean[] { false, false });
721
722      FileSystem fs = util.getTestFileSystem();
723      final Path dir = util.getDataTestDirOnTestFS(testName).makeQualified(fs);
724      Path familyDir = new Path(dir, Bytes.toString(A));
725
726      createHFile(util.getConfiguration(), fs, new Path(familyDir, Bytes.toString(A)), A, A);
727
728      // Bulk load
729      new LoadIncrementalHFiles(conf).doBulkLoad(dir, util.getAdmin(), table, locator);
730
731      verifyMethodResult(SimpleRegionObserver.class,
732        new String[] { "hadPreBulkLoadHFile", "hadPostBulkLoadHFile" }, tableName,
733        new Boolean[] { true, true });
734    } finally {
735      util.deleteTable(tableName);
736      table.close();
737    }
738  }
739
740  @Test
741  public void testRecovery() throws Exception {
742    LOG.info(TestRegionObserverInterface.class.getName() + "." + name.getMethodName());
743    final TableName tableName =
744      TableName.valueOf(TEST_TABLE.getNameAsString() + "." + name.getMethodName());
745    Table table = util.createTable(tableName, new byte[][] { A, B, C });
746    try (RegionLocator locator = util.getConnection().getRegionLocator(tableName)) {
747
748      JVMClusterUtil.RegionServerThread rs1 = cluster.startRegionServer();
749      ServerName sn2 = rs1.getRegionServer().getServerName();
750      String regEN = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName();
751
752      util.getAdmin().move(Bytes.toBytes(regEN), sn2);
753      while (!sn2.equals(locator.getAllRegionLocations().get(0).getServerName())) {
754        Thread.sleep(100);
755      }
756
757      Put put = new Put(ROW);
758      put.addColumn(A, A, A);
759      put.addColumn(B, B, B);
760      put.addColumn(C, C, C);
761      table.put(put);
762
763      // put two times
764      table.put(put);
765
766      verifyMethodResult(SimpleRegionObserver.class,
767        new String[] { "hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", "hadPreBatchMutate",
768          "hadPostBatchMutate", "hadDelete" },
769        tableName, new Boolean[] { false, false, true, true, true, true, false });
770
771      verifyMethodResult(SimpleRegionObserver.class,
772        new String[] { "getCtPreReplayWALs", "getCtPostReplayWALs", "getCtPreWALRestore",
773          "getCtPostWALRestore", "getCtPrePut", "getCtPostPut" },
774        tableName, new Integer[] { 0, 0, 0, 0, 2, 2 });
775
776      cluster.killRegionServer(rs1.getRegionServer().getServerName());
777      Threads.sleep(1000); // Let the kill soak in.
778      util.waitUntilAllRegionsAssigned(tableName);
779      LOG.info("All regions assigned");
780
781      verifyMethodResult(SimpleRegionObserver.class,
782        new String[] { "getCtPreReplayWALs", "getCtPostReplayWALs", "getCtPreWALRestore",
783          "getCtPostWALRestore", "getCtPrePut", "getCtPostPut" },
784        tableName, new Integer[] { 1, 1, 2, 2, 0, 0 });
785    } finally {
786      util.deleteTable(tableName);
787      table.close();
788    }
789  }
790
791  @Test
792  public void testPreWALRestoreSkip() throws Exception {
793    LOG.info(TestRegionObserverInterface.class.getName() + "." + name.getMethodName());
794    TableName tableName = TableName.valueOf(SimpleRegionObserver.TABLE_SKIPPED);
795    Table table = util.createTable(tableName, new byte[][] { A, B, C });
796
797    try (RegionLocator locator = util.getConnection().getRegionLocator(tableName)) {
798      JVMClusterUtil.RegionServerThread rs1 = cluster.startRegionServer();
799      ServerName sn2 = rs1.getRegionServer().getServerName();
800      String regEN = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName();
801
802      util.getAdmin().move(Bytes.toBytes(regEN), sn2);
803      while (!sn2.equals(locator.getAllRegionLocations().get(0).getServerName())) {
804        Thread.sleep(100);
805      }
806
807      Put put = new Put(ROW);
808      put.addColumn(A, A, A);
809      put.addColumn(B, B, B);
810      put.addColumn(C, C, C);
811      table.put(put);
812
813      cluster.killRegionServer(rs1.getRegionServer().getServerName());
814      Threads.sleep(20000); // just to be sure that the kill has fully started.
815      util.waitUntilAllRegionsAssigned(tableName);
816    }
817
818    verifyMethodResult(SimpleRegionObserver.class,
819      new String[] { "getCtPreWALRestore", "getCtPostWALRestore", }, tableName,
820      new Integer[] { 0, 0 });
821
822    util.deleteTable(tableName);
823    table.close();
824  }
825
826  // called from testPreWALAppendIsWrittenToWAL
827  private void testPreWALAppendHook(Table table, TableName tableName) throws IOException {
828    int expectedCalls = 0;
829    String[] methodArray = new String[1];
830    methodArray[0] = "getCtPreWALAppend";
831    Object[] resultArray = new Object[1];
832
833    Put p = new Put(ROW);
834    p.addColumn(A, A, A);
835    table.put(p);
836    resultArray[0] = ++expectedCalls;
837    verifyMethodResult(SimpleRegionObserver.class, methodArray, tableName, resultArray);
838
839    Append a = new Append(ROW);
840    a.addColumn(B, B, B);
841    table.append(a);
842    resultArray[0] = ++expectedCalls;
843    verifyMethodResult(SimpleRegionObserver.class, methodArray, tableName, resultArray);
844
845    Increment i = new Increment(ROW);
846    i.addColumn(C, C, 1);
847    table.increment(i);
848    resultArray[0] = ++expectedCalls;
849    verifyMethodResult(SimpleRegionObserver.class, methodArray, tableName, resultArray);
850
851    Delete d = new Delete(ROW);
852    table.delete(d);
853    resultArray[0] = ++expectedCalls;
854    verifyMethodResult(SimpleRegionObserver.class, methodArray, tableName, resultArray);
855  }
856
857  @Test
858  public void testPreWALAppend() throws Exception {
859    SimpleRegionObserver sro = new SimpleRegionObserver();
860    ObserverContext ctx = Mockito.mock(ObserverContext.class);
861    WALKey key =
862      new WALKeyImpl(Bytes.toBytes("region"), TEST_TABLE, EnvironmentEdgeManager.currentTime());
863    WALEdit edit = new WALEdit();
864    sro.preWALAppend(ctx, key, edit);
865    Assert.assertEquals(1, key.getExtendedAttributes().size());
866    Assert.assertArrayEquals(SimpleRegionObserver.WAL_EXTENDED_ATTRIBUTE_BYTES,
867      key.getExtendedAttribute(Integer.toString(sro.getCtPreWALAppend())));
868  }
869
870  @Test
871  public void testPreWALAppendIsWrittenToWAL() throws Exception {
872    final TableName tableName =
873      TableName.valueOf(TEST_TABLE.getNameAsString() + "." + name.getMethodName());
874    Table table = util.createTable(tableName, new byte[][] { A, B, C });
875
876    PreWALAppendWALActionsListener listener = new PreWALAppendWALActionsListener();
877    List<HRegion> regions = util.getHBaseCluster().getRegions(tableName);
878    // should be only one region
879    HRegion region = regions.get(0);
880    region.getWAL().registerWALActionsListener(listener);
881    testPreWALAppendHook(table, tableName);
882    boolean[] expectedResults = { true, true, true, true };
883    Assert.assertArrayEquals(expectedResults, listener.getWalKeysCorrectArray());
884
885  }
886
887  @Test
888  public void testPreWALAppendNotCalledOnMetaEdit() throws Exception {
889    final TableName tableName =
890      TableName.valueOf(TEST_TABLE.getNameAsString() + "." + name.getMethodName());
891    TableDescriptorBuilder tdBuilder = TableDescriptorBuilder.newBuilder(tableName);
892    ColumnFamilyDescriptorBuilder cfBuilder = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY);
893    tdBuilder.setColumnFamily(cfBuilder.build());
894    tdBuilder.setCoprocessor(SimpleRegionObserver.class.getName());
895    TableDescriptor td = tdBuilder.build();
896    Table table = util.createTable(td, new byte[][] { A, B, C });
897
898    PreWALAppendWALActionsListener listener = new PreWALAppendWALActionsListener();
899    List<HRegion> regions = util.getHBaseCluster().getRegions(tableName);
900    // should be only one region
901    HRegion region = regions.get(0);
902
903    region.getWAL().registerWALActionsListener(listener);
904    // flushing should write to the WAL
905    region.flush(true);
906    // so should compaction
907    region.compact(false);
908    // and so should closing the region
909    region.close();
910
911    // but we still shouldn't have triggered preWALAppend because no user data was written
912    String[] methods = new String[] { "getCtPreWALAppend" };
913    Object[] expectedResult = new Integer[] { 0 };
914    verifyMethodResult(SimpleRegionObserver.class, methods, tableName, expectedResult);
915  }
916
917  // check each region whether the coprocessor upcalls are called or not.
918  private void verifyMethodResult(Class<?> coprocessor, String methodName[], TableName tableName,
919    Object value[]) throws IOException {
920    try {
921      for (JVMClusterUtil.RegionServerThread t : cluster.getRegionServerThreads()) {
922        if (!t.isAlive() || t.getRegionServer().isAborted() || t.getRegionServer().isStopping()) {
923          continue;
924        }
925        for (RegionInfo r : ProtobufUtil.getOnlineRegions(t.getRegionServer().getRSRpcServices())) {
926          if (!r.getTable().equals(tableName)) {
927            continue;
928          }
929          RegionCoprocessorHost cph =
930            t.getRegionServer().getOnlineRegion(r.getRegionName()).getCoprocessorHost();
931
932          Coprocessor cp = cph.findCoprocessor(coprocessor.getName());
933          assertNotNull(cp);
934          for (int i = 0; i < methodName.length; ++i) {
935            Method m = coprocessor.getMethod(methodName[i]);
936            Object o = m.invoke(cp);
937            assertTrue("Result of " + coprocessor.getName() + "." + methodName[i]
938              + " is expected to be " + value[i].toString() + ", while we get " + o.toString(),
939              o.equals(value[i]));
940          }
941        }
942      }
943    } catch (Exception e) {
944      throw new IOException(e.toString());
945    }
946  }
947
948  private static void createHFile(Configuration conf, FileSystem fs, Path path, byte[] family,
949    byte[] qualifier) throws IOException {
950    HFileContext context = new HFileContextBuilder().build();
951    HFile.Writer writer = HFile.getWriterFactory(conf, new CacheConfig(conf)).withPath(fs, path)
952      .withFileContext(context).create();
953    long now = EnvironmentEdgeManager.currentTime();
954    try {
955      for (int i = 1; i <= 9; i++) {
956        KeyValue kv =
957          new KeyValue(Bytes.toBytes(i + ""), family, qualifier, now, Bytes.toBytes(i + ""));
958        writer.append(kv);
959      }
960    } finally {
961      writer.close();
962    }
963  }
964
965  private static class PreWALAppendWALActionsListener implements WALActionsListener {
966    boolean[] walKeysCorrect = { false, false, false, false };
967
968    @Override
969    public void postAppend(long entryLen, long elapsedTimeMillis, WALKey logKey, WALEdit logEdit)
970      throws IOException {
971      for (int k = 0; k < 4; k++) {
972        if (!walKeysCorrect[k]) {
973          walKeysCorrect[k] = Arrays.equals(SimpleRegionObserver.WAL_EXTENDED_ATTRIBUTE_BYTES,
974            logKey.getExtendedAttribute(Integer.toString(k + 1)));
975        }
976      }
977    }
978
979    boolean[] getWalKeysCorrectArray() {
980      return walKeysCorrect;
981    }
982  }
983}