001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.thrift;
019
020import static org.apache.hadoop.hbase.thrift.Constants.COALESCE_INC_KEY;
021import static org.junit.Assert.assertArrayEquals;
022import static org.junit.Assert.assertEquals;
023import static org.junit.Assert.assertFalse;
024import static org.junit.Assert.assertTrue;
025import static org.junit.Assert.fail;
026
027import java.io.IOException;
028import java.net.InetAddress;
029import java.nio.ByteBuffer;
030import java.util.ArrayList;
031import java.util.Collection;
032import java.util.HashMap;
033import java.util.List;
034import java.util.Map;
035import java.util.concurrent.atomic.AtomicInteger;
036import java.util.stream.Collectors;
037import org.apache.hadoop.conf.Configuration;
038import org.apache.hadoop.hbase.CompatibilityFactory;
039import org.apache.hadoop.hbase.HBaseClassTestRule;
040import org.apache.hadoop.hbase.HBaseTestingUtil;
041import org.apache.hadoop.hbase.HConstants;
042import org.apache.hadoop.hbase.TableName;
043import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
044import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
045import org.apache.hadoop.hbase.client.Put;
046import org.apache.hadoop.hbase.client.RegionInfo;
047import org.apache.hadoop.hbase.client.Table;
048import org.apache.hadoop.hbase.client.TableDescriptor;
049import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
050import org.apache.hadoop.hbase.filter.ParseFilter;
051import org.apache.hadoop.hbase.security.UserProvider;
052import org.apache.hadoop.hbase.test.MetricsAssertHelper;
053import org.apache.hadoop.hbase.testclassification.ClientTests;
054import org.apache.hadoop.hbase.testclassification.LargeTests;
055import org.apache.hadoop.hbase.thrift.ThriftMetrics.ThriftServerType;
056import org.apache.hadoop.hbase.thrift.generated.BatchMutation;
057import org.apache.hadoop.hbase.thrift.generated.ColumnDescriptor;
058import org.apache.hadoop.hbase.thrift.generated.Hbase;
059import org.apache.hadoop.hbase.thrift.generated.IOError;
060import org.apache.hadoop.hbase.thrift.generated.Mutation;
061import org.apache.hadoop.hbase.thrift.generated.TAppend;
062import org.apache.hadoop.hbase.thrift.generated.TCell;
063import org.apache.hadoop.hbase.thrift.generated.TIncrement;
064import org.apache.hadoop.hbase.thrift.generated.TRegionInfo;
065import org.apache.hadoop.hbase.thrift.generated.TRowResult;
066import org.apache.hadoop.hbase.thrift.generated.TScan;
067import org.apache.hadoop.hbase.thrift.generated.TThriftServerType;
068import org.apache.hadoop.hbase.util.Bytes;
069import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
070import org.apache.hadoop.hbase.util.TableDescriptorChecker;
071import org.apache.hadoop.hbase.util.Threads;
072import org.apache.thrift.protocol.TBinaryProtocol;
073import org.apache.thrift.protocol.TProtocol;
074import org.apache.thrift.transport.TSocket;
075import org.apache.thrift.transport.TTransport;
076import org.junit.AfterClass;
077import org.junit.BeforeClass;
078import org.junit.ClassRule;
079import org.junit.Rule;
080import org.junit.Test;
081import org.junit.experimental.categories.Category;
082import org.junit.rules.TestName;
083import org.slf4j.Logger;
084import org.slf4j.LoggerFactory;
085
086/**
087 * Unit testing for ThriftServerRunner.HBaseServiceHandler, a part of the
088 * org.apache.hadoop.hbase.thrift package.
089 */
090@Category({ ClientTests.class, LargeTests.class })
091public class TestThriftServer {
092
093  @ClassRule
094  public static final HBaseClassTestRule CLASS_RULE =
095    HBaseClassTestRule.forClass(TestThriftServer.class);
096
097  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
098  private static final Logger LOG = LoggerFactory.getLogger(TestThriftServer.class);
099  private static final MetricsAssertHelper metricsHelper =
100    CompatibilityFactory.getInstance(MetricsAssertHelper.class);
101  protected static final int MAXVERSIONS = 3;
102
103  private static ByteBuffer asByteBuffer(String i) {
104    return ByteBuffer.wrap(Bytes.toBytes(i));
105  }
106
107  private static ByteBuffer asByteBuffer(long l) {
108    return ByteBuffer.wrap(Bytes.toBytes(l));
109  }
110
111  // Static names for tables, columns, rows, and values
112  private static ByteBuffer tableAname = asByteBuffer("tableA");
113  private static ByteBuffer tableBname = asByteBuffer("tableB");
114  private static ByteBuffer columnAname = asByteBuffer("columnA:");
115  private static ByteBuffer columnAAname = asByteBuffer("columnA:A");
116  private static ByteBuffer columnBname = asByteBuffer("columnB:");
117  private static ByteBuffer rowAname = asByteBuffer("rowA");
118  private static ByteBuffer rowBname = asByteBuffer("rowB");
119  private static ByteBuffer valueAname = asByteBuffer("valueA");
120  private static ByteBuffer valueBname = asByteBuffer("valueB");
121  private static ByteBuffer valueCname = asByteBuffer("valueC");
122  private static ByteBuffer valueDname = asByteBuffer("valueD");
123  private static ByteBuffer valueEname = asByteBuffer(100L);
124
125  @Rule
126  public TestName name = new TestName();
127
128  @BeforeClass
129  public static void beforeClass() throws Exception {
130    UTIL.getConfiguration().setBoolean(COALESCE_INC_KEY, true);
131    UTIL.getConfiguration().setBoolean(TableDescriptorChecker.TABLE_SANITY_CHECKS, false);
132    UTIL.getConfiguration().setInt("hbase.client.retries.number", 3);
133    UTIL.startMiniCluster();
134  }
135
136  @AfterClass
137  public static void afterClass() throws Exception {
138    UTIL.shutdownMiniCluster();
139  }
140
141  /**
142   * Runs all of the tests under a single JUnit test method. We consolidate all testing to one
143   * method because HBaseClusterTestCase is prone to OutOfMemoryExceptions when there are three or
144   * more JUnit test methods.
145   */
146  @Test
147  public void testAll() throws Exception {
148    // Run all tests
149    doTestTableCreateDrop();
150    doTestThriftMetrics();
151    doTestTableMutations();
152    doTestTableTimestampsAndColumns();
153    doTestTableScanners();
154    doTestGetTableRegions();
155    doTestFilterRegistration();
156    doTestGetRegionInfo();
157    doTestIncrements();
158    doTestAppend();
159    doTestCheckAndPut();
160  }
161
162  /**
163   * Tests for creating, enabling, disabling, and deleting tables. Also tests that creating a table
164   * with an invalid column name yields an IllegalArgument exception.
165   */
166  public void doTestTableCreateDrop() throws Exception {
167    ThriftHBaseServiceHandler handler = new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
168      UserProvider.instantiate(UTIL.getConfiguration()));
169    doTestTableCreateDrop(handler);
170  }
171
172  public static void doTestTableCreateDrop(Hbase.Iface handler) throws Exception {
173    createTestTables(handler);
174    dropTestTables(handler);
175  }
176
177  public static final class MySlowHBaseHandler extends ThriftHBaseServiceHandler
178    implements Hbase.Iface {
179
180    protected MySlowHBaseHandler(Configuration c) throws IOException {
181      super(c, UserProvider.instantiate(c));
182    }
183
184    @Override
185    public List<ByteBuffer> getTableNames() throws IOError {
186      Threads.sleepWithoutInterrupt(3000);
187      return super.getTableNames();
188    }
189  }
190
191  /**
192   * TODO: These counts are supposed to be zero but sometimes they are not, they are equal to the
193   * passed in maybe. Investigate why. My guess is they are set by the test that runs just previous
194   * to this one. Sometimes they are cleared. Sometimes not.
195   */
196  private int getCurrentCount(final String name, final int maybe, final ThriftMetrics metrics) {
197    int currentCount = 0;
198    try {
199      metricsHelper.assertCounter(name, maybe, metrics.getSource());
200      LOG.info("Shouldn't this be null? name=" + name + ", equals=" + maybe);
201      currentCount = maybe;
202    } catch (AssertionError e) {
203      // Ignore
204    }
205    return currentCount;
206  }
207
208  /**
209   * Tests if the metrics for thrift handler work correctly
210   */
211  public void doTestThriftMetrics() throws Exception {
212    LOG.info("START doTestThriftMetrics");
213    Configuration conf = UTIL.getConfiguration();
214    ThriftMetrics metrics = getMetrics(conf);
215    Hbase.Iface handler = getHandlerForMetricsTest(metrics, conf);
216    int currentCountCreateTable = getCurrentCount("createTable_num_ops", 2, metrics);
217    int currentCountDeleteTable = getCurrentCount("deleteTable_num_ops", 2, metrics);
218    int currentCountDisableTable = getCurrentCount("disableTable_num_ops", 2, metrics);
219    createTestTables(handler);
220    dropTestTables(handler);
221    metricsHelper.assertCounter("createTable_num_ops", currentCountCreateTable + 2,
222      metrics.getSource());
223    metricsHelper.assertCounter("deleteTable_num_ops", currentCountDeleteTable + 2,
224      metrics.getSource());
225    metricsHelper.assertCounter("disableTable_num_ops", currentCountDisableTable + 2,
226      metrics.getSource());
227    handler.getTableNames(); // This will have an artificial delay.
228
229    // 3 to 6 seconds (to account for potential slowness), measured in nanoseconds
230    try {
231      metricsHelper.assertGaugeGt("getTableNames_avg_time", 3L * 1000 * 1000 * 1000,
232        metrics.getSource());
233      metricsHelper.assertGaugeLt("getTableNames_avg_time", 6L * 1000 * 1000 * 1000,
234        metrics.getSource());
235    } catch (AssertionError e) {
236      LOG.info("Fix me!  Why does this happen?  A concurrent cluster running?", e);
237    }
238  }
239
240  private static Hbase.Iface getHandlerForMetricsTest(ThriftMetrics metrics, Configuration conf)
241    throws Exception {
242    Hbase.Iface handler = new MySlowHBaseHandler(conf);
243    return HbaseHandlerMetricsProxy.newInstance((ThriftHBaseServiceHandler) handler, metrics, conf);
244  }
245
246  private static ThriftMetrics getMetrics(Configuration conf) throws Exception {
247    return new ThriftMetrics(conf, ThriftMetrics.ThriftServerType.ONE);
248  }
249
250  public static void createTestTables(Hbase.Iface handler) throws Exception {
251    // Create/enable/disable/delete tables, ensure methods act correctly
252    List<java.nio.ByteBuffer> bbs = handler.getTableNames();
253    assertEquals(bbs.stream().map(b -> Bytes.toString(b.array())).collect(Collectors.joining(",")),
254      0, bbs.size());
255    handler.createTable(tableAname, getColumnDescriptors());
256    assertEquals(1, handler.getTableNames().size());
257    assertEquals(2, handler.getColumnDescriptors(tableAname).size());
258    assertTrue(handler.isTableEnabled(tableAname));
259    handler.createTable(tableBname, getColumnDescriptors());
260    assertEquals(2, handler.getTableNames().size());
261  }
262
263  public static void checkTableList(Hbase.Iface handler) throws Exception {
264    assertTrue(handler.getTableNames().contains(tableAname));
265  }
266
267  public static void dropTestTables(Hbase.Iface handler) throws Exception {
268    handler.disableTable(tableBname);
269    assertFalse(handler.isTableEnabled(tableBname));
270    handler.deleteTable(tableBname);
271    assertEquals(1, handler.getTableNames().size());
272    handler.disableTable(tableAname);
273    assertFalse(handler.isTableEnabled(tableAname));
274    /*
275     * TODO Reenable. assertFalse(handler.isTableEnabled(tableAname));
276     * handler.enableTable(tableAname); assertTrue(handler.isTableEnabled(tableAname));
277     * handler.disableTable(tableAname);
278     */
279    handler.deleteTable(tableAname);
280    assertEquals(0, handler.getTableNames().size());
281  }
282
283  public void doTestIncrements() throws Exception {
284    ThriftHBaseServiceHandler handler = new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
285      UserProvider.instantiate(UTIL.getConfiguration()));
286    createTestTables(handler);
287    doTestIncrements(handler);
288    dropTestTables(handler);
289  }
290
291  public static void doTestIncrements(ThriftHBaseServiceHandler handler) throws Exception {
292    List<Mutation> mutations = new ArrayList<>(1);
293    mutations.add(new Mutation(false, columnAAname, valueEname, true));
294    mutations.add(new Mutation(false, columnAname, valueEname, true));
295    handler.mutateRow(tableAname, rowAname, mutations, null);
296    handler.mutateRow(tableAname, rowBname, mutations, null);
297
298    List<TIncrement> increments = new ArrayList<>(3);
299    increments.add(new TIncrement(tableAname, rowBname, columnAAname, 7));
300    increments.add(new TIncrement(tableAname, rowBname, columnAAname, 7));
301    increments.add(new TIncrement(tableAname, rowBname, columnAAname, 7));
302
303    int numIncrements = 60000;
304    for (int i = 0; i < numIncrements; i++) {
305      handler.increment(new TIncrement(tableAname, rowAname, columnAname, 2));
306      handler.incrementRows(increments);
307    }
308
309    Thread.sleep(1000);
310    long lv = handler.get(tableAname, rowAname, columnAname, null).get(0).value.getLong();
311    // Wait on all increments being flushed
312    while (handler.coalescer.getQueueSize() != 0) {
313      Threads.sleep(10);
314    }
315
316    assertEquals((100 + (2 * numIncrements)), lv);
317
318    lv = handler.get(tableAname, rowBname, columnAAname, null).get(0).value.getLong();
319    assertEquals((100 + (3 * 7 * numIncrements)), lv);
320
321    assertTrue(handler.coalescer.getSuccessfulCoalescings() > 0);
322
323  }
324
325  /**
326   * Tests adding a series of Mutations and BatchMutations, including a delete mutation. Also tests
327   * data retrieval, and getting back multiple versions.
328   */
329  public void doTestTableMutations() throws Exception {
330    ThriftHBaseServiceHandler handler = new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
331      UserProvider.instantiate(UTIL.getConfiguration()));
332    doTestTableMutations(handler);
333  }
334
335  public static void doTestTableMutations(Hbase.Iface handler) throws Exception {
336    // Setup
337    handler.createTable(tableAname, getColumnDescriptors());
338
339    // Apply a few Mutations to rowA
340    // mutations.add(new Mutation(false, columnAname, valueAname));
341    // mutations.add(new Mutation(false, columnBname, valueBname));
342    handler.mutateRow(tableAname, rowAname, getMutations(), null);
343
344    // Assert that the changes were made
345    assertEquals(valueAname, handler.get(tableAname, rowAname, columnAname, null).get(0).value);
346    TRowResult rowResult1 = handler.getRow(tableAname, rowAname, null).get(0);
347    assertEquals(rowAname, rowResult1.row);
348    assertEquals(valueBname, rowResult1.columns.get(columnBname).value);
349
350    // Apply a few BatchMutations for rowA and rowB
351    // rowAmutations.add(new Mutation(true, columnAname, null));
352    // rowAmutations.add(new Mutation(false, columnBname, valueCname));
353    // batchMutations.add(new BatchMutation(rowAname, rowAmutations));
354    // Mutations to rowB
355    // rowBmutations.add(new Mutation(false, columnAname, valueCname));
356    // rowBmutations.add(new Mutation(false, columnBname, valueDname));
357    // batchMutations.add(new BatchMutation(rowBname, rowBmutations));
358    handler.mutateRows(tableAname, getBatchMutations(), null);
359
360    // Assert that changes were made to rowA
361    List<TCell> cells = handler.get(tableAname, rowAname, columnAname, null);
362    assertFalse(cells.size() > 0);
363    assertEquals(valueCname, handler.get(tableAname, rowAname, columnBname, null).get(0).value);
364    List<TCell> versions = handler.getVer(tableAname, rowAname, columnBname, MAXVERSIONS, null);
365    assertEquals(valueCname, versions.get(0).value);
366    assertEquals(valueBname, versions.get(1).value);
367
368    // Assert that changes were made to rowB
369    TRowResult rowResult2 = handler.getRow(tableAname, rowBname, null).get(0);
370    assertEquals(rowBname, rowResult2.row);
371    assertEquals(valueCname, rowResult2.columns.get(columnAname).value);
372    assertEquals(valueDname, rowResult2.columns.get(columnBname).value);
373
374    // Apply some deletes
375    handler.deleteAll(tableAname, rowAname, columnBname, null);
376    handler.deleteAllRow(tableAname, rowBname, null);
377
378    // Assert that the deletes were applied
379    int size = handler.get(tableAname, rowAname, columnBname, null).size();
380    assertEquals(0, size);
381    size = handler.getRow(tableAname, rowBname, null).size();
382    assertEquals(0, size);
383
384    // Try null mutation
385    List<Mutation> mutations = new ArrayList<>(1);
386    mutations.add(new Mutation(false, columnAname, null, true));
387    handler.mutateRow(tableAname, rowAname, mutations, null);
388    TRowResult rowResult3 = handler.getRow(tableAname, rowAname, null).get(0);
389    assertEquals(rowAname, rowResult3.row);
390    assertEquals(0, rowResult3.columns.get(columnAname).value.remaining());
391
392    // Teardown
393    handler.disableTable(tableAname);
394    handler.deleteTable(tableAname);
395  }
396
397  /**
398   * Similar to testTableMutations(), except Mutations are applied with specific timestamps and data
399   * retrieval uses these timestamps to extract specific versions of data.
400   */
401  public void doTestTableTimestampsAndColumns() throws Exception {
402    // Setup
403    ThriftHBaseServiceHandler handler = new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
404      UserProvider.instantiate(UTIL.getConfiguration()));
405    handler.createTable(tableAname, getColumnDescriptors());
406
407    // Apply timestamped Mutations to rowA
408    long time1 = EnvironmentEdgeManager.currentTime();
409    handler.mutateRowTs(tableAname, rowAname, getMutations(), time1, null);
410
411    Thread.sleep(1000);
412
413    // Apply timestamped BatchMutations for rowA and rowB
414    long time2 = EnvironmentEdgeManager.currentTime();
415    handler.mutateRowsTs(tableAname, getBatchMutations(), time2, null);
416
417    // Apply an overlapping timestamped mutation to rowB
418    handler.mutateRowTs(tableAname, rowBname, getMutations(), time2, null);
419
420    // the getVerTs is [inf, ts) so you need to increment one.
421    time1 += 1;
422    time2 += 2;
423
424    // Assert that the timestamp-related methods retrieve the correct data
425    assertEquals(2,
426      handler.getVerTs(tableAname, rowAname, columnBname, time2, MAXVERSIONS, null).size());
427    assertEquals(1,
428      handler.getVerTs(tableAname, rowAname, columnBname, time1, MAXVERSIONS, null).size());
429
430    TRowResult rowResult1 = handler.getRowTs(tableAname, rowAname, time1, null).get(0);
431    TRowResult rowResult2 = handler.getRowTs(tableAname, rowAname, time2, null).get(0);
432    // columnA was completely deleted
433    // assertTrue(Bytes.equals(rowResult1.columns.get(columnAname).value, valueAname));
434    assertEquals(rowResult1.columns.get(columnBname).value, valueBname);
435    assertEquals(rowResult2.columns.get(columnBname).value, valueCname);
436
437    // ColumnAname has been deleted, and will never be visible even with a getRowTs()
438    assertFalse(rowResult2.columns.containsKey(columnAname));
439
440    List<ByteBuffer> columns = new ArrayList<>(1);
441    columns.add(columnBname);
442
443    rowResult1 = handler.getRowWithColumns(tableAname, rowAname, columns, null).get(0);
444    assertEquals(rowResult1.columns.get(columnBname).value, valueCname);
445    assertFalse(rowResult1.columns.containsKey(columnAname));
446
447    rowResult1 = handler.getRowWithColumnsTs(tableAname, rowAname, columns, time1, null).get(0);
448    assertEquals(rowResult1.columns.get(columnBname).value, valueBname);
449    assertFalse(rowResult1.columns.containsKey(columnAname));
450
451    // Apply some timestamped deletes
452    // this actually deletes _everything_.
453    // nukes everything in columnB: forever.
454    handler.deleteAllTs(tableAname, rowAname, columnBname, time1, null);
455    handler.deleteAllRowTs(tableAname, rowBname, time2, null);
456
457    // Assert that the timestamp-related methods retrieve the correct data
458    int size = handler.getVerTs(tableAname, rowAname, columnBname, time1, MAXVERSIONS, null).size();
459    assertEquals(0, size);
460
461    size = handler.getVerTs(tableAname, rowAname, columnBname, time2, MAXVERSIONS, null).size();
462    assertEquals(1, size);
463
464    // should be available....
465    assertEquals(handler.get(tableAname, rowAname, columnBname, null).get(0).value, valueCname);
466
467    assertEquals(0, handler.getRow(tableAname, rowBname, null).size());
468
469    // Teardown
470    handler.disableTable(tableAname);
471    handler.deleteTable(tableAname);
472  }
473
474  /**
475   * Tests the four different scanner-opening methods (with and without a stoprow, with and without
476   * a timestamp).
477   */
478  public void doTestTableScanners() throws Exception {
479    // Setup
480    ThriftHBaseServiceHandler handler = new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
481      UserProvider.instantiate(UTIL.getConfiguration()));
482    handler.createTable(tableAname, getColumnDescriptors());
483
484    // Apply timestamped Mutations to rowA
485    long time1 = EnvironmentEdgeManager.currentTime();
486    handler.mutateRowTs(tableAname, rowAname, getMutations(), time1, null);
487
488    // Sleep to assure that 'time1' and 'time2' will be different even with a
489    // coarse grained system timer.
490    Thread.sleep(1000);
491
492    // Apply timestamped BatchMutations for rowA and rowB
493    long time2 = EnvironmentEdgeManager.currentTime();
494    handler.mutateRowsTs(tableAname, getBatchMutations(), time2, null);
495
496    time1 += 1;
497
498    // Test a scanner on all rows and all columns, no timestamp
499    int scanner1 = handler.scannerOpen(tableAname, rowAname, getColumnList(true, true), null);
500    TRowResult rowResult1a = handler.scannerGet(scanner1).get(0);
501    assertEquals(rowResult1a.row, rowAname);
502    // This used to be '1'. I don't know why when we are asking for two columns
503    // and when the mutations above would seem to add two columns to the row.
504    // -- St.Ack 05/12/2009
505    assertEquals(1, rowResult1a.columns.size());
506    assertEquals(rowResult1a.columns.get(columnBname).value, valueCname);
507
508    TRowResult rowResult1b = handler.scannerGet(scanner1).get(0);
509    assertEquals(rowResult1b.row, rowBname);
510    assertEquals(2, rowResult1b.columns.size());
511    assertEquals(rowResult1b.columns.get(columnAname).value, valueCname);
512    assertEquals(rowResult1b.columns.get(columnBname).value, valueDname);
513    closeScanner(scanner1, handler);
514
515    // Test a scanner on all rows and all columns, with timestamp
516    int scanner2 =
517      handler.scannerOpenTs(tableAname, rowAname, getColumnList(true, true), time1, null);
518    TRowResult rowResult2a = handler.scannerGet(scanner2).get(0);
519    assertEquals(1, rowResult2a.columns.size());
520    // column A deleted, does not exist.
521    // assertTrue(Bytes.equals(rowResult2a.columns.get(columnAname).value, valueAname));
522    assertEquals(rowResult2a.columns.get(columnBname).value, valueBname);
523    closeScanner(scanner2, handler);
524
525    // Test a scanner on the first row and first column only, no timestamp
526    int scanner3 =
527      handler.scannerOpenWithStop(tableAname, rowAname, rowBname, getColumnList(true, false), null);
528    closeScanner(scanner3, handler);
529
530    // Test a scanner on the first row and second column only, with timestamp
531    int scanner4 = handler.scannerOpenWithStopTs(tableAname, rowAname, rowBname,
532      getColumnList(false, true), time1, null);
533    TRowResult rowResult4a = handler.scannerGet(scanner4).get(0);
534    assertEquals(1, rowResult4a.columns.size());
535    assertEquals(rowResult4a.columns.get(columnBname).value, valueBname);
536
537    // Test scanner using a TScan object once with sortColumns False and once with sortColumns true
538    TScan scanNoSortColumns = new TScan();
539    scanNoSortColumns.setStartRow(rowAname);
540    scanNoSortColumns.setStopRow(rowBname);
541
542    int scanner5 = handler.scannerOpenWithScan(tableAname, scanNoSortColumns, null);
543    TRowResult rowResult5 = handler.scannerGet(scanner5).get(0);
544    assertEquals(1, rowResult5.columns.size());
545    assertEquals(rowResult5.columns.get(columnBname).value, valueCname);
546
547    TScan scanSortColumns = new TScan();
548    scanSortColumns.setStartRow(rowAname);
549    scanSortColumns.setStopRow(rowBname);
550    scanSortColumns = scanSortColumns.setSortColumns(true);
551
552    int scanner6 = handler.scannerOpenWithScan(tableAname, scanSortColumns, null);
553    TRowResult rowResult6 = handler.scannerGet(scanner6).get(0);
554    assertEquals(1, rowResult6.sortedColumns.size());
555    assertEquals(rowResult6.sortedColumns.get(0).getCell().value, valueCname);
556
557    List<Mutation> rowBmutations = new ArrayList<>(20);
558    for (int i = 0; i < 20; i++) {
559      rowBmutations.add(new Mutation(false, asByteBuffer("columnA:" + i), valueCname, true));
560    }
561    ByteBuffer rowC = asByteBuffer("rowC");
562    handler.mutateRow(tableAname, rowC, rowBmutations, null);
563
564    TScan scanSortMultiColumns = new TScan();
565    scanSortMultiColumns.setStartRow(rowC);
566    scanSortMultiColumns = scanSortMultiColumns.setSortColumns(true);
567    int scanner7 = handler.scannerOpenWithScan(tableAname, scanSortMultiColumns, null);
568    TRowResult rowResult7 = handler.scannerGet(scanner7).get(0);
569
570    ByteBuffer smallerColumn = asByteBuffer("columnA:");
571    for (int i = 0; i < 20; i++) {
572      ByteBuffer currentColumn = rowResult7.sortedColumns.get(i).columnName;
573      assertTrue(Bytes.compareTo(smallerColumn.array(), currentColumn.array()) < 0);
574      smallerColumn = currentColumn;
575    }
576
577    TScan reversedScan = new TScan();
578    reversedScan.setReversed(true);
579    reversedScan.setStartRow(rowBname);
580    reversedScan.setStopRow(rowAname);
581
582    int scanner8 = handler.scannerOpenWithScan(tableAname, reversedScan, null);
583    List<TRowResult> results = handler.scannerGet(scanner8);
584    handler.scannerClose(scanner8);
585    assertEquals(1, results.size());
586    assertEquals(ByteBuffer.wrap(results.get(0).getRow()), rowBname);
587
588    // Teardown
589    handler.disableTable(tableAname);
590    handler.deleteTable(tableAname);
591  }
592
593  /**
594   * For HBASE-2556 Tests for GetTableRegions
595   */
596  public void doTestGetTableRegions() throws Exception {
597    ThriftHBaseServiceHandler handler = new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
598      UserProvider.instantiate(UTIL.getConfiguration()));
599    doTestGetTableRegions(handler);
600  }
601
602  public static void doTestGetTableRegions(Hbase.Iface handler) throws Exception {
603    assertEquals(0, handler.getTableNames().size());
604    handler.createTable(tableAname, getColumnDescriptors());
605    assertEquals(1, handler.getTableNames().size());
606    List<TRegionInfo> regions = handler.getTableRegions(tableAname);
607    int regionCount = regions.size();
608    assertEquals("empty table should have only 1 region, " + "but found " + regionCount, 1,
609      regionCount);
610    LOG.info("Region found:" + regions.get(0));
611    handler.disableTable(tableAname);
612    handler.deleteTable(tableAname);
613    regionCount = handler.getTableRegions(tableAname).size();
614    assertEquals("non-existing table should have 0 region, " + "but found " + regionCount, 0,
615      regionCount);
616  }
617
618  public void doTestFilterRegistration() throws Exception {
619    Configuration conf = UTIL.getConfiguration();
620
621    conf.set("hbase.thrift.filters", "MyFilter:filterclass");
622
623    ThriftServer.registerFilters(conf);
624
625    Map<String, String> registeredFilters = ParseFilter.getAllFilters();
626
627    assertEquals("filterclass", registeredFilters.get("MyFilter"));
628  }
629
630  public void doTestGetRegionInfo() throws Exception {
631    ThriftHBaseServiceHandler handler = new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
632      UserProvider.instantiate(UTIL.getConfiguration()));
633    doTestGetRegionInfo(handler);
634  }
635
636  public static void doTestGetRegionInfo(Hbase.Iface handler) throws Exception {
637    // Create tableA and add two columns to rowA
638    handler.createTable(tableAname, getColumnDescriptors());
639    try {
640      handler.mutateRow(tableAname, rowAname, getMutations(), null);
641      byte[] searchRow = RegionInfo.createRegionName(TableName.valueOf(tableAname.array()),
642        rowAname.array(), HConstants.NINES, false);
643      TRegionInfo regionInfo = handler.getRegionInfo(ByteBuffer.wrap(searchRow));
644      assertTrue(
645        Bytes.toStringBinary(regionInfo.getName()).startsWith(Bytes.toStringBinary(tableAname)));
646    } finally {
647      handler.disableTable(tableAname);
648      handler.deleteTable(tableAname);
649    }
650  }
651
652  /**
653   * Appends the value to a cell and checks that the cell value is updated properly.
654   */
655  public static void doTestAppend() throws Exception {
656    ThriftHBaseServiceHandler handler = new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
657      UserProvider.instantiate(UTIL.getConfiguration()));
658    handler.createTable(tableAname, getColumnDescriptors());
659    try {
660      List<Mutation> mutations = new ArrayList<>(1);
661      mutations.add(new Mutation(false, columnAname, valueAname, true));
662      handler.mutateRow(tableAname, rowAname, mutations, null);
663
664      List<ByteBuffer> columnList = new ArrayList<>(1);
665      columnList.add(columnAname);
666      List<ByteBuffer> valueList = new ArrayList<>(1);
667      valueList.add(valueBname);
668
669      TAppend append = new TAppend(tableAname, rowAname, columnList, valueList);
670      handler.append(append);
671
672      TRowResult rowResult = handler.getRow(tableAname, rowAname, null).get(0);
673      assertEquals(rowAname, rowResult.row);
674      assertArrayEquals(Bytes.add(valueAname.array(), valueBname.array()),
675        rowResult.columns.get(columnAname).value.array());
676    } finally {
677      handler.disableTable(tableAname);
678      handler.deleteTable(tableAname);
679    }
680  }
681
682  /**
683   * Check that checkAndPut fails if the cell does not exist, then put in the cell, then check that
684   * the checkAndPut succeeds.
685   */
686  public static void doTestCheckAndPut() throws Exception {
687    ThriftHBaseServiceHandler handler = new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
688      UserProvider.instantiate(UTIL.getConfiguration()));
689    handler.createTable(tableAname, getColumnDescriptors());
690    try {
691      List<Mutation> mutations = new ArrayList<>(1);
692      mutations.add(new Mutation(false, columnAname, valueAname, true));
693      Mutation putB = (new Mutation(false, columnBname, valueBname, true));
694
695      assertFalse(handler.checkAndPut(tableAname, rowAname, columnAname, valueAname, putB, null));
696
697      handler.mutateRow(tableAname, rowAname, mutations, null);
698
699      assertTrue(handler.checkAndPut(tableAname, rowAname, columnAname, valueAname, putB, null));
700
701      TRowResult rowResult = handler.getRow(tableAname, rowAname, null).get(0);
702      assertEquals(rowAname, rowResult.row);
703      assertEquals(valueBname, rowResult.columns.get(columnBname).value);
704    } finally {
705      handler.disableTable(tableAname);
706      handler.deleteTable(tableAname);
707    }
708  }
709
710  @Test
711  public void testGetTableNamesWithStatus() throws Exception {
712    ThriftHBaseServiceHandler handler = new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
713      UserProvider.instantiate(UTIL.getConfiguration()));
714
715    createTestTables(handler);
716
717    assertEquals(2, handler.getTableNamesWithIsTableEnabled().size());
718    assertEquals(2, countTablesByStatus(true, handler));
719    handler.disableTable(tableBname);
720    assertEquals(1, countTablesByStatus(true, handler));
721    assertEquals(1, countTablesByStatus(false, handler));
722    assertEquals(2, handler.getTableNamesWithIsTableEnabled().size());
723    handler.enableTable(tableBname);
724    assertEquals(2, countTablesByStatus(true, handler));
725
726    dropTestTables(handler);
727  }
728
729  private static int countTablesByStatus(Boolean isEnabled, Hbase.Iface handler) throws Exception {
730    AtomicInteger counter = new AtomicInteger(0);
731    handler.getTableNamesWithIsTableEnabled().forEach((table, tableStatus) -> {
732      if (tableStatus.equals(isEnabled)) counter.getAndIncrement();
733    });
734    return counter.get();
735  }
736
737  @Test
738  public void testMetricsWithException() throws Exception {
739    String rowkey = "row1";
740    String family = "f";
741    String col = "c";
742    // create a table which will throw exceptions for requests
743    final TableName tableName = TableName.valueOf(name.getMethodName());
744    try {
745      ColumnFamilyDescriptor columnFamilyDescriptor =
746        ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)).build();
747      TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
748        .setCoprocessor(ErrorThrowingGetObserver.class.getName())
749        .setColumnFamily(columnFamilyDescriptor).build();
750
751      Table table = UTIL.createTable(tableDescriptor, null);
752      long now = EnvironmentEdgeManager.currentTime();
753      table.put(new Put(Bytes.toBytes(rowkey)).addColumn(Bytes.toBytes(family), Bytes.toBytes(col),
754        now, Bytes.toBytes("val1")));
755
756      Configuration conf = UTIL.getConfiguration();
757      ThriftMetrics metrics = getMetrics(conf);
758      ThriftHBaseServiceHandler hbaseHandler = new ThriftHBaseServiceHandler(
759        UTIL.getConfiguration(), UserProvider.instantiate(UTIL.getConfiguration()));
760      Hbase.Iface handler = HbaseHandlerMetricsProxy.newInstance(hbaseHandler, metrics, conf);
761
762      ByteBuffer tTableName = asByteBuffer(tableName.getNameAsString());
763
764      // check metrics increment with a successful get
765      long preGetCounter = metricsHelper.checkCounterExists("getRow_num_ops", metrics.getSource())
766        ? metricsHelper.getCounter("getRow_num_ops", metrics.getSource())
767        : 0;
768      List<TRowResult> tRowResult = handler.getRow(tTableName, asByteBuffer(rowkey), null);
769      assertEquals(1, tRowResult.size());
770      TRowResult tResult = tRowResult.get(0);
771
772      TCell expectedColumnValue = new TCell(asByteBuffer("val1"), now);
773
774      assertArrayEquals(Bytes.toBytes(rowkey), tResult.getRow());
775      Collection<TCell> returnedColumnValues = tResult.getColumns().values();
776      assertEquals(1, returnedColumnValues.size());
777      assertEquals(expectedColumnValue, returnedColumnValues.iterator().next());
778
779      metricsHelper.assertCounter("getRow_num_ops", preGetCounter + 1, metrics.getSource());
780
781      // check metrics increment when the get throws each exception type
782      for (ErrorThrowingGetObserver.ErrorType type : ErrorThrowingGetObserver.ErrorType.values()) {
783        testExceptionType(handler, metrics, tTableName, rowkey, type);
784      }
785    } finally {
786      UTIL.deleteTable(tableName);
787    }
788  }
789
790  private void testExceptionType(Hbase.Iface handler, ThriftMetrics metrics, ByteBuffer tTableName,
791    String rowkey, ErrorThrowingGetObserver.ErrorType errorType) throws Exception {
792    long preGetCounter = metricsHelper.getCounter("getRow_num_ops", metrics.getSource());
793    String exceptionKey = errorType.getMetricName();
794    long preExceptionCounter = metricsHelper.checkCounterExists(exceptionKey, metrics.getSource())
795      ? metricsHelper.getCounter(exceptionKey, metrics.getSource())
796      : 0;
797    Map<ByteBuffer, ByteBuffer> attributes = new HashMap<>();
798    attributes.put(asByteBuffer(ErrorThrowingGetObserver.SHOULD_ERROR_ATTRIBUTE),
799      asByteBuffer(errorType.name()));
800    try {
801      List<TRowResult> tRowResult = handler.getRow(tTableName, asByteBuffer(rowkey), attributes);
802      fail("Get with error attribute should have thrown an exception");
803    } catch (IOError e) {
804      LOG.info("Received exception: ", e);
805      metricsHelper.assertCounter("getRow_num_ops", preGetCounter + 1, metrics.getSource());
806      metricsHelper.assertCounter(exceptionKey, preExceptionCounter + 1, metrics.getSource());
807    }
808  }
809
810  /**
811   * @return a List of ColumnDescriptors for use in creating a table. Has one default
812   *         ColumnDescriptor and one ColumnDescriptor with fewer versions
813   */
814  private static List<ColumnDescriptor> getColumnDescriptors() {
815    ArrayList<ColumnDescriptor> cDescriptors = new ArrayList<>(2);
816
817    // A default ColumnDescriptor
818    ColumnDescriptor cDescA = new ColumnDescriptor();
819    cDescA.name = columnAname;
820    cDescriptors.add(cDescA);
821
822    // A slightly customized ColumnDescriptor (only 2 versions)
823    ColumnDescriptor cDescB =
824      new ColumnDescriptor(columnBname, 2, "NONE", false, "NONE", 0, 0, false, -1);
825    cDescriptors.add(cDescB);
826
827    return cDescriptors;
828  }
829
830  /**
831   * @param includeA whether or not to include columnA
832   * @param includeB whether or not to include columnB
833   * @return a List of column names for use in retrieving a scanner
834   */
835  private List<ByteBuffer> getColumnList(boolean includeA, boolean includeB) {
836    List<ByteBuffer> columnList = new ArrayList<>();
837
838    if (includeA) {
839      columnList.add(columnAname);
840    }
841    if (includeB) {
842      columnList.add(columnBname);
843    }
844
845    return columnList;
846  }
847
848  /**
849   * @return a List of Mutations for a row, with columnA having valueA and columnB having valueB
850   */
851  private static List<Mutation> getMutations() {
852    List<Mutation> mutations = new ArrayList<>(2);
853    mutations.add(new Mutation(false, columnAname, valueAname, true));
854    mutations.add(new Mutation(false, columnBname, valueBname, true));
855    return mutations;
856  }
857
858  /**
859   * @return a List of BatchMutations with the following effects: (rowA, columnA): delete (rowA,
860   *         columnB): place valueC (rowB, columnA): place valueC (rowB, columnB): place valueD
861   */
862  private static List<BatchMutation> getBatchMutations() {
863    List<BatchMutation> batchMutations = new ArrayList<>(3);
864
865    // Mutations to rowA. You can't mix delete and put anymore.
866    List<Mutation> rowAmutations = new ArrayList<>(1);
867    rowAmutations.add(new Mutation(true, columnAname, null, true));
868    batchMutations.add(new BatchMutation(rowAname, rowAmutations));
869
870    rowAmutations = new ArrayList<>(1);
871    rowAmutations.add(new Mutation(false, columnBname, valueCname, true));
872    batchMutations.add(new BatchMutation(rowAname, rowAmutations));
873
874    // Mutations to rowB
875    List<Mutation> rowBmutations = new ArrayList<>(2);
876    rowBmutations.add(new Mutation(false, columnAname, valueCname, true));
877    rowBmutations.add(new Mutation(false, columnBname, valueDname, true));
878    batchMutations.add(new BatchMutation(rowBname, rowBmutations));
879
880    return batchMutations;
881  }
882
883  /**
884   * Asserts that the passed scanner is exhausted, and then closes the scanner.
885   * @param scannerId the scanner to close
886   * @param handler   the HBaseServiceHandler interfacing to HBase
887   */
888  private void closeScanner(int scannerId, ThriftHBaseServiceHandler handler) throws Exception {
889    handler.scannerGet(scannerId);
890    handler.scannerClose(scannerId);
891  }
892
893  @Test
894  public void testGetThriftServerType() throws Exception {
895    ThriftHBaseServiceHandler handler = new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
896      UserProvider.instantiate(UTIL.getConfiguration()));
897    assertEquals(TThriftServerType.ONE, handler.getThriftServerType());
898  }
899
900  /**
901   * Verify that thrift client calling thrift2 server can get the thrift2 server type correctly.
902   */
903  @Test
904  public void testGetThriftServerOneType() throws Exception {
905    // start a thrift2 server
906    HBaseThriftTestingUtility THRIFT_TEST_UTIL = new HBaseThriftTestingUtility();
907
908    LOG.info("Starting HBase Thrift Server Two");
909    THRIFT_TEST_UTIL.startThriftServer(UTIL.getConfiguration(), ThriftServerType.TWO);
910    try (TTransport transport =
911      new TSocket(InetAddress.getLocalHost().getHostName(), THRIFT_TEST_UTIL.getServerPort())) {
912      TProtocol protocol = new TBinaryProtocol(transport);
913      // This is our thrift client.
914      Hbase.Client client = new Hbase.Client(protocol);
915      // open the transport
916      transport.open();
917      assertEquals(TThriftServerType.TWO.name(), client.getThriftServerType().name());
918    } finally {
919      THRIFT_TEST_UTIL.stopThriftServer();
920    }
921  }
922}