001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.thrift;
019
020import static org.apache.hadoop.hbase.thrift.Constants.COALESCE_INC_KEY;
021import static org.junit.Assert.assertArrayEquals;
022import static org.junit.Assert.assertEquals;
023import static org.junit.Assert.assertFalse;
024import static org.junit.Assert.assertTrue;
025import static org.junit.Assert.fail;
026
027import java.io.IOException;
028import java.net.InetAddress;
029import java.nio.ByteBuffer;
030import java.util.ArrayList;
031import java.util.Collection;
032import java.util.HashMap;
033import java.util.List;
034import java.util.Map;
035import java.util.concurrent.atomic.AtomicInteger;
036import java.util.stream.Collectors;
037import org.apache.hadoop.conf.Configuration;
038import org.apache.hadoop.hbase.CompatibilityFactory;
039import org.apache.hadoop.hbase.HBaseClassTestRule;
040import org.apache.hadoop.hbase.HBaseTestingUtil;
041import org.apache.hadoop.hbase.HConstants;
042import org.apache.hadoop.hbase.TableName;
043import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
044import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
045import org.apache.hadoop.hbase.client.Put;
046import org.apache.hadoop.hbase.client.RegionInfo;
047import org.apache.hadoop.hbase.client.Table;
048import org.apache.hadoop.hbase.client.TableDescriptor;
049import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
050import org.apache.hadoop.hbase.filter.ParseFilter;
051import org.apache.hadoop.hbase.security.UserProvider;
052import org.apache.hadoop.hbase.test.MetricsAssertHelper;
053import org.apache.hadoop.hbase.testclassification.ClientTests;
054import org.apache.hadoop.hbase.testclassification.LargeTests;
055import org.apache.hadoop.hbase.thrift.ThriftMetrics.ThriftServerType;
056import org.apache.hadoop.hbase.thrift.generated.BatchMutation;
057import org.apache.hadoop.hbase.thrift.generated.ColumnDescriptor;
058import org.apache.hadoop.hbase.thrift.generated.Hbase;
059import org.apache.hadoop.hbase.thrift.generated.IOError;
060import org.apache.hadoop.hbase.thrift.generated.Mutation;
061import org.apache.hadoop.hbase.thrift.generated.TAppend;
062import org.apache.hadoop.hbase.thrift.generated.TCell;
063import org.apache.hadoop.hbase.thrift.generated.TIncrement;
064import org.apache.hadoop.hbase.thrift.generated.TRegionInfo;
065import org.apache.hadoop.hbase.thrift.generated.TRowResult;
066import org.apache.hadoop.hbase.thrift.generated.TScan;
067import org.apache.hadoop.hbase.thrift.generated.TThriftServerType;
068import org.apache.hadoop.hbase.util.Bytes;
069import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
070import org.apache.hadoop.hbase.util.TableDescriptorChecker;
071import org.apache.hadoop.hbase.util.Threads;
072import org.apache.thrift.protocol.TBinaryProtocol;
073import org.apache.thrift.protocol.TProtocol;
074import org.apache.thrift.transport.TSocket;
075import org.apache.thrift.transport.TTransport;
076import org.junit.AfterClass;
077import org.junit.BeforeClass;
078import org.junit.ClassRule;
079import org.junit.Rule;
080import org.junit.Test;
081import org.junit.experimental.categories.Category;
082import org.junit.rules.TestName;
083import org.slf4j.Logger;
084import org.slf4j.LoggerFactory;
085
086/**
087 * Unit testing for ThriftServerRunner.HBaseServiceHandler, a part of the
088 * org.apache.hadoop.hbase.thrift package.
089 */
090@Category({ClientTests.class, LargeTests.class})
091public class TestThriftServer {
092
093  @ClassRule
094  public static final HBaseClassTestRule CLASS_RULE =
095      HBaseClassTestRule.forClass(TestThriftServer.class);
096
097  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
098  private static final Logger LOG = LoggerFactory.getLogger(TestThriftServer.class);
099  private static final MetricsAssertHelper metricsHelper = CompatibilityFactory
100      .getInstance(MetricsAssertHelper.class);
101  protected static final int MAXVERSIONS = 3;
102
103  private static ByteBuffer asByteBuffer(String i) {
104    return ByteBuffer.wrap(Bytes.toBytes(i));
105  }
106  private static ByteBuffer asByteBuffer(long l) {
107    return ByteBuffer.wrap(Bytes.toBytes(l));
108  }
109
110  // Static names for tables, columns, rows, and values
111  private static ByteBuffer tableAname = asByteBuffer("tableA");
112  private static ByteBuffer tableBname = asByteBuffer("tableB");
113  private static ByteBuffer columnAname = asByteBuffer("columnA:");
114  private static ByteBuffer columnAAname = asByteBuffer("columnA:A");
115  private static ByteBuffer columnBname = asByteBuffer("columnB:");
116  private static ByteBuffer rowAname = asByteBuffer("rowA");
117  private static ByteBuffer rowBname = asByteBuffer("rowB");
118  private static ByteBuffer valueAname = asByteBuffer("valueA");
119  private static ByteBuffer valueBname = asByteBuffer("valueB");
120  private static ByteBuffer valueCname = asByteBuffer("valueC");
121  private static ByteBuffer valueDname = asByteBuffer("valueD");
122  private static ByteBuffer valueEname = asByteBuffer(100L);
123
124  @Rule
125  public TestName name = new TestName();
126
127  @BeforeClass
128  public static void beforeClass() throws Exception {
129    UTIL.getConfiguration().setBoolean(COALESCE_INC_KEY, true);
130    UTIL.getConfiguration().setBoolean(TableDescriptorChecker.TABLE_SANITY_CHECKS, false);
131    UTIL.getConfiguration().setInt("hbase.client.retries.number", 3);
132    UTIL.startMiniCluster();
133  }
134
135  @AfterClass
136  public static void afterClass() throws Exception {
137    UTIL.shutdownMiniCluster();
138  }
139
140  /**
141   * Runs all of the tests under a single JUnit test method.  We
142   * consolidate all testing to one method because HBaseClusterTestCase
143   * is prone to OutOfMemoryExceptions when there are three or more
144   * JUnit test methods.
145   */
146  @Test
147  public void testAll() throws Exception {
148    // Run all tests
149    doTestTableCreateDrop();
150    doTestThriftMetrics();
151    doTestTableMutations();
152    doTestTableTimestampsAndColumns();
153    doTestTableScanners();
154    doTestGetTableRegions();
155    doTestFilterRegistration();
156    doTestGetRegionInfo();
157    doTestIncrements();
158    doTestAppend();
159    doTestCheckAndPut();
160  }
161
162  /**
163   * Tests for creating, enabling, disabling, and deleting tables.  Also
164   * tests that creating a table with an invalid column name yields an
165   * IllegalArgument exception.
166   */
167  public void doTestTableCreateDrop() throws Exception {
168    ThriftHBaseServiceHandler handler =
169      new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
170        UserProvider.instantiate(UTIL.getConfiguration()));
171    doTestTableCreateDrop(handler);
172  }
173
174  public static void doTestTableCreateDrop(Hbase.Iface handler) throws Exception {
175    createTestTables(handler);
176    dropTestTables(handler);
177  }
178
179  public static final class MySlowHBaseHandler extends ThriftHBaseServiceHandler
180      implements Hbase.Iface {
181
182    protected MySlowHBaseHandler(Configuration c)
183        throws IOException {
184      super(c, UserProvider.instantiate(c));
185    }
186
187    @Override
188    public List<ByteBuffer> getTableNames() throws IOError {
189      Threads.sleepWithoutInterrupt(3000);
190      return super.getTableNames();
191    }
192  }
193
194  /**
195   * TODO: These counts are supposed to be zero but sometimes they are not, they are equal to the
196   * passed in maybe.  Investigate why.  My guess is they are set by the test that runs just
197   * previous to this one.  Sometimes they are cleared.  Sometimes not.
198   */
199  private int getCurrentCount(final String name, final int maybe, final ThriftMetrics metrics) {
200    int currentCount = 0;
201    try {
202      metricsHelper.assertCounter(name, maybe, metrics.getSource());
203      LOG.info("Shouldn't this be null? name=" + name + ", equals=" + maybe);
204      currentCount = maybe;
205    } catch (AssertionError e) {
206      // Ignore
207    }
208    return currentCount;
209  }
210
211  /**
212   * Tests if the metrics for thrift handler work correctly
213   */
214  public void doTestThriftMetrics() throws Exception {
215    LOG.info("START doTestThriftMetrics");
216    Configuration conf = UTIL.getConfiguration();
217    ThriftMetrics metrics = getMetrics(conf);
218    Hbase.Iface handler = getHandlerForMetricsTest(metrics, conf);
219    int currentCountCreateTable = getCurrentCount("createTable_num_ops", 2, metrics);
220    int currentCountDeleteTable = getCurrentCount("deleteTable_num_ops", 2, metrics);
221    int currentCountDisableTable = getCurrentCount("disableTable_num_ops", 2, metrics);
222    createTestTables(handler);
223    dropTestTables(handler);
224    metricsHelper.assertCounter("createTable_num_ops", currentCountCreateTable + 2,
225      metrics.getSource());
226    metricsHelper.assertCounter("deleteTable_num_ops", currentCountDeleteTable + 2,
227      metrics.getSource());
228    metricsHelper.assertCounter("disableTable_num_ops", currentCountDisableTable + 2,
229      metrics.getSource());
230    handler.getTableNames(); // This will have an artificial delay.
231
232    // 3 to 6 seconds (to account for potential slowness), measured in nanoseconds
233    try {
234      metricsHelper.assertGaugeGt("getTableNames_avg_time", 3L * 1000 * 1000 * 1000,
235              metrics.getSource());
236      metricsHelper.assertGaugeLt("getTableNames_avg_time",6L * 1000 * 1000 * 1000,
237              metrics.getSource());
238    } catch (AssertionError e) {
239      LOG.info("Fix me!  Why does this happen?  A concurrent cluster running?", e);
240    }
241  }
242
243  private static Hbase.Iface getHandlerForMetricsTest(ThriftMetrics metrics, Configuration conf)
244      throws Exception {
245    Hbase.Iface handler = new MySlowHBaseHandler(conf);
246    return HbaseHandlerMetricsProxy.newInstance((ThriftHBaseServiceHandler)handler, metrics, conf);
247  }
248
249  private static ThriftMetrics getMetrics(Configuration conf) throws Exception {
250    return new ThriftMetrics(conf, ThriftMetrics.ThriftServerType.ONE);
251  }
252
253
254  public static void createTestTables(Hbase.Iface handler) throws Exception {
255    // Create/enable/disable/delete tables, ensure methods act correctly
256    List<java.nio.ByteBuffer> bbs = handler.getTableNames();
257    assertEquals(bbs.stream().map(b -> Bytes.toString(b.array())).
258      collect(Collectors.joining(",")), 0, bbs.size());
259    handler.createTable(tableAname, getColumnDescriptors());
260    assertEquals(1, handler.getTableNames().size());
261    assertEquals(2, handler.getColumnDescriptors(tableAname).size());
262    assertTrue(handler.isTableEnabled(tableAname));
263    handler.createTable(tableBname, getColumnDescriptors());
264    assertEquals(2, handler.getTableNames().size());
265  }
266
267  public static void checkTableList(Hbase.Iface handler) throws Exception {
268    assertTrue(handler.getTableNames().contains(tableAname));
269  }
270
271  public static void dropTestTables(Hbase.Iface handler) throws Exception {
272    handler.disableTable(tableBname);
273    assertFalse(handler.isTableEnabled(tableBname));
274    handler.deleteTable(tableBname);
275    assertEquals(1, handler.getTableNames().size());
276    handler.disableTable(tableAname);
277    assertFalse(handler.isTableEnabled(tableAname));
278    /* TODO Reenable.
279    assertFalse(handler.isTableEnabled(tableAname));
280    handler.enableTable(tableAname);
281    assertTrue(handler.isTableEnabled(tableAname));
282    handler.disableTable(tableAname);*/
283    handler.deleteTable(tableAname);
284    assertEquals(0, handler.getTableNames().size());
285  }
286
287  public void doTestIncrements() throws Exception {
288    ThriftHBaseServiceHandler handler =
289      new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
290        UserProvider.instantiate(UTIL.getConfiguration()));
291    createTestTables(handler);
292    doTestIncrements(handler);
293    dropTestTables(handler);
294  }
295
296  public static void doTestIncrements(ThriftHBaseServiceHandler handler) throws Exception {
297    List<Mutation> mutations = new ArrayList<>(1);
298    mutations.add(new Mutation(false, columnAAname, valueEname, true));
299    mutations.add(new Mutation(false, columnAname, valueEname, true));
300    handler.mutateRow(tableAname, rowAname, mutations, null);
301    handler.mutateRow(tableAname, rowBname, mutations, null);
302
303    List<TIncrement> increments = new ArrayList<>(3);
304    increments.add(new TIncrement(tableAname, rowBname, columnAAname, 7));
305    increments.add(new TIncrement(tableAname, rowBname, columnAAname, 7));
306    increments.add(new TIncrement(tableAname, rowBname, columnAAname, 7));
307
308    int numIncrements = 60000;
309    for (int i = 0; i < numIncrements; i++) {
310      handler.increment(new TIncrement(tableAname, rowAname, columnAname, 2));
311      handler.incrementRows(increments);
312    }
313
314    Thread.sleep(1000);
315    long lv = handler.get(tableAname, rowAname, columnAname, null).get(0).value.getLong();
316    // Wait on all increments being flushed
317    while (handler.coalescer.getQueueSize() != 0) {
318      Threads.sleep(10);
319    }
320
321    assertEquals((100 + (2 * numIncrements)), lv);
322
323    lv = handler.get(tableAname, rowBname, columnAAname, null).get(0).value.getLong();
324    assertEquals((100 + (3 * 7 * numIncrements)), lv);
325
326    assertTrue(handler.coalescer.getSuccessfulCoalescings() > 0);
327
328  }
329
330  /**
331   * Tests adding a series of Mutations and BatchMutations, including a
332   * delete mutation.  Also tests data retrieval, and getting back multiple
333   * versions.
334   */
335  public void doTestTableMutations() throws Exception {
336    ThriftHBaseServiceHandler handler =
337      new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
338        UserProvider.instantiate(UTIL.getConfiguration()));
339    doTestTableMutations(handler);
340  }
341
342  public static void doTestTableMutations(Hbase.Iface handler) throws Exception {
343    // Setup
344    handler.createTable(tableAname, getColumnDescriptors());
345
346    // Apply a few Mutations to rowA
347    //     mutations.add(new Mutation(false, columnAname, valueAname));
348    //     mutations.add(new Mutation(false, columnBname, valueBname));
349    handler.mutateRow(tableAname, rowAname, getMutations(), null);
350
351    // Assert that the changes were made
352    assertEquals(valueAname,
353      handler.get(tableAname, rowAname, columnAname, null).get(0).value);
354    TRowResult rowResult1 = handler.getRow(tableAname, rowAname, null).get(0);
355    assertEquals(rowAname, rowResult1.row);
356    assertEquals(valueBname,
357      rowResult1.columns.get(columnBname).value);
358
359    // Apply a few BatchMutations for rowA and rowB
360    // rowAmutations.add(new Mutation(true, columnAname, null));
361    // rowAmutations.add(new Mutation(false, columnBname, valueCname));
362    // batchMutations.add(new BatchMutation(rowAname, rowAmutations));
363    // Mutations to rowB
364    // rowBmutations.add(new Mutation(false, columnAname, valueCname));
365    // rowBmutations.add(new Mutation(false, columnBname, valueDname));
366    // batchMutations.add(new BatchMutation(rowBname, rowBmutations));
367    handler.mutateRows(tableAname, getBatchMutations(), null);
368
369    // Assert that changes were made to rowA
370    List<TCell> cells = handler.get(tableAname, rowAname, columnAname, null);
371    assertFalse(cells.size() > 0);
372    assertEquals(valueCname, handler.get(tableAname, rowAname, columnBname, null).get(0).value);
373    List<TCell> versions = handler.getVer(tableAname, rowAname, columnBname, MAXVERSIONS, null);
374    assertEquals(valueCname, versions.get(0).value);
375    assertEquals(valueBname, versions.get(1).value);
376
377    // Assert that changes were made to rowB
378    TRowResult rowResult2 = handler.getRow(tableAname, rowBname, null).get(0);
379    assertEquals(rowBname, rowResult2.row);
380    assertEquals(valueCname, rowResult2.columns.get(columnAname).value);
381    assertEquals(valueDname, rowResult2.columns.get(columnBname).value);
382
383    // Apply some deletes
384    handler.deleteAll(tableAname, rowAname, columnBname, null);
385    handler.deleteAllRow(tableAname, rowBname, null);
386
387    // Assert that the deletes were applied
388    int size = handler.get(tableAname, rowAname, columnBname, null).size();
389    assertEquals(0, size);
390    size = handler.getRow(tableAname, rowBname, null).size();
391    assertEquals(0, size);
392
393    // Try null mutation
394    List<Mutation> mutations = new ArrayList<>(1);
395    mutations.add(new Mutation(false, columnAname, null, true));
396    handler.mutateRow(tableAname, rowAname, mutations, null);
397    TRowResult rowResult3 = handler.getRow(tableAname, rowAname, null).get(0);
398    assertEquals(rowAname, rowResult3.row);
399    assertEquals(0, rowResult3.columns.get(columnAname).value.remaining());
400
401    // Teardown
402    handler.disableTable(tableAname);
403    handler.deleteTable(tableAname);
404  }
405
406  /**
407   * Similar to testTableMutations(), except Mutations are applied with
408   * specific timestamps and data retrieval uses these timestamps to
409   * extract specific versions of data.
410   */
411  public void doTestTableTimestampsAndColumns() throws Exception {
412    // Setup
413    ThriftHBaseServiceHandler handler =
414      new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
415        UserProvider.instantiate(UTIL.getConfiguration()));
416    handler.createTable(tableAname, getColumnDescriptors());
417
418    // Apply timestamped Mutations to rowA
419    long time1 = EnvironmentEdgeManager.currentTime();
420    handler.mutateRowTs(tableAname, rowAname, getMutations(), time1, null);
421
422    Thread.sleep(1000);
423
424    // Apply timestamped BatchMutations for rowA and rowB
425    long time2 = EnvironmentEdgeManager.currentTime();
426    handler.mutateRowsTs(tableAname, getBatchMutations(), time2, null);
427
428    // Apply an overlapping timestamped mutation to rowB
429    handler.mutateRowTs(tableAname, rowBname, getMutations(), time2, null);
430
431    // the getVerTs is [inf, ts) so you need to increment one.
432    time1 += 1;
433    time2 += 2;
434
435    // Assert that the timestamp-related methods retrieve the correct data
436    assertEquals(2, handler.getVerTs(tableAname, rowAname, columnBname, time2,
437      MAXVERSIONS, null).size());
438    assertEquals(1, handler.getVerTs(tableAname, rowAname, columnBname, time1,
439      MAXVERSIONS, null).size());
440
441    TRowResult rowResult1 = handler.getRowTs(tableAname, rowAname, time1, null).get(0);
442    TRowResult rowResult2 = handler.getRowTs(tableAname, rowAname, time2, null).get(0);
443    // columnA was completely deleted
444    //assertTrue(Bytes.equals(rowResult1.columns.get(columnAname).value, valueAname));
445    assertEquals(rowResult1.columns.get(columnBname).value, valueBname);
446    assertEquals(rowResult2.columns.get(columnBname).value, valueCname);
447
448    // ColumnAname has been deleted, and will never be visible even with a getRowTs()
449    assertFalse(rowResult2.columns.containsKey(columnAname));
450
451    List<ByteBuffer> columns = new ArrayList<>(1);
452    columns.add(columnBname);
453
454    rowResult1 = handler.getRowWithColumns(tableAname, rowAname, columns, null).get(0);
455    assertEquals(rowResult1.columns.get(columnBname).value, valueCname);
456    assertFalse(rowResult1.columns.containsKey(columnAname));
457
458    rowResult1 = handler.getRowWithColumnsTs(tableAname, rowAname, columns, time1, null).get(0);
459    assertEquals(rowResult1.columns.get(columnBname).value, valueBname);
460    assertFalse(rowResult1.columns.containsKey(columnAname));
461
462    // Apply some timestamped deletes
463    // this actually deletes _everything_.
464    // nukes everything in columnB: forever.
465    handler.deleteAllTs(tableAname, rowAname, columnBname, time1, null);
466    handler.deleteAllRowTs(tableAname, rowBname, time2, null);
467
468    // Assert that the timestamp-related methods retrieve the correct data
469    int size = handler.getVerTs(tableAname, rowAname, columnBname, time1, MAXVERSIONS, null).size();
470    assertEquals(0, size);
471
472    size = handler.getVerTs(tableAname, rowAname, columnBname, time2, MAXVERSIONS, null).size();
473    assertEquals(1, size);
474
475    // should be available....
476    assertEquals(handler.get(tableAname, rowAname, columnBname, null).get(0).value, valueCname);
477
478    assertEquals(0, handler.getRow(tableAname, rowBname, null).size());
479
480    // Teardown
481    handler.disableTable(tableAname);
482    handler.deleteTable(tableAname);
483  }
484
485  /**
486   * Tests the four different scanner-opening methods (with and without
487   * a stoprow, with and without a timestamp).
488   */
489  public void doTestTableScanners() throws Exception {
490    // Setup
491    ThriftHBaseServiceHandler handler =
492      new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
493        UserProvider.instantiate(UTIL.getConfiguration()));
494    handler.createTable(tableAname, getColumnDescriptors());
495
496    // Apply timestamped Mutations to rowA
497    long time1 = EnvironmentEdgeManager.currentTime();
498    handler.mutateRowTs(tableAname, rowAname, getMutations(), time1, null);
499
500    // Sleep to assure that 'time1' and 'time2' will be different even with a
501    // coarse grained system timer.
502    Thread.sleep(1000);
503
504    // Apply timestamped BatchMutations for rowA and rowB
505    long time2 = EnvironmentEdgeManager.currentTime();
506    handler.mutateRowsTs(tableAname, getBatchMutations(), time2, null);
507
508    time1 += 1;
509
510    // Test a scanner on all rows and all columns, no timestamp
511    int scanner1 = handler.scannerOpen(tableAname, rowAname, getColumnList(true, true), null);
512    TRowResult rowResult1a = handler.scannerGet(scanner1).get(0);
513    assertEquals(rowResult1a.row, rowAname);
514    // This used to be '1'.  I don't know why when we are asking for two columns
515    // and when the mutations above would seem to add two columns to the row.
516    // -- St.Ack 05/12/2009
517    assertEquals(1, rowResult1a.columns.size());
518    assertEquals(rowResult1a.columns.get(columnBname).value, valueCname);
519
520    TRowResult rowResult1b = handler.scannerGet(scanner1).get(0);
521    assertEquals(rowResult1b.row, rowBname);
522    assertEquals(2, rowResult1b.columns.size());
523    assertEquals(rowResult1b.columns.get(columnAname).value, valueCname);
524    assertEquals(rowResult1b.columns.get(columnBname).value, valueDname);
525    closeScanner(scanner1, handler);
526
527    // Test a scanner on all rows and all columns, with timestamp
528    int scanner2 = handler.scannerOpenTs(tableAname, rowAname, getColumnList(true, true), time1,
529            null);
530    TRowResult rowResult2a = handler.scannerGet(scanner2).get(0);
531    assertEquals(1, rowResult2a.columns.size());
532    // column A deleted, does not exist.
533    //assertTrue(Bytes.equals(rowResult2a.columns.get(columnAname).value, valueAname));
534    assertEquals(rowResult2a.columns.get(columnBname).value, valueBname);
535    closeScanner(scanner2, handler);
536
537    // Test a scanner on the first row and first column only, no timestamp
538    int scanner3 = handler.scannerOpenWithStop(tableAname, rowAname, rowBname,
539        getColumnList(true, false), null);
540    closeScanner(scanner3, handler);
541
542    // Test a scanner on the first row and second column only, with timestamp
543    int scanner4 = handler.scannerOpenWithStopTs(tableAname, rowAname, rowBname,
544        getColumnList(false, true), time1, null);
545    TRowResult rowResult4a = handler.scannerGet(scanner4).get(0);
546    assertEquals(1, rowResult4a.columns.size());
547    assertEquals(rowResult4a.columns.get(columnBname).value, valueBname);
548
549    // Test scanner using a TScan object once with sortColumns False and once with sortColumns true
550    TScan scanNoSortColumns = new TScan();
551    scanNoSortColumns.setStartRow(rowAname);
552    scanNoSortColumns.setStopRow(rowBname);
553
554    int scanner5 = handler.scannerOpenWithScan(tableAname , scanNoSortColumns, null);
555    TRowResult rowResult5 = handler.scannerGet(scanner5).get(0);
556    assertEquals(1, rowResult5.columns.size());
557    assertEquals(rowResult5.columns.get(columnBname).value, valueCname);
558
559    TScan scanSortColumns = new TScan();
560    scanSortColumns.setStartRow(rowAname);
561    scanSortColumns.setStopRow(rowBname);
562    scanSortColumns = scanSortColumns.setSortColumns(true);
563
564    int scanner6 = handler.scannerOpenWithScan(tableAname ,scanSortColumns, null);
565    TRowResult rowResult6 = handler.scannerGet(scanner6).get(0);
566    assertEquals(1, rowResult6.sortedColumns.size());
567    assertEquals(rowResult6.sortedColumns.get(0).getCell().value, valueCname);
568
569    List<Mutation> rowBmutations = new ArrayList<>(20);
570    for (int i = 0; i < 20; i++) {
571      rowBmutations.add(new Mutation(false, asByteBuffer("columnA:" + i), valueCname, true));
572    }
573    ByteBuffer rowC = asByteBuffer("rowC");
574    handler.mutateRow(tableAname, rowC, rowBmutations, null);
575
576    TScan scanSortMultiColumns = new TScan();
577    scanSortMultiColumns.setStartRow(rowC);
578    scanSortMultiColumns = scanSortMultiColumns.setSortColumns(true);
579    int scanner7 = handler.scannerOpenWithScan(tableAname, scanSortMultiColumns, null);
580    TRowResult rowResult7 = handler.scannerGet(scanner7).get(0);
581
582    ByteBuffer smallerColumn = asByteBuffer("columnA:");
583    for (int i = 0; i < 20; i++) {
584      ByteBuffer currentColumn = rowResult7.sortedColumns.get(i).columnName;
585      assertTrue(Bytes.compareTo(smallerColumn.array(), currentColumn.array()) < 0);
586      smallerColumn = currentColumn;
587    }
588
589    TScan reversedScan = new TScan();
590    reversedScan.setReversed(true);
591    reversedScan.setStartRow(rowBname);
592    reversedScan.setStopRow(rowAname);
593
594    int scanner8 = handler.scannerOpenWithScan(tableAname , reversedScan, null);
595    List<TRowResult> results = handler.scannerGet(scanner8);
596    handler.scannerClose(scanner8);
597    assertEquals(1, results.size());
598    assertEquals(ByteBuffer.wrap(results.get(0).getRow()), rowBname);
599
600    // Teardown
601    handler.disableTable(tableAname);
602    handler.deleteTable(tableAname);
603  }
604
605  /**
606   * For HBASE-2556
607   * Tests for GetTableRegions
608   */
609  public void doTestGetTableRegions() throws Exception {
610    ThriftHBaseServiceHandler handler =
611      new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
612        UserProvider.instantiate(UTIL.getConfiguration()));
613    doTestGetTableRegions(handler);
614  }
615
616  public static void doTestGetTableRegions(Hbase.Iface handler)
617      throws Exception {
618    assertEquals(0, handler.getTableNames().size());
619    handler.createTable(tableAname, getColumnDescriptors());
620    assertEquals(1, handler.getTableNames().size());
621    List<TRegionInfo> regions = handler.getTableRegions(tableAname);
622    int regionCount = regions.size();
623    assertEquals("empty table should have only 1 region, " +
624            "but found " + regionCount, 1, regionCount);
625    LOG.info("Region found:" + regions.get(0));
626    handler.disableTable(tableAname);
627    handler.deleteTable(tableAname);
628    regionCount = handler.getTableRegions(tableAname).size();
629    assertEquals("non-existing table should have 0 region, " +
630            "but found " + regionCount, 0, regionCount);
631  }
632
633  public void doTestFilterRegistration() throws Exception {
634    Configuration conf = UTIL.getConfiguration();
635
636    conf.set("hbase.thrift.filters", "MyFilter:filterclass");
637
638    ThriftServer.registerFilters(conf);
639
640    Map<String, String> registeredFilters = ParseFilter.getAllFilters();
641
642    assertEquals("filterclass", registeredFilters.get("MyFilter"));
643  }
644
645  public void doTestGetRegionInfo() throws Exception {
646    ThriftHBaseServiceHandler handler =
647      new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
648        UserProvider.instantiate(UTIL.getConfiguration()));
649    doTestGetRegionInfo(handler);
650  }
651
652  public static void doTestGetRegionInfo(Hbase.Iface handler) throws Exception {
653    // Create tableA and add two columns to rowA
654    handler.createTable(tableAname, getColumnDescriptors());
655    try {
656      handler.mutateRow(tableAname, rowAname, getMutations(), null);
657      byte[] searchRow = RegionInfo.createRegionName(
658          TableName.valueOf(tableAname.array()), rowAname.array(),
659          HConstants.NINES, false);
660      TRegionInfo regionInfo = handler.getRegionInfo(ByteBuffer.wrap(searchRow));
661      assertTrue(Bytes.toStringBinary(regionInfo.getName()).startsWith(
662            Bytes.toStringBinary(tableAname)));
663    } finally {
664      handler.disableTable(tableAname);
665      handler.deleteTable(tableAname);
666    }
667  }
668
669  /**
670   * Appends the value to a cell and checks that the cell value is updated properly.
671   */
672  public static void doTestAppend() throws Exception {
673    ThriftHBaseServiceHandler handler =
674      new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
675        UserProvider.instantiate(UTIL.getConfiguration()));
676    handler.createTable(tableAname, getColumnDescriptors());
677    try {
678      List<Mutation> mutations = new ArrayList<>(1);
679      mutations.add(new Mutation(false, columnAname, valueAname, true));
680      handler.mutateRow(tableAname, rowAname, mutations, null);
681
682      List<ByteBuffer> columnList = new ArrayList<>(1);
683      columnList.add(columnAname);
684      List<ByteBuffer> valueList = new ArrayList<>(1);
685      valueList.add(valueBname);
686
687      TAppend append = new TAppend(tableAname, rowAname, columnList, valueList);
688      handler.append(append);
689
690      TRowResult rowResult = handler.getRow(tableAname, rowAname, null).get(0);
691      assertEquals(rowAname, rowResult.row);
692      assertArrayEquals(Bytes.add(valueAname.array(), valueBname.array()),
693        rowResult.columns.get(columnAname).value.array());
694    } finally {
695      handler.disableTable(tableAname);
696      handler.deleteTable(tableAname);
697    }
698  }
699
700  /**
701   * Check that checkAndPut fails if the cell does not exist, then put in the cell, then check that
702   * the checkAndPut succeeds.
703   */
704  public static void doTestCheckAndPut() throws Exception {
705    ThriftHBaseServiceHandler handler =
706      new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
707        UserProvider.instantiate(UTIL.getConfiguration()));
708    handler.createTable(tableAname, getColumnDescriptors());
709    try {
710      List<Mutation> mutations = new ArrayList<>(1);
711      mutations.add(new Mutation(false, columnAname, valueAname, true));
712      Mutation putB = (new Mutation(false, columnBname, valueBname, true));
713
714      assertFalse(handler.checkAndPut(tableAname, rowAname, columnAname, valueAname, putB, null));
715
716      handler.mutateRow(tableAname, rowAname, mutations, null);
717
718      assertTrue(handler.checkAndPut(tableAname, rowAname, columnAname, valueAname, putB, null));
719
720      TRowResult rowResult = handler.getRow(tableAname, rowAname, null).get(0);
721      assertEquals(rowAname, rowResult.row);
722      assertEquals(valueBname, rowResult.columns.get(columnBname).value);
723    } finally {
724      handler.disableTable(tableAname);
725      handler.deleteTable(tableAname);
726    }
727  }
728
729  @Test
730  public void testGetTableNamesWithStatus() throws Exception{
731    ThriftHBaseServiceHandler handler =
732      new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
733        UserProvider.instantiate(UTIL.getConfiguration()));
734
735    createTestTables(handler);
736
737    assertEquals(2, handler.getTableNamesWithIsTableEnabled().size());
738    assertEquals(2, countTablesByStatus(true, handler));
739    handler.disableTable(tableBname);
740    assertEquals(1, countTablesByStatus(true, handler));
741    assertEquals(1, countTablesByStatus(false, handler));
742    assertEquals(2, handler.getTableNamesWithIsTableEnabled().size());
743    handler.enableTable(tableBname);
744    assertEquals(2, countTablesByStatus(true, handler));
745
746    dropTestTables(handler);
747  }
748
749  private static int countTablesByStatus(Boolean isEnabled, Hbase.Iface handler) throws Exception {
750    AtomicInteger counter = new AtomicInteger(0);
751    handler.getTableNamesWithIsTableEnabled().forEach(
752      (table, tableStatus) -> {
753        if (tableStatus.equals(isEnabled)) counter.getAndIncrement();
754      });
755    return counter.get();
756  }
757
758  @Test
759  public void testMetricsWithException() throws Exception {
760    String rowkey = "row1";
761    String family = "f";
762    String col = "c";
763    // create a table which will throw exceptions for requests
764    final TableName tableName = TableName.valueOf(name.getMethodName());
765    try {
766      ColumnFamilyDescriptor columnFamilyDescriptor =
767        ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)).build();
768      TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
769        .setCoprocessor(ErrorThrowingGetObserver.class.getName())
770        .setColumnFamily(columnFamilyDescriptor).build();
771
772      Table table = UTIL.createTable(tableDescriptor, null);
773      long now = EnvironmentEdgeManager.currentTime();
774      table.put(new Put(Bytes.toBytes(rowkey))
775        .addColumn(Bytes.toBytes(family), Bytes.toBytes(col), now, Bytes.toBytes("val1")));
776
777      Configuration conf = UTIL.getConfiguration();
778      ThriftMetrics metrics = getMetrics(conf);
779      ThriftHBaseServiceHandler hbaseHandler =
780        new ThriftHBaseServiceHandler(UTIL.getConfiguration(), UserProvider.instantiate(UTIL.getConfiguration()));
781      Hbase.Iface handler = HbaseHandlerMetricsProxy.newInstance(hbaseHandler, metrics, conf);
782
783      ByteBuffer tTableName = asByteBuffer(tableName.getNameAsString());
784
785      // check metrics increment with a successful get
786      long preGetCounter = metricsHelper.checkCounterExists("getRow_num_ops", metrics.getSource()) ?
787        metricsHelper.getCounter("getRow_num_ops", metrics.getSource()) :
788        0;
789      List<TRowResult> tRowResult = handler.getRow(tTableName, asByteBuffer(rowkey), null);
790      assertEquals(1, tRowResult.size());
791      TRowResult tResult = tRowResult.get(0);
792
793      TCell expectedColumnValue = new TCell(asByteBuffer("val1"), now);
794
795      assertArrayEquals(Bytes.toBytes(rowkey), tResult.getRow());
796      Collection<TCell> returnedColumnValues = tResult.getColumns().values();
797      assertEquals(1, returnedColumnValues.size());
798      assertEquals(expectedColumnValue, returnedColumnValues.iterator().next());
799
800      metricsHelper.assertCounter("getRow_num_ops", preGetCounter + 1, metrics.getSource());
801
802      // check metrics increment when the get throws each exception type
803      for (ErrorThrowingGetObserver.ErrorType type : ErrorThrowingGetObserver.ErrorType.values()) {
804        testExceptionType(handler, metrics, tTableName, rowkey, type);
805      }
806    } finally {
807      UTIL.deleteTable(tableName);
808    }
809  }
810
811  private void testExceptionType(Hbase.Iface handler, ThriftMetrics metrics,
812                                 ByteBuffer tTableName, String rowkey,
813                                 ErrorThrowingGetObserver.ErrorType errorType) throws Exception {
814    long preGetCounter = metricsHelper.getCounter("getRow_num_ops", metrics.getSource());
815    String exceptionKey = errorType.getMetricName();
816    long preExceptionCounter = metricsHelper.checkCounterExists(exceptionKey, metrics.getSource()) ?
817        metricsHelper.getCounter(exceptionKey, metrics.getSource()) :
818        0;
819    Map<ByteBuffer, ByteBuffer> attributes = new HashMap<>();
820    attributes.put(asByteBuffer(ErrorThrowingGetObserver.SHOULD_ERROR_ATTRIBUTE),
821        asByteBuffer(errorType.name()));
822    try {
823      List<TRowResult> tRowResult = handler.getRow(tTableName, asByteBuffer(rowkey), attributes);
824      fail("Get with error attribute should have thrown an exception");
825    } catch (IOError e) {
826      LOG.info("Received exception: ", e);
827      metricsHelper.assertCounter("getRow_num_ops", preGetCounter + 1, metrics.getSource());
828      metricsHelper.assertCounter(exceptionKey, preExceptionCounter + 1, metrics.getSource());
829    }
830  }
831
832  /**
833   * @return a List of ColumnDescriptors for use in creating a table.  Has one
834   *         default ColumnDescriptor and one ColumnDescriptor with fewer versions
835   */
836  private static List<ColumnDescriptor> getColumnDescriptors() {
837    ArrayList<ColumnDescriptor> cDescriptors = new ArrayList<>(2);
838
839    // A default ColumnDescriptor
840    ColumnDescriptor cDescA = new ColumnDescriptor();
841    cDescA.name = columnAname;
842    cDescriptors.add(cDescA);
843
844    // A slightly customized ColumnDescriptor (only 2 versions)
845    ColumnDescriptor cDescB = new ColumnDescriptor(columnBname, 2, "NONE",
846        false, "NONE", 0, 0, false, -1);
847    cDescriptors.add(cDescB);
848
849    return cDescriptors;
850  }
851
852  /**
853   *
854   * @param includeA whether or not to include columnA
855   * @param includeB whether or not to include columnB
856   * @return a List of column names for use in retrieving a scanner
857   */
858  private List<ByteBuffer> getColumnList(boolean includeA, boolean includeB) {
859    List<ByteBuffer> columnList = new ArrayList<>();
860
861    if (includeA) {
862      columnList.add(columnAname);
863    }
864    if (includeB) {
865      columnList.add(columnBname);
866    }
867
868    return columnList;
869  }
870
871  /**
872   * @return a List of Mutations for a row, with columnA having valueA
873   *         and columnB having valueB
874   */
875  private static List<Mutation> getMutations() {
876    List<Mutation> mutations = new ArrayList<>(2);
877    mutations.add(new Mutation(false, columnAname, valueAname, true));
878    mutations.add(new Mutation(false, columnBname, valueBname, true));
879    return mutations;
880  }
881
882  /**
883   * @return a List of BatchMutations with the following effects:
884   *         (rowA, columnA): delete
885   *         (rowA, columnB): place valueC
886   *         (rowB, columnA): place valueC
887   *         (rowB, columnB): place valueD
888   */
889  private static List<BatchMutation> getBatchMutations() {
890    List<BatchMutation> batchMutations = new ArrayList<>(3);
891
892    // Mutations to rowA.  You can't mix delete and put anymore.
893    List<Mutation> rowAmutations = new ArrayList<>(1);
894    rowAmutations.add(new Mutation(true, columnAname, null, true));
895    batchMutations.add(new BatchMutation(rowAname, rowAmutations));
896
897    rowAmutations = new ArrayList<>(1);
898    rowAmutations.add(new Mutation(false, columnBname, valueCname, true));
899    batchMutations.add(new BatchMutation(rowAname, rowAmutations));
900
901    // Mutations to rowB
902    List<Mutation> rowBmutations = new ArrayList<>(2);
903    rowBmutations.add(new Mutation(false, columnAname, valueCname, true));
904    rowBmutations.add(new Mutation(false, columnBname, valueDname, true));
905    batchMutations.add(new BatchMutation(rowBname, rowBmutations));
906
907    return batchMutations;
908  }
909
910  /**
911   * Asserts that the passed scanner is exhausted, and then closes
912   * the scanner.
913   *
914   * @param scannerId the scanner to close
915   * @param handler the HBaseServiceHandler interfacing to HBase
916   */
917  private void closeScanner(
918      int scannerId, ThriftHBaseServiceHandler handler) throws Exception {
919    handler.scannerGet(scannerId);
920    handler.scannerClose(scannerId);
921  }
922
923  @Test
924  public void testGetThriftServerType() throws Exception {
925    ThriftHBaseServiceHandler handler =
926        new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
927            UserProvider.instantiate(UTIL.getConfiguration()));
928    assertEquals(TThriftServerType.ONE, handler.getThriftServerType());
929  }
930
931  /**
932   * Verify that thrift client calling thrift2 server can get the thrift2 server type correctly.
933   */
934  @Test
935  public void testGetThriftServerOneType() throws Exception {
936    // start a thrift2 server
937    HBaseThriftTestingUtility THRIFT_TEST_UTIL = new HBaseThriftTestingUtility();
938
939    LOG.info("Starting HBase Thrift Server Two");
940    THRIFT_TEST_UTIL.startThriftServer(UTIL.getConfiguration(), ThriftServerType.TWO);
941    try (TTransport transport = new TSocket(InetAddress.getLocalHost().getHostName(),
942        THRIFT_TEST_UTIL.getServerPort())){
943      TProtocol protocol = new TBinaryProtocol(transport);
944      // This is our thrift client.
945      Hbase.Client client = new Hbase.Client(protocol);
946      // open the transport
947      transport.open();
948      assertEquals(TThriftServerType.TWO.name(), client.getThriftServerType().name());
949    } finally {
950      THRIFT_TEST_UTIL.stopThriftServer();
951    }
952  }
953}