001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.thrift;
019
020import static org.apache.hadoop.hbase.thrift.Constants.COALESCE_INC_KEY;
021import static org.junit.Assert.assertArrayEquals;
022import static org.junit.Assert.assertEquals;
023import static org.junit.Assert.assertFalse;
024import static org.junit.Assert.assertTrue;
025import static org.junit.Assert.fail;
026
027import java.io.IOException;
028import java.nio.ByteBuffer;
029import java.util.ArrayList;
030import java.util.Collection;
031import java.util.HashMap;
032import java.util.List;
033import java.util.Map;
034import org.apache.hadoop.conf.Configuration;
035import org.apache.hadoop.hbase.CompatibilityFactory;
036import org.apache.hadoop.hbase.HBaseClassTestRule;
037import org.apache.hadoop.hbase.HBaseTestingUtility;
038import org.apache.hadoop.hbase.HColumnDescriptor;
039import org.apache.hadoop.hbase.HConstants;
040import org.apache.hadoop.hbase.HRegionInfo;
041import org.apache.hadoop.hbase.HTableDescriptor;
042import org.apache.hadoop.hbase.TableName;
043import org.apache.hadoop.hbase.client.Put;
044import org.apache.hadoop.hbase.client.Table;
045import org.apache.hadoop.hbase.filter.ParseFilter;
046import org.apache.hadoop.hbase.security.UserProvider;
047import org.apache.hadoop.hbase.test.MetricsAssertHelper;
048import org.apache.hadoop.hbase.testclassification.ClientTests;
049import org.apache.hadoop.hbase.testclassification.LargeTests;
050import org.apache.hadoop.hbase.thrift.generated.BatchMutation;
051import org.apache.hadoop.hbase.thrift.generated.ColumnDescriptor;
052import org.apache.hadoop.hbase.thrift.generated.Hbase;
053import org.apache.hadoop.hbase.thrift.generated.IOError;
054import org.apache.hadoop.hbase.thrift.generated.Mutation;
055import org.apache.hadoop.hbase.thrift.generated.TAppend;
056import org.apache.hadoop.hbase.thrift.generated.TCell;
057import org.apache.hadoop.hbase.thrift.generated.TIncrement;
058import org.apache.hadoop.hbase.thrift.generated.TRegionInfo;
059import org.apache.hadoop.hbase.thrift.generated.TRowResult;
060import org.apache.hadoop.hbase.thrift.generated.TScan;
061import org.apache.hadoop.hbase.util.Bytes;
062import org.apache.hadoop.hbase.util.TableDescriptorChecker;
063import org.apache.hadoop.hbase.util.Threads;
064import org.junit.AfterClass;
065import org.junit.BeforeClass;
066import org.junit.ClassRule;
067import org.junit.Rule;
068import org.junit.Test;
069import org.junit.experimental.categories.Category;
070import org.junit.rules.TestName;
071import org.slf4j.Logger;
072import org.slf4j.LoggerFactory;
073
074/**
075 * Unit testing for ThriftServerRunner.HBaseServiceHandler, a part of the
076 * org.apache.hadoop.hbase.thrift package.
077 */
078@Category({ClientTests.class, LargeTests.class})
079public class TestThriftServer {
080
081  @ClassRule
082  public static final HBaseClassTestRule CLASS_RULE =
083      HBaseClassTestRule.forClass(TestThriftServer.class);
084
085  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
086  private static final Logger LOG = LoggerFactory.getLogger(TestThriftServer.class);
087  private static final MetricsAssertHelper metricsHelper = CompatibilityFactory
088      .getInstance(MetricsAssertHelper.class);
089  protected static final int MAXVERSIONS = 3;
090
091  private static ByteBuffer asByteBuffer(String i) {
092    return ByteBuffer.wrap(Bytes.toBytes(i));
093  }
094  private static ByteBuffer asByteBuffer(long l) {
095    return ByteBuffer.wrap(Bytes.toBytes(l));
096  }
097
098  // Static names for tables, columns, rows, and values
099  private static ByteBuffer tableAname = asByteBuffer("tableA");
100  private static ByteBuffer tableBname = asByteBuffer("tableB");
101  private static ByteBuffer columnAname = asByteBuffer("columnA:");
102  private static ByteBuffer columnAAname = asByteBuffer("columnA:A");
103  private static ByteBuffer columnBname = asByteBuffer("columnB:");
104  private static ByteBuffer rowAname = asByteBuffer("rowA");
105  private static ByteBuffer rowBname = asByteBuffer("rowB");
106  private static ByteBuffer valueAname = asByteBuffer("valueA");
107  private static ByteBuffer valueBname = asByteBuffer("valueB");
108  private static ByteBuffer valueCname = asByteBuffer("valueC");
109  private static ByteBuffer valueDname = asByteBuffer("valueD");
110  private static ByteBuffer valueEname = asByteBuffer(100L);
111
112  @Rule
113  public TestName name = new TestName();
114
115  @BeforeClass
116  public static void beforeClass() throws Exception {
117    UTIL.getConfiguration().setBoolean(COALESCE_INC_KEY, true);
118    UTIL.getConfiguration().setBoolean(TableDescriptorChecker.TABLE_SANITY_CHECKS, false);
119    UTIL.getConfiguration().setInt("hbase.client.retries.number", 3);
120    UTIL.startMiniCluster();
121  }
122
123  @AfterClass
124  public static void afterClass() throws Exception {
125    UTIL.shutdownMiniCluster();
126  }
127
128  /**
129   * Runs all of the tests under a single JUnit test method.  We
130   * consolidate all testing to one method because HBaseClusterTestCase
131   * is prone to OutOfMemoryExceptions when there are three or more
132   * JUnit test methods.
133   */
134  @Test
135  public void testAll() throws Exception {
136    // Run all tests
137    doTestTableCreateDrop();
138    doTestThriftMetrics();
139    doTestTableMutations();
140    doTestTableTimestampsAndColumns();
141    doTestTableScanners();
142    doTestGetTableRegions();
143    doTestFilterRegistration();
144    doTestGetRegionInfo();
145    doTestIncrements();
146    doTestAppend();
147    doTestCheckAndPut();
148  }
149
150  /**
151   * Tests for creating, enabling, disabling, and deleting tables.  Also
152   * tests that creating a table with an invalid column name yields an
153   * IllegalArgument exception.
154   */
155  public void doTestTableCreateDrop() throws Exception {
156    ThriftHBaseServiceHandler handler =
157      new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
158        UserProvider.instantiate(UTIL.getConfiguration()));
159    doTestTableCreateDrop(handler);
160  }
161
162  public static void doTestTableCreateDrop(Hbase.Iface handler) throws Exception {
163    createTestTables(handler);
164    dropTestTables(handler);
165  }
166
167  public static final class MySlowHBaseHandler extends ThriftHBaseServiceHandler
168      implements Hbase.Iface {
169
170    protected MySlowHBaseHandler(Configuration c)
171        throws IOException {
172      super(c, UserProvider.instantiate(c));
173    }
174
175    @Override
176    public List<ByteBuffer> getTableNames() throws IOError {
177      Threads.sleepWithoutInterrupt(3000);
178      return super.getTableNames();
179    }
180  }
181
182  /**
183   * TODO: These counts are supposed to be zero but sometimes they are not, they are equal to the
184   * passed in maybe.  Investigate why.  My guess is they are set by the test that runs just
185   * previous to this one.  Sometimes they are cleared.  Sometimes not.
186   */
187  private int getCurrentCount(final String name, final int maybe, final ThriftMetrics metrics) {
188    int currentCount = 0;
189    try {
190      metricsHelper.assertCounter(name, maybe, metrics.getSource());
191      LOG.info("Shouldn't this be null? name=" + name + ", equals=" + maybe);
192      currentCount = maybe;
193    } catch (AssertionError e) {
194      // Ignore
195    }
196    return currentCount;
197  }
198
199  /**
200   * Tests if the metrics for thrift handler work correctly
201   */
202  public void doTestThriftMetrics() throws Exception {
203    LOG.info("START doTestThriftMetrics");
204    Configuration conf = UTIL.getConfiguration();
205    ThriftMetrics metrics = getMetrics(conf);
206    Hbase.Iface handler = getHandlerForMetricsTest(metrics, conf);
207    int currentCountCreateTable = getCurrentCount("createTable_num_ops", 2, metrics);
208    int currentCountDeleteTable = getCurrentCount("deleteTable_num_ops", 2, metrics);
209    int currentCountDisableTable = getCurrentCount("disableTable_num_ops", 2, metrics);
210    createTestTables(handler);
211    dropTestTables(handler);
212    metricsHelper.assertCounter("createTable_num_ops", currentCountCreateTable + 2,
213      metrics.getSource());
214    metricsHelper.assertCounter("deleteTable_num_ops", currentCountDeleteTable + 2,
215      metrics.getSource());
216    metricsHelper.assertCounter("disableTable_num_ops", currentCountDisableTable + 2,
217      metrics.getSource());
218    handler.getTableNames(); // This will have an artificial delay.
219
220    // 3 to 6 seconds (to account for potential slowness), measured in nanoseconds
221    try {
222      metricsHelper.assertGaugeGt("getTableNames_avg_time", 3L * 1000 * 1000 * 1000,
223              metrics.getSource());
224      metricsHelper.assertGaugeLt("getTableNames_avg_time",6L * 1000 * 1000 * 1000,
225              metrics.getSource());
226    } catch (AssertionError e) {
227      LOG.info("Fix me!  Why does this happen?  A concurrent cluster running?", e);
228    }
229  }
230
231  private static Hbase.Iface getHandlerForMetricsTest(ThriftMetrics metrics, Configuration conf)
232      throws Exception {
233    Hbase.Iface handler = new MySlowHBaseHandler(conf);
234    return HbaseHandlerMetricsProxy.newInstance((ThriftHBaseServiceHandler)handler, metrics, conf);
235  }
236
237  private static ThriftMetrics getMetrics(Configuration conf) throws Exception {
238    return new ThriftMetrics(conf, ThriftMetrics.ThriftServerType.ONE);
239  }
240
241
242  public static void createTestTables(Hbase.Iface handler) throws Exception {
243    // Create/enable/disable/delete tables, ensure methods act correctly
244    assertEquals(0, handler.getTableNames().size());
245    handler.createTable(tableAname, getColumnDescriptors());
246    assertEquals(1, handler.getTableNames().size());
247    assertEquals(2, handler.getColumnDescriptors(tableAname).size());
248    assertTrue(handler.isTableEnabled(tableAname));
249    handler.createTable(tableBname, getColumnDescriptors());
250    assertEquals(2, handler.getTableNames().size());
251  }
252
253  public static void checkTableList(Hbase.Iface handler) throws Exception {
254    assertTrue(handler.getTableNames().contains(tableAname));
255  }
256
257  public static void dropTestTables(Hbase.Iface handler) throws Exception {
258    handler.disableTable(tableBname);
259    assertFalse(handler.isTableEnabled(tableBname));
260    handler.deleteTable(tableBname);
261    assertEquals(1, handler.getTableNames().size());
262    handler.disableTable(tableAname);
263    assertFalse(handler.isTableEnabled(tableAname));
264    /* TODO Reenable.
265    assertFalse(handler.isTableEnabled(tableAname));
266    handler.enableTable(tableAname);
267    assertTrue(handler.isTableEnabled(tableAname));
268    handler.disableTable(tableAname);*/
269    handler.deleteTable(tableAname);
270    assertEquals(0, handler.getTableNames().size());
271  }
272
273  public void doTestIncrements() throws Exception {
274    ThriftHBaseServiceHandler handler =
275      new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
276        UserProvider.instantiate(UTIL.getConfiguration()));
277    createTestTables(handler);
278    doTestIncrements(handler);
279    dropTestTables(handler);
280  }
281
282  public static void doTestIncrements(ThriftHBaseServiceHandler handler) throws Exception {
283    List<Mutation> mutations = new ArrayList<>(1);
284    mutations.add(new Mutation(false, columnAAname, valueEname, true));
285    mutations.add(new Mutation(false, columnAname, valueEname, true));
286    handler.mutateRow(tableAname, rowAname, mutations, null);
287    handler.mutateRow(tableAname, rowBname, mutations, null);
288
289    List<TIncrement> increments = new ArrayList<>(3);
290    increments.add(new TIncrement(tableAname, rowBname, columnAAname, 7));
291    increments.add(new TIncrement(tableAname, rowBname, columnAAname, 7));
292    increments.add(new TIncrement(tableAname, rowBname, columnAAname, 7));
293
294    int numIncrements = 60000;
295    for (int i = 0; i < numIncrements; i++) {
296      handler.increment(new TIncrement(tableAname, rowAname, columnAname, 2));
297      handler.incrementRows(increments);
298    }
299
300    Thread.sleep(1000);
301    long lv = handler.get(tableAname, rowAname, columnAname, null).get(0).value.getLong();
302    // Wait on all increments being flushed
303    while (handler.coalescer.getQueueSize() != 0) {
304      Threads.sleep(10);
305    }
306
307    assertEquals((100 + (2 * numIncrements)), lv);
308
309    lv = handler.get(tableAname, rowBname, columnAAname, null).get(0).value.getLong();
310    assertEquals((100 + (3 * 7 * numIncrements)), lv);
311
312    assertTrue(handler.coalescer.getSuccessfulCoalescings() > 0);
313
314  }
315
316  /**
317   * Tests adding a series of Mutations and BatchMutations, including a
318   * delete mutation.  Also tests data retrieval, and getting back multiple
319   * versions.
320   */
321  public void doTestTableMutations() throws Exception {
322    ThriftHBaseServiceHandler handler =
323      new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
324        UserProvider.instantiate(UTIL.getConfiguration()));
325    doTestTableMutations(handler);
326  }
327
328  public static void doTestTableMutations(Hbase.Iface handler) throws Exception {
329    // Setup
330    handler.createTable(tableAname, getColumnDescriptors());
331
332    // Apply a few Mutations to rowA
333    //     mutations.add(new Mutation(false, columnAname, valueAname));
334    //     mutations.add(new Mutation(false, columnBname, valueBname));
335    handler.mutateRow(tableAname, rowAname, getMutations(), null);
336
337    // Assert that the changes were made
338    assertEquals(valueAname,
339      handler.get(tableAname, rowAname, columnAname, null).get(0).value);
340    TRowResult rowResult1 = handler.getRow(tableAname, rowAname, null).get(0);
341    assertEquals(rowAname, rowResult1.row);
342    assertEquals(valueBname,
343      rowResult1.columns.get(columnBname).value);
344
345    // Apply a few BatchMutations for rowA and rowB
346    // rowAmutations.add(new Mutation(true, columnAname, null));
347    // rowAmutations.add(new Mutation(false, columnBname, valueCname));
348    // batchMutations.add(new BatchMutation(rowAname, rowAmutations));
349    // Mutations to rowB
350    // rowBmutations.add(new Mutation(false, columnAname, valueCname));
351    // rowBmutations.add(new Mutation(false, columnBname, valueDname));
352    // batchMutations.add(new BatchMutation(rowBname, rowBmutations));
353    handler.mutateRows(tableAname, getBatchMutations(), null);
354
355    // Assert that changes were made to rowA
356    List<TCell> cells = handler.get(tableAname, rowAname, columnAname, null);
357    assertFalse(cells.size() > 0);
358    assertEquals(valueCname, handler.get(tableAname, rowAname, columnBname, null).get(0).value);
359    List<TCell> versions = handler.getVer(tableAname, rowAname, columnBname, MAXVERSIONS, null);
360    assertEquals(valueCname, versions.get(0).value);
361    assertEquals(valueBname, versions.get(1).value);
362
363    // Assert that changes were made to rowB
364    TRowResult rowResult2 = handler.getRow(tableAname, rowBname, null).get(0);
365    assertEquals(rowBname, rowResult2.row);
366    assertEquals(valueCname, rowResult2.columns.get(columnAname).value);
367    assertEquals(valueDname, rowResult2.columns.get(columnBname).value);
368
369    // Apply some deletes
370    handler.deleteAll(tableAname, rowAname, columnBname, null);
371    handler.deleteAllRow(tableAname, rowBname, null);
372
373    // Assert that the deletes were applied
374    int size = handler.get(tableAname, rowAname, columnBname, null).size();
375    assertEquals(0, size);
376    size = handler.getRow(tableAname, rowBname, null).size();
377    assertEquals(0, size);
378
379    // Try null mutation
380    List<Mutation> mutations = new ArrayList<>(1);
381    mutations.add(new Mutation(false, columnAname, null, true));
382    handler.mutateRow(tableAname, rowAname, mutations, null);
383    TRowResult rowResult3 = handler.getRow(tableAname, rowAname, null).get(0);
384    assertEquals(rowAname, rowResult3.row);
385    assertEquals(0, rowResult3.columns.get(columnAname).value.remaining());
386
387    // Teardown
388    handler.disableTable(tableAname);
389    handler.deleteTable(tableAname);
390  }
391
392  /**
393   * Similar to testTableMutations(), except Mutations are applied with
394   * specific timestamps and data retrieval uses these timestamps to
395   * extract specific versions of data.
396   */
397  public void doTestTableTimestampsAndColumns() throws Exception {
398    // Setup
399    ThriftHBaseServiceHandler handler =
400      new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
401        UserProvider.instantiate(UTIL.getConfiguration()));
402    handler.createTable(tableAname, getColumnDescriptors());
403
404    // Apply timestamped Mutations to rowA
405    long time1 = System.currentTimeMillis();
406    handler.mutateRowTs(tableAname, rowAname, getMutations(), time1, null);
407
408    Thread.sleep(1000);
409
410    // Apply timestamped BatchMutations for rowA and rowB
411    long time2 = System.currentTimeMillis();
412    handler.mutateRowsTs(tableAname, getBatchMutations(), time2, null);
413
414    // Apply an overlapping timestamped mutation to rowB
415    handler.mutateRowTs(tableAname, rowBname, getMutations(), time2, null);
416
417    // the getVerTs is [inf, ts) so you need to increment one.
418    time1 += 1;
419    time2 += 2;
420
421    // Assert that the timestamp-related methods retrieve the correct data
422    assertEquals(2, handler.getVerTs(tableAname, rowAname, columnBname, time2,
423      MAXVERSIONS, null).size());
424    assertEquals(1, handler.getVerTs(tableAname, rowAname, columnBname, time1,
425      MAXVERSIONS, null).size());
426
427    TRowResult rowResult1 = handler.getRowTs(tableAname, rowAname, time1, null).get(0);
428    TRowResult rowResult2 = handler.getRowTs(tableAname, rowAname, time2, null).get(0);
429    // columnA was completely deleted
430    //assertTrue(Bytes.equals(rowResult1.columns.get(columnAname).value, valueAname));
431    assertEquals(rowResult1.columns.get(columnBname).value, valueBname);
432    assertEquals(rowResult2.columns.get(columnBname).value, valueCname);
433
434    // ColumnAname has been deleted, and will never be visible even with a getRowTs()
435    assertFalse(rowResult2.columns.containsKey(columnAname));
436
437    List<ByteBuffer> columns = new ArrayList<>(1);
438    columns.add(columnBname);
439
440    rowResult1 = handler.getRowWithColumns(tableAname, rowAname, columns, null).get(0);
441    assertEquals(rowResult1.columns.get(columnBname).value, valueCname);
442    assertFalse(rowResult1.columns.containsKey(columnAname));
443
444    rowResult1 = handler.getRowWithColumnsTs(tableAname, rowAname, columns, time1, null).get(0);
445    assertEquals(rowResult1.columns.get(columnBname).value, valueBname);
446    assertFalse(rowResult1.columns.containsKey(columnAname));
447
448    // Apply some timestamped deletes
449    // this actually deletes _everything_.
450    // nukes everything in columnB: forever.
451    handler.deleteAllTs(tableAname, rowAname, columnBname, time1, null);
452    handler.deleteAllRowTs(tableAname, rowBname, time2, null);
453
454    // Assert that the timestamp-related methods retrieve the correct data
455    int size = handler.getVerTs(tableAname, rowAname, columnBname, time1, MAXVERSIONS, null).size();
456    assertEquals(0, size);
457
458    size = handler.getVerTs(tableAname, rowAname, columnBname, time2, MAXVERSIONS, null).size();
459    assertEquals(1, size);
460
461    // should be available....
462    assertEquals(handler.get(tableAname, rowAname, columnBname, null).get(0).value, valueCname);
463
464    assertEquals(0, handler.getRow(tableAname, rowBname, null).size());
465
466    // Teardown
467    handler.disableTable(tableAname);
468    handler.deleteTable(tableAname);
469  }
470
471  /**
472   * Tests the four different scanner-opening methods (with and without
473   * a stoprow, with and without a timestamp).
474   */
475  public void doTestTableScanners() throws Exception {
476    // Setup
477    ThriftHBaseServiceHandler handler =
478      new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
479        UserProvider.instantiate(UTIL.getConfiguration()));
480    handler.createTable(tableAname, getColumnDescriptors());
481
482    // Apply timestamped Mutations to rowA
483    long time1 = System.currentTimeMillis();
484    handler.mutateRowTs(tableAname, rowAname, getMutations(), time1, null);
485
486    // Sleep to assure that 'time1' and 'time2' will be different even with a
487    // coarse grained system timer.
488    Thread.sleep(1000);
489
490    // Apply timestamped BatchMutations for rowA and rowB
491    long time2 = System.currentTimeMillis();
492    handler.mutateRowsTs(tableAname, getBatchMutations(), time2, null);
493
494    time1 += 1;
495
496    // Test a scanner on all rows and all columns, no timestamp
497    int scanner1 = handler.scannerOpen(tableAname, rowAname, getColumnList(true, true), null);
498    TRowResult rowResult1a = handler.scannerGet(scanner1).get(0);
499    assertEquals(rowResult1a.row, rowAname);
500    // This used to be '1'.  I don't know why when we are asking for two columns
501    // and when the mutations above would seem to add two columns to the row.
502    // -- St.Ack 05/12/2009
503    assertEquals(1, rowResult1a.columns.size());
504    assertEquals(rowResult1a.columns.get(columnBname).value, valueCname);
505
506    TRowResult rowResult1b = handler.scannerGet(scanner1).get(0);
507    assertEquals(rowResult1b.row, rowBname);
508    assertEquals(2, rowResult1b.columns.size());
509    assertEquals(rowResult1b.columns.get(columnAname).value, valueCname);
510    assertEquals(rowResult1b.columns.get(columnBname).value, valueDname);
511    closeScanner(scanner1, handler);
512
513    // Test a scanner on all rows and all columns, with timestamp
514    int scanner2 = handler.scannerOpenTs(tableAname, rowAname, getColumnList(true, true), time1,
515            null);
516    TRowResult rowResult2a = handler.scannerGet(scanner2).get(0);
517    assertEquals(1, rowResult2a.columns.size());
518    // column A deleted, does not exist.
519    //assertTrue(Bytes.equals(rowResult2a.columns.get(columnAname).value, valueAname));
520    assertEquals(rowResult2a.columns.get(columnBname).value, valueBname);
521    closeScanner(scanner2, handler);
522
523    // Test a scanner on the first row and first column only, no timestamp
524    int scanner3 = handler.scannerOpenWithStop(tableAname, rowAname, rowBname,
525        getColumnList(true, false), null);
526    closeScanner(scanner3, handler);
527
528    // Test a scanner on the first row and second column only, with timestamp
529    int scanner4 = handler.scannerOpenWithStopTs(tableAname, rowAname, rowBname,
530        getColumnList(false, true), time1, null);
531    TRowResult rowResult4a = handler.scannerGet(scanner4).get(0);
532    assertEquals(1, rowResult4a.columns.size());
533    assertEquals(rowResult4a.columns.get(columnBname).value, valueBname);
534
535    // Test scanner using a TScan object once with sortColumns False and once with sortColumns true
536    TScan scanNoSortColumns = new TScan();
537    scanNoSortColumns.setStartRow(rowAname);
538    scanNoSortColumns.setStopRow(rowBname);
539
540    int scanner5 = handler.scannerOpenWithScan(tableAname , scanNoSortColumns, null);
541    TRowResult rowResult5 = handler.scannerGet(scanner5).get(0);
542    assertEquals(1, rowResult5.columns.size());
543    assertEquals(rowResult5.columns.get(columnBname).value, valueCname);
544
545    TScan scanSortColumns = new TScan();
546    scanSortColumns.setStartRow(rowAname);
547    scanSortColumns.setStopRow(rowBname);
548    scanSortColumns = scanSortColumns.setSortColumns(true);
549
550    int scanner6 = handler.scannerOpenWithScan(tableAname ,scanSortColumns, null);
551    TRowResult rowResult6 = handler.scannerGet(scanner6).get(0);
552    assertEquals(1, rowResult6.sortedColumns.size());
553    assertEquals(rowResult6.sortedColumns.get(0).getCell().value, valueCname);
554
555    List<Mutation> rowBmutations = new ArrayList<>(20);
556    for (int i = 0; i < 20; i++) {
557      rowBmutations.add(new Mutation(false, asByteBuffer("columnA:" + i), valueCname, true));
558    }
559    ByteBuffer rowC = asByteBuffer("rowC");
560    handler.mutateRow(tableAname, rowC, rowBmutations, null);
561
562    TScan scanSortMultiColumns = new TScan();
563    scanSortMultiColumns.setStartRow(rowC);
564    scanSortMultiColumns = scanSortMultiColumns.setSortColumns(true);
565    int scanner7 = handler.scannerOpenWithScan(tableAname, scanSortMultiColumns, null);
566    TRowResult rowResult7 = handler.scannerGet(scanner7).get(0);
567
568    ByteBuffer smallerColumn = asByteBuffer("columnA:");
569    for (int i = 0; i < 20; i++) {
570      ByteBuffer currentColumn = rowResult7.sortedColumns.get(i).columnName;
571      assertTrue(Bytes.compareTo(smallerColumn.array(), currentColumn.array()) < 0);
572      smallerColumn = currentColumn;
573    }
574
575    TScan reversedScan = new TScan();
576    reversedScan.setReversed(true);
577    reversedScan.setStartRow(rowBname);
578    reversedScan.setStopRow(rowAname);
579
580    int scanner8 = handler.scannerOpenWithScan(tableAname , reversedScan, null);
581    List<TRowResult> results = handler.scannerGet(scanner8);
582    handler.scannerClose(scanner8);
583    assertEquals(1, results.size());
584    assertEquals(ByteBuffer.wrap(results.get(0).getRow()), rowBname);
585
586    // Teardown
587    handler.disableTable(tableAname);
588    handler.deleteTable(tableAname);
589  }
590
591  /**
592   * For HBASE-2556
593   * Tests for GetTableRegions
594   */
595  public void doTestGetTableRegions() throws Exception {
596    ThriftHBaseServiceHandler handler =
597      new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
598        UserProvider.instantiate(UTIL.getConfiguration()));
599    doTestGetTableRegions(handler);
600  }
601
602  public static void doTestGetTableRegions(Hbase.Iface handler)
603      throws Exception {
604    assertEquals(0, handler.getTableNames().size());
605    handler.createTable(tableAname, getColumnDescriptors());
606    assertEquals(1, handler.getTableNames().size());
607    List<TRegionInfo> regions = handler.getTableRegions(tableAname);
608    int regionCount = regions.size();
609    assertEquals("empty table should have only 1 region, " +
610            "but found " + regionCount, 1, regionCount);
611    LOG.info("Region found:" + regions.get(0));
612    handler.disableTable(tableAname);
613    handler.deleteTable(tableAname);
614    regionCount = handler.getTableRegions(tableAname).size();
615    assertEquals("non-existing table should have 0 region, " +
616            "but found " + regionCount, 0, regionCount);
617  }
618
619  public void doTestFilterRegistration() throws Exception {
620    Configuration conf = UTIL.getConfiguration();
621
622    conf.set("hbase.thrift.filters", "MyFilter:filterclass");
623
624    ThriftServer.registerFilters(conf);
625
626    Map<String, String> registeredFilters = ParseFilter.getAllFilters();
627
628    assertEquals("filterclass", registeredFilters.get("MyFilter"));
629  }
630
631  public void doTestGetRegionInfo() throws Exception {
632    ThriftHBaseServiceHandler handler =
633      new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
634        UserProvider.instantiate(UTIL.getConfiguration()));
635    doTestGetRegionInfo(handler);
636  }
637
638  public static void doTestGetRegionInfo(Hbase.Iface handler) throws Exception {
639    // Create tableA and add two columns to rowA
640    handler.createTable(tableAname, getColumnDescriptors());
641    try {
642      handler.mutateRow(tableAname, rowAname, getMutations(), null);
643      byte[] searchRow = HRegionInfo.createRegionName(
644          TableName.valueOf(tableAname.array()), rowAname.array(),
645          HConstants.NINES, false);
646      TRegionInfo regionInfo = handler.getRegionInfo(ByteBuffer.wrap(searchRow));
647      assertTrue(Bytes.toStringBinary(regionInfo.getName()).startsWith(
648            Bytes.toStringBinary(tableAname)));
649    } finally {
650      handler.disableTable(tableAname);
651      handler.deleteTable(tableAname);
652    }
653  }
654
655  /**
656   * Appends the value to a cell and checks that the cell value is updated properly.
657   */
658  public static void doTestAppend() throws Exception {
659    ThriftHBaseServiceHandler handler =
660      new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
661        UserProvider.instantiate(UTIL.getConfiguration()));
662    handler.createTable(tableAname, getColumnDescriptors());
663    try {
664      List<Mutation> mutations = new ArrayList<>(1);
665      mutations.add(new Mutation(false, columnAname, valueAname, true));
666      handler.mutateRow(tableAname, rowAname, mutations, null);
667
668      List<ByteBuffer> columnList = new ArrayList<>(1);
669      columnList.add(columnAname);
670      List<ByteBuffer> valueList = new ArrayList<>(1);
671      valueList.add(valueBname);
672
673      TAppend append = new TAppend(tableAname, rowAname, columnList, valueList);
674      handler.append(append);
675
676      TRowResult rowResult = handler.getRow(tableAname, rowAname, null).get(0);
677      assertEquals(rowAname, rowResult.row);
678      assertArrayEquals(Bytes.add(valueAname.array(), valueBname.array()),
679        rowResult.columns.get(columnAname).value.array());
680    } finally {
681      handler.disableTable(tableAname);
682      handler.deleteTable(tableAname);
683    }
684  }
685
686  /**
687   * Check that checkAndPut fails if the cell does not exist, then put in the cell, then check that
688   * the checkAndPut succeeds.
689   */
690  public static void doTestCheckAndPut() throws Exception {
691    ThriftHBaseServiceHandler handler =
692      new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
693        UserProvider.instantiate(UTIL.getConfiguration()));
694    handler.createTable(tableAname, getColumnDescriptors());
695    try {
696      List<Mutation> mutations = new ArrayList<>(1);
697      mutations.add(new Mutation(false, columnAname, valueAname, true));
698      Mutation putB = (new Mutation(false, columnBname, valueBname, true));
699
700      assertFalse(handler.checkAndPut(tableAname, rowAname, columnAname, valueAname, putB, null));
701
702      handler.mutateRow(tableAname, rowAname, mutations, null);
703
704      assertTrue(handler.checkAndPut(tableAname, rowAname, columnAname, valueAname, putB, null));
705
706      TRowResult rowResult = handler.getRow(tableAname, rowAname, null).get(0);
707      assertEquals(rowAname, rowResult.row);
708      assertEquals(valueBname, rowResult.columns.get(columnBname).value);
709    } finally {
710      handler.disableTable(tableAname);
711      handler.deleteTable(tableAname);
712    }
713  }
714
715  @Test
716  public void testMetricsWithException() throws Exception {
717    String rowkey = "row1";
718    String family = "f";
719    String col = "c";
720    // create a table which will throw exceptions for requests
721    final TableName tableName = TableName.valueOf(name.getMethodName());
722    HTableDescriptor tableDesc = new HTableDescriptor(tableName);
723    tableDesc.addCoprocessor(ErrorThrowingGetObserver.class.getName());
724    tableDesc.addFamily(new HColumnDescriptor(family));
725
726    Table table = UTIL.createTable(tableDesc, null);
727    long now = System.currentTimeMillis();
728    table.put(new Put(Bytes.toBytes(rowkey))
729        .addColumn(Bytes.toBytes(family), Bytes.toBytes(col), now, Bytes.toBytes("val1")));
730
731    Configuration conf = UTIL.getConfiguration();
732    ThriftMetrics metrics = getMetrics(conf);
733    ThriftHBaseServiceHandler hbaseHandler =
734        new ThriftHBaseServiceHandler(UTIL.getConfiguration(),
735            UserProvider.instantiate(UTIL.getConfiguration()));
736    Hbase.Iface handler = HbaseHandlerMetricsProxy.newInstance(hbaseHandler, metrics, conf);
737
738    ByteBuffer tTableName = asByteBuffer(tableName.getNameAsString());
739
740    // check metrics increment with a successful get
741    long preGetCounter = metricsHelper.checkCounterExists("getRow_num_ops", metrics.getSource()) ?
742        metricsHelper.getCounter("getRow_num_ops", metrics.getSource()) :
743        0;
744    List<TRowResult> tRowResult = handler.getRow(tTableName, asByteBuffer(rowkey), null);
745    assertEquals(1, tRowResult.size());
746    TRowResult tResult = tRowResult.get(0);
747
748    TCell expectedColumnValue = new TCell(asByteBuffer("val1"), now);
749
750    assertArrayEquals(Bytes.toBytes(rowkey), tResult.getRow());
751    Collection<TCell> returnedColumnValues = tResult.getColumns().values();
752    assertEquals(1, returnedColumnValues.size());
753    assertEquals(expectedColumnValue, returnedColumnValues.iterator().next());
754
755    metricsHelper.assertCounter("getRow_num_ops", preGetCounter + 1, metrics.getSource());
756
757    // check metrics increment when the get throws each exception type
758    for (ErrorThrowingGetObserver.ErrorType type : ErrorThrowingGetObserver.ErrorType.values()) {
759      testExceptionType(handler, metrics, tTableName, rowkey, type);
760    }
761  }
762
763  private void testExceptionType(Hbase.Iface handler, ThriftMetrics metrics,
764                                 ByteBuffer tTableName, String rowkey,
765                                 ErrorThrowingGetObserver.ErrorType errorType) throws Exception {
766    long preGetCounter = metricsHelper.getCounter("getRow_num_ops", metrics.getSource());
767    String exceptionKey = errorType.getMetricName();
768    long preExceptionCounter = metricsHelper.checkCounterExists(exceptionKey, metrics.getSource()) ?
769        metricsHelper.getCounter(exceptionKey, metrics.getSource()) :
770        0;
771    Map<ByteBuffer, ByteBuffer> attributes = new HashMap<>();
772    attributes.put(asByteBuffer(ErrorThrowingGetObserver.SHOULD_ERROR_ATTRIBUTE),
773        asByteBuffer(errorType.name()));
774    try {
775      List<TRowResult> tRowResult = handler.getRow(tTableName, asByteBuffer(rowkey), attributes);
776      fail("Get with error attribute should have thrown an exception");
777    } catch (IOError e) {
778      LOG.info("Received exception: ", e);
779      metricsHelper.assertCounter("getRow_num_ops", preGetCounter + 1, metrics.getSource());
780      metricsHelper.assertCounter(exceptionKey, preExceptionCounter + 1, metrics.getSource());
781    }
782  }
783
784  /**
785   * @return a List of ColumnDescriptors for use in creating a table.  Has one
786   *         default ColumnDescriptor and one ColumnDescriptor with fewer versions
787   */
788  private static List<ColumnDescriptor> getColumnDescriptors() {
789    ArrayList<ColumnDescriptor> cDescriptors = new ArrayList<>(2);
790
791    // A default ColumnDescriptor
792    ColumnDescriptor cDescA = new ColumnDescriptor();
793    cDescA.name = columnAname;
794    cDescriptors.add(cDescA);
795
796    // A slightly customized ColumnDescriptor (only 2 versions)
797    ColumnDescriptor cDescB = new ColumnDescriptor(columnBname, 2, "NONE",
798        false, "NONE", 0, 0, false, -1);
799    cDescriptors.add(cDescB);
800
801    return cDescriptors;
802  }
803
804  /**
805   *
806   * @param includeA whether or not to include columnA
807   * @param includeB whether or not to include columnB
808   * @return a List of column names for use in retrieving a scanner
809   */
810  private List<ByteBuffer> getColumnList(boolean includeA, boolean includeB) {
811    List<ByteBuffer> columnList = new ArrayList<>();
812
813    if (includeA) {
814      columnList.add(columnAname);
815    }
816    if (includeB) {
817      columnList.add(columnBname);
818    }
819
820    return columnList;
821  }
822
823  /**
824   * @return a List of Mutations for a row, with columnA having valueA
825   *         and columnB having valueB
826   */
827  private static List<Mutation> getMutations() {
828    List<Mutation> mutations = new ArrayList<>(2);
829    mutations.add(new Mutation(false, columnAname, valueAname, true));
830    mutations.add(new Mutation(false, columnBname, valueBname, true));
831    return mutations;
832  }
833
834  /**
835   * @return a List of BatchMutations with the following effects:
836   *         (rowA, columnA): delete
837   *         (rowA, columnB): place valueC
838   *         (rowB, columnA): place valueC
839   *         (rowB, columnB): place valueD
840   */
841  private static List<BatchMutation> getBatchMutations() {
842    List<BatchMutation> batchMutations = new ArrayList<>(3);
843
844    // Mutations to rowA.  You can't mix delete and put anymore.
845    List<Mutation> rowAmutations = new ArrayList<>(1);
846    rowAmutations.add(new Mutation(true, columnAname, null, true));
847    batchMutations.add(new BatchMutation(rowAname, rowAmutations));
848
849    rowAmutations = new ArrayList<>(1);
850    rowAmutations.add(new Mutation(false, columnBname, valueCname, true));
851    batchMutations.add(new BatchMutation(rowAname, rowAmutations));
852
853    // Mutations to rowB
854    List<Mutation> rowBmutations = new ArrayList<>(2);
855    rowBmutations.add(new Mutation(false, columnAname, valueCname, true));
856    rowBmutations.add(new Mutation(false, columnBname, valueDname, true));
857    batchMutations.add(new BatchMutation(rowBname, rowBmutations));
858
859    return batchMutations;
860  }
861
862  /**
863   * Asserts that the passed scanner is exhausted, and then closes
864   * the scanner.
865   *
866   * @param scannerId the scanner to close
867   * @param handler the HBaseServiceHandler interfacing to HBase
868   */
869  private void closeScanner(
870      int scannerId, ThriftHBaseServiceHandler handler) throws Exception {
871    handler.scannerGet(scannerId);
872    handler.scannerClose(scannerId);
873  }
874}