001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.thrift2;
019
020import static java.nio.ByteBuffer.wrap;
021import static org.apache.hadoop.hbase.HConstants.DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD;
022import static org.apache.hadoop.hbase.HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD;
023import static org.apache.hadoop.hbase.thrift.HBaseServiceHandler.CLEANUP_INTERVAL;
024import static org.apache.hadoop.hbase.thrift.HBaseServiceHandler.MAX_IDLETIME;
025import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.deleteFromThrift;
026import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.getFromThrift;
027import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.incrementFromThrift;
028import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.putFromThrift;
029import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.scanFromThrift;
030import static org.junit.jupiter.api.Assertions.assertArrayEquals;
031import static org.junit.jupiter.api.Assertions.assertEquals;
032import static org.junit.jupiter.api.Assertions.assertFalse;
033import static org.junit.jupiter.api.Assertions.assertNull;
034import static org.junit.jupiter.api.Assertions.assertTrue;
035import static org.junit.jupiter.api.Assertions.fail;
036
037import java.io.IOException;
038import java.io.InterruptedIOException;
039import java.net.InetAddress;
040import java.nio.ByteBuffer;
041import java.util.ArrayList;
042import java.util.Arrays;
043import java.util.Collection;
044import java.util.Collections;
045import java.util.Comparator;
046import java.util.HashMap;
047import java.util.HashSet;
048import java.util.List;
049import java.util.Map;
050import java.util.Optional;
051import java.util.Set;
052import java.util.concurrent.TimeUnit;
053import org.apache.hadoop.conf.Configuration;
054import org.apache.hadoop.hbase.Cell;
055import org.apache.hadoop.hbase.CompatibilityFactory;
056import org.apache.hadoop.hbase.CoprocessorEnvironment;
057import org.apache.hadoop.hbase.HBaseTestingUtil;
058import org.apache.hadoop.hbase.ServerName;
059import org.apache.hadoop.hbase.TableName;
060import org.apache.hadoop.hbase.client.Admin;
061import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
062import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
063import org.apache.hadoop.hbase.client.Consistency;
064import org.apache.hadoop.hbase.client.Delete;
065import org.apache.hadoop.hbase.client.Durability;
066import org.apache.hadoop.hbase.client.Get;
067import org.apache.hadoop.hbase.client.Increment;
068import org.apache.hadoop.hbase.client.LogQueryFilter;
069import org.apache.hadoop.hbase.client.Put;
070import org.apache.hadoop.hbase.client.Scan;
071import org.apache.hadoop.hbase.client.Table;
072import org.apache.hadoop.hbase.client.TableDescriptor;
073import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
074import org.apache.hadoop.hbase.coprocessor.ObserverContext;
075import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
076import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
077import org.apache.hadoop.hbase.coprocessor.RegionObserver;
078import org.apache.hadoop.hbase.filter.ParseFilter;
079import org.apache.hadoop.hbase.security.UserProvider;
080import org.apache.hadoop.hbase.security.access.AccessControlClient;
081import org.apache.hadoop.hbase.security.access.Permission;
082import org.apache.hadoop.hbase.security.access.UserPermission;
083import org.apache.hadoop.hbase.test.MetricsAssertHelper;
084import org.apache.hadoop.hbase.testclassification.ClientTests;
085import org.apache.hadoop.hbase.testclassification.LargeTests;
086import org.apache.hadoop.hbase.thrift.ErrorThrowingGetObserver;
087import org.apache.hadoop.hbase.thrift.HBaseThriftTestingUtility;
088import org.apache.hadoop.hbase.thrift.HbaseHandlerMetricsProxy;
089import org.apache.hadoop.hbase.thrift.ThriftMetrics;
090import org.apache.hadoop.hbase.thrift.ThriftMetrics.ThriftServerType;
091import org.apache.hadoop.hbase.thrift2.generated.TAccessControlEntity;
092import org.apache.hadoop.hbase.thrift2.generated.TAppend;
093import org.apache.hadoop.hbase.thrift2.generated.TColumn;
094import org.apache.hadoop.hbase.thrift2.generated.TColumnFamilyDescriptor;
095import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement;
096import org.apache.hadoop.hbase.thrift2.generated.TColumnValue;
097import org.apache.hadoop.hbase.thrift2.generated.TCompareOperator;
098import org.apache.hadoop.hbase.thrift2.generated.TConsistency;
099import org.apache.hadoop.hbase.thrift2.generated.TDataBlockEncoding;
100import org.apache.hadoop.hbase.thrift2.generated.TDelete;
101import org.apache.hadoop.hbase.thrift2.generated.TDeleteType;
102import org.apache.hadoop.hbase.thrift2.generated.TDurability;
103import org.apache.hadoop.hbase.thrift2.generated.TFilterByOperator;
104import org.apache.hadoop.hbase.thrift2.generated.TGet;
105import org.apache.hadoop.hbase.thrift2.generated.THBaseService;
106import org.apache.hadoop.hbase.thrift2.generated.TIOError;
107import org.apache.hadoop.hbase.thrift2.generated.TIllegalArgument;
108import org.apache.hadoop.hbase.thrift2.generated.TIncrement;
109import org.apache.hadoop.hbase.thrift2.generated.TLogQueryFilter;
110import org.apache.hadoop.hbase.thrift2.generated.TMutation;
111import org.apache.hadoop.hbase.thrift2.generated.TNamespaceDescriptor;
112import org.apache.hadoop.hbase.thrift2.generated.TOnlineLogRecord;
113import org.apache.hadoop.hbase.thrift2.generated.TPermissionScope;
114import org.apache.hadoop.hbase.thrift2.generated.TPut;
115import org.apache.hadoop.hbase.thrift2.generated.TReadType;
116import org.apache.hadoop.hbase.thrift2.generated.TResult;
117import org.apache.hadoop.hbase.thrift2.generated.TRowMutations;
118import org.apache.hadoop.hbase.thrift2.generated.TScan;
119import org.apache.hadoop.hbase.thrift2.generated.TServerName;
120import org.apache.hadoop.hbase.thrift2.generated.TTableDescriptor;
121import org.apache.hadoop.hbase.thrift2.generated.TTableName;
122import org.apache.hadoop.hbase.thrift2.generated.TThriftServerType;
123import org.apache.hadoop.hbase.thrift2.generated.TTimeRange;
124import org.apache.hadoop.hbase.util.Bytes;
125import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
126import org.apache.thrift.TException;
127import org.apache.thrift.protocol.TBinaryProtocol;
128import org.apache.thrift.protocol.TProtocol;
129import org.apache.thrift.transport.TSocket;
130import org.apache.thrift.transport.TTransport;
131import org.junit.jupiter.api.AfterAll;
132import org.junit.jupiter.api.Assertions;
133import org.junit.jupiter.api.BeforeAll;
134import org.junit.jupiter.api.BeforeEach;
135import org.junit.jupiter.api.Tag;
136import org.junit.jupiter.api.Test;
137import org.junit.jupiter.api.TestInfo;
138import org.slf4j.Logger;
139import org.slf4j.LoggerFactory;
140
141import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
142import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils;
143
144/**
145 * Unit testing for ThriftServer.HBaseServiceHandler, a part of the org.apache.hadoop.hbase.thrift2
146 * package.
147 */
148@Tag(ClientTests.TAG)
149@Tag(LargeTests.TAG)
150public class TestThriftHBaseServiceHandler {
151
152  private static final Logger LOG = LoggerFactory.getLogger(TestThriftHBaseServiceHandler.class);
153  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
154
155  // Static names for tables, columns, rows, and values
156  private static byte[] tableAname = Bytes.toBytes("tableA");
157  private static byte[] familyAname = Bytes.toBytes("familyA");
158  private static byte[] familyBname = Bytes.toBytes("familyB");
159  private static byte[] qualifierAname = Bytes.toBytes("qualifierA");
160  private static byte[] qualifierBname = Bytes.toBytes("qualifierB");
161  private static byte[] valueAname = Bytes.toBytes("valueA");
162  private static byte[] valueBname = Bytes.toBytes("valueB");
163  private static ColumnFamilyDescriptor[] families = new ColumnFamilyDescriptor[] {
164    ColumnFamilyDescriptorBuilder.newBuilder(familyAname).setMaxVersions(3).build(),
165    ColumnFamilyDescriptorBuilder.newBuilder(familyBname).setMaxVersions(2).build() };
166
167  private static final MetricsAssertHelper metricsHelper =
168    CompatibilityFactory.getInstance(MetricsAssertHelper.class);
169
170  public void assertTColumnValuesEqual(List<TColumnValue> columnValuesA,
171    List<TColumnValue> columnValuesB) {
172    assertEquals(columnValuesA.size(), columnValuesB.size());
173    Comparator<TColumnValue> comparator = new Comparator<TColumnValue>() {
174      @Override
175      public int compare(TColumnValue o1, TColumnValue o2) {
176        return Bytes.compareTo(Bytes.add(o1.getFamily(), o1.getQualifier()),
177          Bytes.add(o2.getFamily(), o2.getQualifier()));
178      }
179    };
180    Collections.sort(columnValuesA, comparator);
181    Collections.sort(columnValuesB, comparator);
182
183    for (int i = 0; i < columnValuesA.size(); i++) {
184      TColumnValue a = columnValuesA.get(i);
185      TColumnValue b = columnValuesB.get(i);
186      assertTColumnValueEqual(a, b);
187    }
188  }
189
190  public void assertTColumnValueEqual(TColumnValue a, TColumnValue b) {
191    assertArrayEquals(a.getFamily(), b.getFamily());
192    assertArrayEquals(a.getQualifier(), b.getQualifier());
193    assertArrayEquals(a.getValue(), b.getValue());
194  }
195
196  @BeforeAll
197  public static void beforeClass() throws Exception {
198    UTIL.getConfiguration().set("hbase.client.retries.number", "3");
199    UTIL.getConfiguration().setBoolean("hbase.regionserver.slowlog.buffer.enabled", true);
200
201    UTIL.getConfiguration().set("hbase.client.retries.number", "3");
202    UTIL.getConfiguration().setBoolean("hbase.security.authorization", true);
203    UTIL.getConfiguration().set("hbase.coprocessor.master.classes",
204      "org.apache.hadoop.hbase.security.access.AccessController");
205    UTIL.getConfiguration().set("hbase.coprocessor.region.classes",
206      "org.apache.hadoop.hbase.security.access.AccessController");
207    UTIL.getConfiguration().set("hbase.coprocessor.regionserver.classes",
208      "org.apache.hadoop.hbase.security.access.AccessController");
209
210    // as we opened access control, we need to start as a superuser. Otherwise, we will not have
211    // sufficient permission to do operations.
212    UTIL.getConfiguration().set("hbase.superuser", System.getProperty("user.name"));
213
214    UTIL.startMiniCluster();
215    TableDescriptor tableDescriptor = TableDescriptorBuilder
216      .newBuilder(TableName.valueOf(tableAname)).setColumnFamilies(Arrays.asList(families)).build();
217    try (Admin admin = UTIL.getAdmin()) {
218      admin.createTable(tableDescriptor);
219    }
220  }
221
222  @AfterAll
223  public static void afterClass() throws Exception {
224    UTIL.shutdownMiniCluster();
225  }
226
227  @BeforeEach
228  public void setup() throws Exception {
229
230  }
231
232  private ThriftHBaseServiceHandler createHandler() throws TException {
233    try {
234      Configuration conf = UTIL.getConfiguration();
235      return new ThriftHBaseServiceHandler(conf, UserProvider.instantiate(conf));
236    } catch (IOException ie) {
237      throw new TException(ie);
238    }
239  }
240
241  @Test
242  public void testExists() throws TIOError, TException {
243    ThriftHBaseServiceHandler handler = createHandler();
244    byte[] rowName = Bytes.toBytes("testExists");
245    ByteBuffer table = wrap(tableAname);
246
247    TGet get = new TGet(wrap(rowName));
248    assertFalse(handler.exists(table, get));
249
250    List<TColumnValue> columnValues = new ArrayList<>(2);
251    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
252    columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname)));
253    TPut put = new TPut(wrap(rowName), columnValues);
254    put.setColumnValues(columnValues);
255
256    handler.put(table, put);
257
258    assertTrue(handler.exists(table, get));
259  }
260
261  @Test
262  public void testExistsAll() throws TIOError, TException {
263    ThriftHBaseServiceHandler handler = createHandler();
264    byte[] rowName1 = Bytes.toBytes("testExistsAll1");
265    byte[] rowName2 = Bytes.toBytes("testExistsAll2");
266    ByteBuffer table = wrap(tableAname);
267
268    List<TGet> gets = new ArrayList<>();
269    gets.add(new TGet(wrap(rowName2)));
270    gets.add(new TGet(wrap(rowName2)));
271    List<Boolean> existsResult1 = handler.existsAll(table, gets);
272    assertFalse(existsResult1.get(0));
273    assertFalse(existsResult1.get(1));
274
275    List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
276    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
277    columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname)));
278    List<TPut> puts = new ArrayList<TPut>();
279    puts.add(new TPut(wrap(rowName1), columnValues));
280    puts.add(new TPut(wrap(rowName2), columnValues));
281
282    handler.putMultiple(table, puts);
283    List<Boolean> existsResult2 = handler.existsAll(table, gets);
284
285    assertTrue(existsResult2.get(0));
286    assertTrue(existsResult2.get(1));
287  }
288
289  @Test
290  public void testPutGet() throws Exception {
291    ThriftHBaseServiceHandler handler = createHandler();
292    byte[] rowName = Bytes.toBytes("testPutGet");
293    ByteBuffer table = wrap(tableAname);
294
295    List<TColumnValue> columnValues = new ArrayList<>(2);
296    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
297    columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname)));
298    TPut put = new TPut(wrap(rowName), columnValues);
299
300    put.setColumnValues(columnValues);
301
302    handler.put(table, put);
303
304    TGet get = new TGet(wrap(rowName));
305
306    TResult result = handler.get(table, get);
307    assertArrayEquals(rowName, result.getRow());
308    List<TColumnValue> returnedColumnValues = result.getColumnValues();
309    assertTColumnValuesEqual(columnValues, returnedColumnValues);
310  }
311
312  @Test
313  public void testPutGetMultiple() throws Exception {
314    ThriftHBaseServiceHandler handler = createHandler();
315    ByteBuffer table = wrap(tableAname);
316    byte[] rowName1 = Bytes.toBytes("testPutGetMultiple1");
317    byte[] rowName2 = Bytes.toBytes("testPutGetMultiple2");
318
319    List<TColumnValue> columnValues = new ArrayList<>(2);
320    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
321    columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname)));
322    List<TPut> puts = new ArrayList<>(2);
323    puts.add(new TPut(wrap(rowName1), columnValues));
324    puts.add(new TPut(wrap(rowName2), columnValues));
325
326    handler.putMultiple(table, puts);
327
328    List<TGet> gets = new ArrayList<>(2);
329    gets.add(new TGet(wrap(rowName1)));
330    gets.add(new TGet(wrap(rowName2)));
331
332    List<TResult> results = handler.getMultiple(table, gets);
333    assertEquals(2, results.size());
334
335    assertArrayEquals(rowName1, results.get(0).getRow());
336    assertTColumnValuesEqual(columnValues, results.get(0).getColumnValues());
337
338    assertArrayEquals(rowName2, results.get(1).getRow());
339    assertTColumnValuesEqual(columnValues, results.get(1).getColumnValues());
340  }
341
342  @Test
343  public void testDeleteMultiple() throws Exception {
344    ThriftHBaseServiceHandler handler = createHandler();
345    ByteBuffer table = wrap(tableAname);
346    byte[] rowName1 = Bytes.toBytes("testDeleteMultiple1");
347    byte[] rowName2 = Bytes.toBytes("testDeleteMultiple2");
348
349    List<TColumnValue> columnValues = new ArrayList<>(2);
350    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
351    columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname)));
352    List<TPut> puts = new ArrayList<>(2);
353    puts.add(new TPut(wrap(rowName1), columnValues));
354    puts.add(new TPut(wrap(rowName2), columnValues));
355
356    handler.putMultiple(table, puts);
357
358    List<TDelete> deletes = new ArrayList<>(2);
359    deletes.add(new TDelete(wrap(rowName1)));
360    deletes.add(new TDelete(wrap(rowName2)));
361
362    List<TDelete> deleteResults = handler.deleteMultiple(table, deletes);
363    // 0 means they were all successfully applies
364    assertEquals(0, deleteResults.size());
365
366    assertFalse(handler.exists(table, new TGet(wrap(rowName1))));
367    assertFalse(handler.exists(table, new TGet(wrap(rowName2))));
368  }
369
370  @Test
371  public void testDelete() throws Exception {
372    ThriftHBaseServiceHandler handler = createHandler();
373    byte[] rowName = Bytes.toBytes("testDelete");
374    ByteBuffer table = wrap(tableAname);
375
376    List<TColumnValue> columnValues = new ArrayList<>(2);
377    TColumnValue columnValueA =
378      new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
379    TColumnValue columnValueB =
380      new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname));
381    columnValues.add(columnValueA);
382    columnValues.add(columnValueB);
383    TPut put = new TPut(wrap(rowName), columnValues);
384
385    put.setColumnValues(columnValues);
386
387    handler.put(table, put);
388
389    TDelete delete = new TDelete(wrap(rowName));
390    List<TColumn> deleteColumns = new ArrayList<>(1);
391    TColumn deleteColumn = new TColumn(wrap(familyAname));
392    deleteColumn.setQualifier(qualifierAname);
393    deleteColumns.add(deleteColumn);
394    delete.setColumns(deleteColumns);
395
396    handler.deleteSingle(table, delete);
397
398    TGet get = new TGet(wrap(rowName));
399    TResult result = handler.get(table, get);
400    assertArrayEquals(rowName, result.getRow());
401    List<TColumnValue> returnedColumnValues = result.getColumnValues();
402    List<TColumnValue> expectedColumnValues = new ArrayList<>(1);
403    expectedColumnValues.add(columnValueB);
404    assertTColumnValuesEqual(expectedColumnValues, returnedColumnValues);
405  }
406
407  @Test
408  public void testDeleteAllTimestamps() throws Exception {
409    ThriftHBaseServiceHandler handler = createHandler();
410    byte[] rowName = Bytes.toBytes("testDeleteAllTimestamps");
411    ByteBuffer table = wrap(tableAname);
412
413    List<TColumnValue> columnValues = new ArrayList<>(1);
414    TColumnValue columnValueA =
415      new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
416    columnValueA.setTimestamp(EnvironmentEdgeManager.currentTime() - 10);
417    columnValues.add(columnValueA);
418    TPut put = new TPut(wrap(rowName), columnValues);
419
420    put.setColumnValues(columnValues);
421
422    handler.put(table, put);
423    columnValueA.setTimestamp(EnvironmentEdgeManager.currentTime());
424    handler.put(table, put);
425
426    TGet get = new TGet(wrap(rowName));
427    get.setMaxVersions(2);
428    TResult result = handler.get(table, get);
429    assertEquals(2, result.getColumnValuesSize());
430
431    TDelete delete = new TDelete(wrap(rowName));
432    List<TColumn> deleteColumns = new ArrayList<>(1);
433    TColumn deleteColumn = new TColumn(wrap(familyAname));
434    deleteColumn.setQualifier(qualifierAname);
435    deleteColumns.add(deleteColumn);
436    delete.setColumns(deleteColumns);
437    delete.setDeleteType(TDeleteType.DELETE_COLUMNS); // This is the default anyway.
438
439    handler.deleteSingle(table, delete);
440
441    get = new TGet(wrap(rowName));
442    result = handler.get(table, get);
443    assertNull(result.getRow());
444    assertEquals(0, result.getColumnValuesSize());
445  }
446
447  @Test
448  public void testDeleteSingleTimestamp() throws Exception {
449    ThriftHBaseServiceHandler handler = createHandler();
450    byte[] rowName = Bytes.toBytes("testDeleteSingleTimestamp");
451    ByteBuffer table = wrap(tableAname);
452
453    long timestamp1 = EnvironmentEdgeManager.currentTime() - 10;
454    long timestamp2 = EnvironmentEdgeManager.currentTime();
455
456    List<TColumnValue> columnValues = new ArrayList<>(1);
457    TColumnValue columnValueA =
458      new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
459    columnValueA.setTimestamp(timestamp1);
460    columnValues.add(columnValueA);
461    TPut put = new TPut(wrap(rowName), columnValues);
462
463    put.setColumnValues(columnValues);
464
465    handler.put(table, put);
466    columnValueA.setTimestamp(timestamp2);
467    handler.put(table, put);
468
469    TGet get = new TGet(wrap(rowName));
470    get.setMaxVersions(2);
471    TResult result = handler.get(table, get);
472    assertEquals(2, result.getColumnValuesSize());
473
474    TDelete delete = new TDelete(wrap(rowName));
475    List<TColumn> deleteColumns = new ArrayList<>(1);
476    TColumn deleteColumn = new TColumn(wrap(familyAname));
477    deleteColumn.setQualifier(qualifierAname);
478    deleteColumns.add(deleteColumn);
479    delete.setColumns(deleteColumns);
480    delete.setDeleteType(TDeleteType.DELETE_COLUMN);
481
482    handler.deleteSingle(table, delete);
483
484    get = new TGet(wrap(rowName));
485    result = handler.get(table, get);
486    assertArrayEquals(rowName, result.getRow());
487    assertEquals(1, result.getColumnValuesSize());
488    // the older timestamp should remain.
489    assertEquals(timestamp1, result.getColumnValues().get(0).getTimestamp());
490  }
491
492  @Test
493  public void testDeleteFamily() throws Exception {
494    ThriftHBaseServiceHandler handler = createHandler();
495    byte[] rowName = Bytes.toBytes("testDeleteFamily");
496    ByteBuffer table = wrap(tableAname);
497
498    long timestamp1 = EnvironmentEdgeManager.currentTime() - 10;
499    long timestamp2 = EnvironmentEdgeManager.currentTime();
500
501    List<TColumnValue> columnValues = new ArrayList<>();
502    TColumnValue columnValueA =
503      new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
504    columnValueA.setTimestamp(timestamp1);
505    columnValues.add(columnValueA);
506    TPut put = new TPut(wrap(rowName), columnValues);
507
508    put.setColumnValues(columnValues);
509
510    handler.put(table, put);
511    columnValueA.setTimestamp(timestamp2);
512    handler.put(table, put);
513
514    TGet get = new TGet(wrap(rowName));
515    get.setMaxVersions(2);
516    TResult result = handler.get(table, get);
517    assertEquals(2, result.getColumnValuesSize());
518
519    TDelete delete = new TDelete(wrap(rowName));
520    List<TColumn> deleteColumns = new ArrayList<>();
521    TColumn deleteColumn = new TColumn(wrap(familyAname));
522    deleteColumns.add(deleteColumn);
523    delete.setColumns(deleteColumns);
524    delete.setDeleteType(TDeleteType.DELETE_FAMILY);
525
526    handler.deleteSingle(table, delete);
527
528    get = new TGet(wrap(rowName));
529    result = handler.get(table, get);
530    assertNull(result.getRow());
531    assertEquals(0, result.getColumnValuesSize());
532  }
533
534  @Test
535  public void testDeleteFamilyVersion() throws Exception {
536    ThriftHBaseServiceHandler handler = createHandler();
537    byte[] rowName = Bytes.toBytes("testDeleteFamilyVersion");
538    ByteBuffer table = wrap(tableAname);
539
540    long timestamp1 = EnvironmentEdgeManager.currentTime() - 10;
541    long timestamp2 = EnvironmentEdgeManager.currentTime();
542
543    List<TColumnValue> columnValues = new ArrayList<>();
544    TColumnValue columnValueA =
545      new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
546    columnValueA.setTimestamp(timestamp1);
547    columnValues.add(columnValueA);
548    TPut put = new TPut(wrap(rowName), columnValues);
549
550    put.setColumnValues(columnValues);
551
552    handler.put(table, put);
553    columnValueA.setTimestamp(timestamp2);
554    handler.put(table, put);
555
556    TGet get = new TGet(wrap(rowName));
557    get.setMaxVersions(2);
558    TResult result = handler.get(table, get);
559    assertEquals(2, result.getColumnValuesSize());
560
561    TDelete delete = new TDelete(wrap(rowName));
562    List<TColumn> deleteColumns = new ArrayList<>();
563    TColumn deleteColumn = new TColumn(wrap(familyAname));
564    deleteColumn.setTimestamp(timestamp1);
565    deleteColumns.add(deleteColumn);
566    delete.setColumns(deleteColumns);
567    delete.setDeleteType(TDeleteType.DELETE_FAMILY_VERSION);
568
569    handler.deleteSingle(table, delete);
570
571    get = new TGet(wrap(rowName));
572    result = handler.get(table, get);
573    assertArrayEquals(rowName, result.getRow());
574    assertEquals(1, result.getColumnValuesSize());
575    assertEquals(timestamp2, result.getColumnValues().get(0).getTimestamp());
576  }
577
578  @Test
579  public void testIncrement() throws Exception {
580    ThriftHBaseServiceHandler handler = createHandler();
581    byte[] rowName = Bytes.toBytes("testIncrement");
582    ByteBuffer table = wrap(tableAname);
583
584    List<TColumnValue> columnValues = new ArrayList<>(1);
585    columnValues
586      .add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(Bytes.toBytes(1L))));
587    TPut put = new TPut(wrap(rowName), columnValues);
588    put.setColumnValues(columnValues);
589    handler.put(table, put);
590
591    List<TColumnIncrement> incrementColumns = new ArrayList<>(1);
592    incrementColumns.add(new TColumnIncrement(wrap(familyAname), wrap(qualifierAname)));
593    TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
594    handler.increment(table, increment);
595
596    TGet get = new TGet(wrap(rowName));
597    TResult result = handler.get(table, get);
598
599    assertArrayEquals(rowName, result.getRow());
600    assertEquals(1, result.getColumnValuesSize());
601    TColumnValue columnValue = result.getColumnValues().get(0);
602    assertArrayEquals(Bytes.toBytes(2L), columnValue.getValue());
603  }
604
605  @Test
606  public void testAppend() throws Exception {
607    ThriftHBaseServiceHandler handler = createHandler();
608    byte[] rowName = Bytes.toBytes("testAppend");
609    ByteBuffer table = wrap(tableAname);
610    byte[] v1 = Bytes.toBytes("42");
611    byte[] v2 = Bytes.toBytes("23");
612    List<TColumnValue> columnValues = new ArrayList<>(1);
613    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(v1)));
614    TPut put = new TPut(wrap(rowName), columnValues);
615    put.setColumnValues(columnValues);
616    handler.put(table, put);
617
618    List<TColumnValue> appendColumns = new ArrayList<>(1);
619    appendColumns.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(v2)));
620    TAppend append = new TAppend(wrap(rowName), appendColumns);
621    handler.append(table, append);
622
623    TGet get = new TGet(wrap(rowName));
624    TResult result = handler.get(table, get);
625
626    assertArrayEquals(rowName, result.getRow());
627    assertEquals(1, result.getColumnValuesSize());
628    TColumnValue columnValue = result.getColumnValues().get(0);
629    assertArrayEquals(Bytes.add(v1, v2), columnValue.getValue());
630  }
631
632  /**
633   * check that checkAndPut fails if the cell does not exist, then put in the cell, then check that
634   * the checkAndPut succeeds.
635   */
636  @Test
637  public void testCheckAndPut() throws Exception {
638    ThriftHBaseServiceHandler handler = createHandler();
639    byte[] rowName = Bytes.toBytes("testCheckAndPut");
640    ByteBuffer table = wrap(tableAname);
641
642    List<TColumnValue> columnValuesA = new ArrayList<>(1);
643    TColumnValue columnValueA =
644      new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
645    columnValuesA.add(columnValueA);
646    TPut putA = new TPut(wrap(rowName), columnValuesA);
647    putA.setColumnValues(columnValuesA);
648
649    List<TColumnValue> columnValuesB = new ArrayList<>(1);
650    TColumnValue columnValueB =
651      new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname));
652    columnValuesB.add(columnValueB);
653    TPut putB = new TPut(wrap(rowName), columnValuesB);
654    putB.setColumnValues(columnValuesB);
655
656    assertFalse(handler.checkAndPut(table, wrap(rowName), wrap(familyAname), wrap(qualifierAname),
657      wrap(valueAname), putB));
658
659    TGet get = new TGet(wrap(rowName));
660    TResult result = handler.get(table, get);
661    assertEquals(0, result.getColumnValuesSize());
662
663    handler.put(table, putA);
664
665    assertTrue(handler.checkAndPut(table, wrap(rowName), wrap(familyAname), wrap(qualifierAname),
666      wrap(valueAname), putB));
667
668    result = handler.get(table, get);
669    assertArrayEquals(rowName, result.getRow());
670    List<TColumnValue> returnedColumnValues = result.getColumnValues();
671    List<TColumnValue> expectedColumnValues = new ArrayList<>(2);
672    expectedColumnValues.add(columnValueA);
673    expectedColumnValues.add(columnValueB);
674    assertTColumnValuesEqual(expectedColumnValues, returnedColumnValues);
675  }
676
677  /**
678   * check that checkAndDelete fails if the cell does not exist, then put in the cell, then check
679   * that the checkAndDelete succeeds.
680   */
681  @Test
682  public void testCheckAndDelete() throws Exception {
683    ThriftHBaseServiceHandler handler = createHandler();
684    byte[] rowName = Bytes.toBytes("testCheckAndDelete");
685    ByteBuffer table = wrap(tableAname);
686
687    List<TColumnValue> columnValuesA = new ArrayList<>(1);
688    TColumnValue columnValueA =
689      new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
690    columnValuesA.add(columnValueA);
691    TPut putA = new TPut(wrap(rowName), columnValuesA);
692    putA.setColumnValues(columnValuesA);
693
694    List<TColumnValue> columnValuesB = new ArrayList<>(1);
695    TColumnValue columnValueB =
696      new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname));
697    columnValuesB.add(columnValueB);
698    TPut putB = new TPut(wrap(rowName), columnValuesB);
699    putB.setColumnValues(columnValuesB);
700
701    // put putB so that we know whether the row has been deleted or not
702    handler.put(table, putB);
703
704    TDelete delete = new TDelete(wrap(rowName));
705
706    assertFalse(handler.checkAndDelete(table, wrap(rowName), wrap(familyAname),
707      wrap(qualifierAname), wrap(valueAname), delete));
708
709    TGet get = new TGet(wrap(rowName));
710    TResult result = handler.get(table, get);
711    assertArrayEquals(rowName, result.getRow());
712    assertTColumnValuesEqual(columnValuesB, result.getColumnValues());
713
714    handler.put(table, putA);
715
716    assertTrue(handler.checkAndDelete(table, wrap(rowName), wrap(familyAname), wrap(qualifierAname),
717      wrap(valueAname), delete));
718
719    result = handler.get(table, get);
720    assertFalse(result.isSetRow());
721    assertEquals(0, result.getColumnValuesSize());
722  }
723
724  @Test
725  public void testScan() throws Exception {
726    ThriftHBaseServiceHandler handler = createHandler();
727    ByteBuffer table = wrap(tableAname);
728
729    // insert data
730    TColumnValue columnValue =
731      new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
732    List<TColumnValue> columnValues = new ArrayList<>(1);
733    columnValues.add(columnValue);
734    for (int i = 0; i < 10; i++) {
735      TPut put = new TPut(wrap(Bytes.toBytes("testScan" + i)), columnValues);
736      handler.put(table, put);
737    }
738
739    // create scan instance
740    TScan scan = new TScan();
741    List<TColumn> columns = new ArrayList<>(1);
742    TColumn column = new TColumn();
743    column.setFamily(familyAname);
744    column.setQualifier(qualifierAname);
745    columns.add(column);
746    scan.setColumns(columns);
747    scan.setStartRow(Bytes.toBytes("testScan"));
748    scan.setStopRow(Bytes.toBytes("testScan\uffff"));
749
750    // get scanner and rows
751    int scanId = handler.openScanner(table, scan);
752    List<TResult> results = handler.getScannerRows(scanId, 10);
753    assertEquals(10, results.size());
754    for (int i = 0; i < 10; i++) {
755      // check if the rows are returned and in order
756      assertArrayEquals(Bytes.toBytes("testScan" + i), results.get(i).getRow());
757    }
758
759    // check that we are at the end of the scan
760    results = handler.getScannerRows(scanId, 10);
761    assertEquals(0, results.size());
762
763    // close scanner and check that it was indeed closed
764    handler.closeScanner(scanId);
765    try {
766      handler.getScannerRows(scanId, 10);
767      fail("Scanner id should be invalid");
768    } catch (TIllegalArgument e) {
769    }
770  }
771
772  /**
773   * Tests keeping a HBase scanner alive for long periods of time. Each call to getScannerRow()
774   * should reset the ConnectionCache timeout for the scanner's connection.
775   */
776  @org.junit.Ignore
777  @Test // Flakey. Diasabled by HBASE-24079. Renable with Fails with HBASE-24083.
778  // Caused by: java.util.concurrent.RejectedExecutionException:
779  // Task org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture@e385431
780  // rejected from java.util.concurrent.ThreadPoolExecutor@ 52b027d[Terminated, pool size = 0,
781  // active threads = 0, queued tasks = 0, completed tasks = 1]
782  // at org.apache.hadoop.hbase.thrift2.TestThriftHBaseServiceHandler.
783  // testLongLivedScan(TestThriftHBaseServiceHandler.java:804)
784  public void testLongLivedScan() throws Exception {
785    int numTrials = 6;
786    int trialPause = 1000;
787    int cleanUpInterval = 100;
788    Configuration conf = new Configuration(UTIL.getConfiguration());
789    // Set the ConnectionCache timeout to trigger halfway through the trials
790    conf.setInt(MAX_IDLETIME, (numTrials / 2) * trialPause);
791    conf.setInt(CLEANUP_INTERVAL, cleanUpInterval);
792    ThriftHBaseServiceHandler handler =
793      new ThriftHBaseServiceHandler(conf, UserProvider.instantiate(conf));
794
795    ByteBuffer table = wrap(tableAname);
796    // insert data
797    TColumnValue columnValue =
798      new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
799    List<TColumnValue> columnValues = new ArrayList<>(1);
800    columnValues.add(columnValue);
801    for (int i = 0; i < numTrials; i++) {
802      TPut put = new TPut(wrap(Bytes.toBytes("testScan" + i)), columnValues);
803      handler.put(table, put);
804    }
805
806    // create scan instance
807    TScan scan = new TScan();
808    List<TColumn> columns = new ArrayList<>(1);
809    TColumn column = new TColumn();
810    column.setFamily(familyAname);
811    column.setQualifier(qualifierAname);
812    columns.add(column);
813    scan.setColumns(columns);
814    scan.setStartRow(Bytes.toBytes("testScan"));
815    scan.setStopRow(Bytes.toBytes("testScan\uffff"));
816    // Prevent the scanner from caching results
817    scan.setCaching(1);
818
819    // get scanner and rows
820    int scanId = handler.openScanner(table, scan);
821    for (int i = 0; i < numTrials; i++) {
822      // Make sure that the Scanner doesn't throw an exception after the ConnectionCache timeout
823      List<TResult> results = handler.getScannerRows(scanId, 1);
824      assertArrayEquals(Bytes.toBytes("testScan" + i), results.get(0).getRow());
825      Thread.sleep(trialPause);
826    }
827  }
828
829  @Test
830  public void testReverseScan() throws Exception {
831    ThriftHBaseServiceHandler handler = createHandler();
832    ByteBuffer table = wrap(tableAname);
833
834    // insert data
835    TColumnValue columnValue =
836      new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
837    List<TColumnValue> columnValues = new ArrayList<>(1);
838    columnValues.add(columnValue);
839    for (int i = 0; i < 10; i++) {
840      TPut put = new TPut(wrap(Bytes.toBytes("testReverseScan" + i)), columnValues);
841      handler.put(table, put);
842    }
843
844    // create reverse scan instance
845    TScan scan = new TScan();
846    scan.setReversed(true);
847    List<TColumn> columns = new ArrayList<>(1);
848    TColumn column = new TColumn();
849    column.setFamily(familyAname);
850    column.setQualifier(qualifierAname);
851    columns.add(column);
852    scan.setColumns(columns);
853    scan.setStartRow(Bytes.toBytes("testReverseScan\uffff"));
854    scan.setStopRow(Bytes.toBytes("testReverseScan"));
855
856    // get scanner and rows
857    int scanId = handler.openScanner(table, scan);
858    List<TResult> results = handler.getScannerRows(scanId, 10);
859    assertEquals(10, results.size());
860    for (int i = 0; i < 10; i++) {
861      // check if the rows are returned and in order
862      assertArrayEquals(Bytes.toBytes("testReverseScan" + (9 - i)), results.get(i).getRow());
863    }
864
865    // check that we are at the end of the scan
866    results = handler.getScannerRows(scanId, 10);
867    assertEquals(0, results.size());
868
869    // close scanner and check that it was indeed closed
870    handler.closeScanner(scanId);
871    try {
872      handler.getScannerRows(scanId, 10);
873      fail("Scanner id should be invalid");
874    } catch (TIllegalArgument e) {
875    }
876  }
877
878  @Test
879  public void testScanWithFilter() throws Exception {
880    ThriftHBaseServiceHandler handler = createHandler();
881    ByteBuffer table = wrap(tableAname);
882
883    // insert data
884    TColumnValue columnValue =
885      new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
886    List<TColumnValue> columnValues = new ArrayList<>(1);
887    columnValues.add(columnValue);
888    for (int i = 0; i < 10; i++) {
889      TPut put = new TPut(wrap(Bytes.toBytes("testScanWithFilter" + i)), columnValues);
890      handler.put(table, put);
891    }
892
893    // create scan instance with filter
894    TScan scan = new TScan();
895    List<TColumn> columns = new ArrayList<>(1);
896    TColumn column = new TColumn();
897    column.setFamily(familyAname);
898    column.setQualifier(qualifierAname);
899    columns.add(column);
900    scan.setColumns(columns);
901    scan.setStartRow(Bytes.toBytes("testScanWithFilter"));
902    scan.setStopRow(Bytes.toBytes("testScanWithFilter\uffff"));
903    // only get the key part
904    scan.setFilterString(wrap(Bytes.toBytes("KeyOnlyFilter()")));
905
906    // get scanner and rows
907    int scanId = handler.openScanner(table, scan);
908    List<TResult> results = handler.getScannerRows(scanId, 10);
909    assertEquals(10, results.size());
910    for (int i = 0; i < 10; i++) {
911      // check if the rows are returned and in order
912      assertArrayEquals(Bytes.toBytes("testScanWithFilter" + i), results.get(i).getRow());
913      // check that the value is indeed stripped by the filter
914      assertEquals(0, results.get(i).getColumnValues().get(0).getValue().length);
915    }
916
917    // check that we are at the end of the scan
918    results = handler.getScannerRows(scanId, 10);
919    assertEquals(0, results.size());
920
921    // close scanner and check that it was indeed closed
922    handler.closeScanner(scanId);
923    try {
924      handler.getScannerRows(scanId, 10);
925      fail("Scanner id should be invalid");
926    } catch (TIllegalArgument e) {
927    }
928  }
929
930  @Test
931  public void testScanWithColumnFamilyTimeRange() throws Exception {
932    ThriftHBaseServiceHandler handler = createHandler();
933    ByteBuffer table = wrap(tableAname);
934
935    // insert data
936    TColumnValue familyAColumnValue =
937      new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
938    TColumnValue familyBColumnValue =
939      new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname));
940    long minTimestamp = EnvironmentEdgeManager.currentTime();
941    for (int i = 0; i < 10; i++) {
942      familyAColumnValue.setTimestamp(minTimestamp + i);
943      familyBColumnValue.setTimestamp(minTimestamp + i);
944      List<TColumnValue> columnValues = new ArrayList<>(2);
945      columnValues.add(familyAColumnValue);
946      columnValues.add(familyBColumnValue);
947      TPut put =
948        new TPut(wrap(Bytes.toBytes("testScanWithColumnFamilyTimeRange" + i)), columnValues);
949      handler.put(table, put);
950    }
951
952    // create scan instance with column family time range
953    TScan scan = new TScan();
954    Map<ByteBuffer, TTimeRange> colFamTimeRangeMap = new HashMap<>(2);
955    colFamTimeRangeMap.put(wrap(familyAname), new TTimeRange(minTimestamp + 3, minTimestamp + 5));
956    colFamTimeRangeMap.put(wrap(familyBname), new TTimeRange(minTimestamp + 6, minTimestamp + 9));
957    scan.setColFamTimeRangeMap(colFamTimeRangeMap);
958
959    // get scanner and rows
960    int scanId = handler.openScanner(table, scan);
961    List<TResult> results = handler.getScannerRows(scanId, 5);
962    assertEquals(5, results.size());
963    int familyACount = 0;
964    int familyBCount = 0;
965    for (TResult result : results) {
966      List<TColumnValue> columnValues = result.getColumnValues();
967      if (CollectionUtils.isNotEmpty(columnValues)) {
968        if (Bytes.equals(familyAname, columnValues.get(0).getFamily())) {
969          familyACount++;
970        } else if (Bytes.equals(familyBname, columnValues.get(0).getFamily())) {
971          familyBCount++;
972        }
973      }
974    }
975    assertEquals(2, familyACount);
976    assertEquals(3, familyBCount);
977
978    // check that we are at the end of the scan
979    results = handler.getScannerRows(scanId, 1);
980    assertEquals(0, results.size());
981
982    // close scanner and check that it was indeed closed
983    handler.closeScanner(scanId);
984    try {
985      handler.getScannerRows(scanId, 1);
986      fail("Scanner id should be invalid");
987    } catch (TIllegalArgument e) {
988    }
989  }
990
991  @Test
992  public void testSmallScan() throws Exception {
993    ThriftHBaseServiceHandler handler = createHandler();
994    ByteBuffer table = wrap(tableAname);
995
996    // insert data
997    TColumnValue columnValue =
998      new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
999    List<TColumnValue> columnValues = new ArrayList<>();
1000    columnValues.add(columnValue);
1001    for (int i = 0; i < 10; i++) {
1002      TPut put = new TPut(wrap(Bytes.toBytes("testSmallScan" + i)), columnValues);
1003      handler.put(table, put);
1004    }
1005
1006    // small scan instance
1007    TScan scan = new TScan();
1008    scan.setStartRow(Bytes.toBytes("testSmallScan"));
1009    scan.setStopRow(Bytes.toBytes("testSmallScan\uffff"));
1010    scan.setReadType(TReadType.PREAD);
1011    scan.setCaching(2);
1012
1013    // get scanner and rows
1014    int scanId = handler.openScanner(table, scan);
1015    List<TResult> results = handler.getScannerRows(scanId, 10);
1016    assertEquals(10, results.size());
1017    for (int i = 0; i < 10; i++) {
1018      // check if the rows are returned and in order
1019      assertArrayEquals(Bytes.toBytes("testSmallScan" + i), results.get(i).getRow());
1020    }
1021
1022    // check that we are at the end of the scan
1023    results = handler.getScannerRows(scanId, 10);
1024    assertEquals(0, results.size());
1025
1026    // close scanner and check that it was indeed closed
1027    handler.closeScanner(scanId);
1028    try {
1029      handler.getScannerRows(scanId, 10);
1030      fail("Scanner id should be invalid");
1031    } catch (TIllegalArgument e) {
1032    }
1033  }
1034
1035  @Test
1036  public void testExpiredScanner() throws Exception {
1037    Configuration conf = UTIL.getConfiguration();
1038    conf.setLong(HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 1000);
1039    ThriftHBaseServiceHandler handler =
1040      new ThriftHBaseServiceHandler(conf, UserProvider.instantiate(conf));
1041
1042    TScan scan = new TScan();
1043    ByteBuffer table = wrap(tableAname);
1044
1045    int scannerId = handler.openScanner(table, scan);
1046    handler.getScannerRows(scannerId, 1);
1047    Thread.sleep(1000);
1048
1049    try {
1050      handler.getScannerRows(scannerId, 1);
1051      fail("The scanner should be expired and have an TIllegalArgument exception here.");
1052    } catch (TIllegalArgument e) {
1053      assertEquals("Invalid scanner Id", e.getMessage());
1054    } finally {
1055      conf.setLong(HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD,
1056        DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD);
1057    }
1058  }
1059
1060  @Test
1061  public void testPutTTL() throws Exception {
1062    ThriftHBaseServiceHandler handler = createHandler();
1063    byte[] rowName = Bytes.toBytes("testPutTTL");
1064    ByteBuffer table = wrap(tableAname);
1065    List<TColumnValue> columnValues = new ArrayList<>(1);
1066
1067    // Add some dummy data
1068    columnValues
1069      .add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(Bytes.toBytes(1L))));
1070
1071    TPut put = new TPut(wrap(rowName), columnValues);
1072    put.setColumnValues(columnValues);
1073
1074    Map<ByteBuffer, ByteBuffer> attributes = new HashMap<>();
1075
1076    // Time in ms for the kv's to live.
1077    long ttlTimeMs = 2000L;
1078
1079    // the _ttl attribute is a number of ms ttl for key values in this put.
1080    attributes.put(wrap(Bytes.toBytes("_ttl")), wrap(Bytes.toBytes(ttlTimeMs)));
1081    // Attach the attributes
1082    put.setAttributes(attributes);
1083    // Send it.
1084    handler.put(table, put);
1085
1086    // Now get the data back
1087    TGet getOne = new TGet(wrap(rowName));
1088    TResult resultOne = handler.get(table, getOne);
1089
1090    // It's there.
1091    assertArrayEquals(rowName, resultOne.getRow());
1092    assertEquals(1, resultOne.getColumnValuesSize());
1093
1094    // Sleep 30 seconds just to make 100% sure that the key value should be expired.
1095    Thread.sleep(ttlTimeMs * 15);
1096
1097    TGet getTwo = new TGet(wrap(rowName));
1098    TResult resultTwo = handler.get(table, getTwo);
1099
1100    // Nothing should be there since it's ttl'd out.
1101    assertNull(resultTwo.getRow());
1102    assertEquals(0, resultTwo.getColumnValuesSize());
1103  }
1104
1105  /**
1106   * Padding numbers to make comparison of sort order easier in a for loop
1107   * @param n   The number to pad.
1108   * @param pad The length to pad up to.
1109   * @return The padded number as a string.
1110   */
1111  private String pad(int n, byte pad) {
1112    String res = Integer.toString(n);
1113    while (res.length() < pad) {
1114      res = "0" + res;
1115    }
1116    return res;
1117  }
1118
1119  @Test
1120  public void testScanWithBatchSize() throws Exception {
1121    ThriftHBaseServiceHandler handler = createHandler();
1122    ByteBuffer table = wrap(tableAname);
1123
1124    // insert data
1125    List<TColumnValue> columnValues = new ArrayList<>(100);
1126    for (int i = 0; i < 100; i++) {
1127      String colNum = pad(i, (byte) 3);
1128      TColumnValue columnValue = new TColumnValue(wrap(familyAname),
1129        wrap(Bytes.toBytes("col" + colNum)), wrap(Bytes.toBytes("val" + colNum)));
1130      columnValues.add(columnValue);
1131    }
1132    TPut put = new TPut(wrap(Bytes.toBytes("testScanWithBatchSize")), columnValues);
1133    handler.put(table, put);
1134
1135    // create scan instance
1136    TScan scan = new TScan();
1137    List<TColumn> columns = new ArrayList<>(1);
1138    TColumn column = new TColumn();
1139    column.setFamily(familyAname);
1140    columns.add(column);
1141    scan.setColumns(columns);
1142    scan.setStartRow(Bytes.toBytes("testScanWithBatchSize"));
1143    scan.setStopRow(Bytes.toBytes("testScanWithBatchSize\uffff"));
1144    // set batch size to 10 columns per call
1145    scan.setBatchSize(10);
1146
1147    // get scanner
1148    int scanId = handler.openScanner(table, scan);
1149    List<TResult> results = null;
1150    for (int i = 0; i < 10; i++) {
1151      // get batch for single row (10x10 is what we expect)
1152      results = handler.getScannerRows(scanId, 1);
1153      assertEquals(1, results.size());
1154      // check length of batch
1155      List<TColumnValue> cols = results.get(0).getColumnValues();
1156      assertEquals(10, cols.size());
1157      // check if the columns are returned and in order
1158      for (int y = 0; y < 10; y++) {
1159        int colNum = y + (10 * i);
1160        String colNumPad = pad(colNum, (byte) 3);
1161        assertArrayEquals(Bytes.toBytes("col" + colNumPad), cols.get(y).getQualifier());
1162      }
1163    }
1164
1165    // check that we are at the end of the scan
1166    results = handler.getScannerRows(scanId, 1);
1167    assertEquals(0, results.size());
1168
1169    // close scanner and check that it was indeed closed
1170    handler.closeScanner(scanId);
1171    try {
1172      handler.getScannerRows(scanId, 1);
1173      fail("Scanner id should be invalid");
1174    } catch (TIllegalArgument e) {
1175    }
1176  }
1177
1178  @Test
1179  public void testGetScannerResults() throws Exception {
1180    ThriftHBaseServiceHandler handler = createHandler();
1181    ByteBuffer table = wrap(tableAname);
1182
1183    // insert data
1184    TColumnValue columnValue =
1185      new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
1186    List<TColumnValue> columnValues = new ArrayList<>(1);
1187    columnValues.add(columnValue);
1188    for (int i = 0; i < 20; i++) {
1189      TPut put =
1190        new TPut(wrap(Bytes.toBytes("testGetScannerResults" + pad(i, (byte) 2))), columnValues);
1191      handler.put(table, put);
1192    }
1193
1194    // create scan instance
1195    TScan scan = new TScan();
1196    List<TColumn> columns = new ArrayList<>(1);
1197    TColumn column = new TColumn();
1198    column.setFamily(familyAname);
1199    column.setQualifier(qualifierAname);
1200    columns.add(column);
1201    scan.setColumns(columns);
1202    scan.setStartRow(Bytes.toBytes("testGetScannerResults"));
1203
1204    // get 5 rows and check the returned results
1205    scan.setStopRow(Bytes.toBytes("testGetScannerResults05"));
1206    List<TResult> results = handler.getScannerResults(table, scan, 5);
1207    assertEquals(5, results.size());
1208    for (int i = 0; i < 5; i++) {
1209      // check if the rows are returned and in order
1210      assertArrayEquals(Bytes.toBytes("testGetScannerResults" + pad(i, (byte) 2)),
1211        results.get(i).getRow());
1212    }
1213
1214    // get 10 rows and check the returned results
1215    scan.setStopRow(Bytes.toBytes("testGetScannerResults10"));
1216    results = handler.getScannerResults(table, scan, 10);
1217    assertEquals(10, results.size());
1218    for (int i = 0; i < 10; i++) {
1219      // check if the rows are returned and in order
1220      assertArrayEquals(Bytes.toBytes("testGetScannerResults" + pad(i, (byte) 2)),
1221        results.get(i).getRow());
1222    }
1223
1224    // get 20 rows and check the returned results
1225    scan.setStopRow(Bytes.toBytes("testGetScannerResults20"));
1226    results = handler.getScannerResults(table, scan, 20);
1227    assertEquals(20, results.size());
1228    for (int i = 0; i < 20; i++) {
1229      // check if the rows are returned and in order
1230      assertArrayEquals(Bytes.toBytes("testGetScannerResults" + pad(i, (byte) 2)),
1231        results.get(i).getRow());
1232    }
1233
1234    // reverse scan
1235    scan = new TScan();
1236    scan.setColumns(columns);
1237    scan.setReversed(true);
1238    scan.setStartRow(Bytes.toBytes("testGetScannerResults20"));
1239    scan.setStopRow(Bytes.toBytes("testGetScannerResults"));
1240    results = handler.getScannerResults(table, scan, 20);
1241    assertEquals(20, results.size());
1242    for (int i = 0; i < 20; i++) {
1243      // check if the rows are returned and in order
1244      assertArrayEquals(Bytes.toBytes("testGetScannerResults" + pad(19 - i, (byte) 2)),
1245        results.get(i).getRow());
1246    }
1247  }
1248
1249  @Test
1250  public void testFilterRegistration() throws Exception {
1251    Configuration conf = UTIL.getConfiguration();
1252    conf.set("hbase.thrift.filters", "MyFilter:filterclass");
1253    ThriftServer.registerFilters(conf);
1254    Map<String, String> registeredFilters = ParseFilter.getAllFilters();
1255    assertEquals("filterclass", registeredFilters.get("MyFilter"));
1256  }
1257
1258  @Test
1259  public void testMetrics() throws Exception {
1260    Configuration conf = UTIL.getConfiguration();
1261    ThriftMetrics metrics = getMetrics(conf);
1262    ThriftHBaseServiceHandler hbaseHandler = createHandler();
1263    THBaseService.Iface handler = HbaseHandlerMetricsProxy.newInstance(hbaseHandler, metrics, conf);
1264    byte[] rowName = Bytes.toBytes("testMetrics");
1265    ByteBuffer table = wrap(tableAname);
1266
1267    TGet get = new TGet(wrap(rowName));
1268    assertFalse(handler.exists(table, get));
1269
1270    List<TColumnValue> columnValues = new ArrayList<>(2);
1271    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
1272    columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname)));
1273    TPut put = new TPut(wrap(rowName), columnValues);
1274    put.setColumnValues(columnValues);
1275
1276    handler.put(table, put);
1277
1278    assertTrue(handler.exists(table, get));
1279    metricsHelper.assertCounter("put_num_ops", 1, metrics.getSource());
1280    metricsHelper.assertCounter("exists_num_ops", 2, metrics.getSource());
1281  }
1282
1283  private static ThriftMetrics getMetrics(Configuration conf) throws Exception {
1284    ThriftMetrics m = new ThriftMetrics(conf, ThriftMetrics.ThriftServerType.TWO);
1285    m.getSource().init(); // Clear all the metrics
1286    return m;
1287  }
1288
1289  @Test
1290  public void testMetricsWithException(TestInfo testInfo) throws Exception {
1291    byte[] rowkey = Bytes.toBytes("row1");
1292    byte[] family = Bytes.toBytes("f");
1293    byte[] col = Bytes.toBytes("c");
1294    // create a table which will throw exceptions for requests
1295    TableName tableName = TableName.valueOf(testInfo.getTestMethod().get().getName());
1296    TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName)
1297      .setCoprocessor(ErrorThrowingGetObserver.class.getName())
1298      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build();
1299
1300    Table table = UTIL.createTable(tableDesc, null);
1301    table.put(new Put(rowkey).addColumn(family, col, Bytes.toBytes("val1")));
1302
1303    ThriftHBaseServiceHandler hbaseHandler = createHandler();
1304    ThriftMetrics metrics = getMetrics(UTIL.getConfiguration());
1305    THBaseService.Iface handler = HbaseHandlerMetricsProxy.newInstance(hbaseHandler, metrics, null);
1306    ByteBuffer tTableName = wrap(tableName.getName());
1307
1308    // check metrics increment with a successful get
1309    long preGetCounter = metricsHelper.checkCounterExists("get_num_ops", metrics.getSource())
1310      ? metricsHelper.getCounter("get_num_ops", metrics.getSource())
1311      : 0;
1312    TGet tGet = new TGet(wrap(rowkey));
1313    TResult tResult = handler.get(tTableName, tGet);
1314
1315    List<TColumnValue> expectedColumnValues =
1316      Lists.newArrayList(new TColumnValue(wrap(family), wrap(col), wrap(Bytes.toBytes("val1"))));
1317    assertArrayEquals(rowkey, tResult.getRow());
1318    List<TColumnValue> returnedColumnValues = tResult.getColumnValues();
1319    assertTColumnValuesEqual(expectedColumnValues, returnedColumnValues);
1320
1321    metricsHelper.assertCounter("get_num_ops", preGetCounter + 1, metrics.getSource());
1322
1323    // check metrics increment when the get throws each exception type
1324    for (ErrorThrowingGetObserver.ErrorType type : ErrorThrowingGetObserver.ErrorType.values()) {
1325      testExceptionType(handler, metrics, tTableName, rowkey, type);
1326    }
1327  }
1328
1329  private void testExceptionType(THBaseService.Iface handler, ThriftMetrics metrics,
1330    ByteBuffer tTableName, byte[] rowkey, ErrorThrowingGetObserver.ErrorType errorType) {
1331    long preGetCounter = metricsHelper.getCounter("get_num_ops", metrics.getSource());
1332    String exceptionKey = errorType.getMetricName();
1333    long preExceptionCounter = metricsHelper.checkCounterExists(exceptionKey, metrics.getSource())
1334      ? metricsHelper.getCounter(exceptionKey, metrics.getSource())
1335      : 0;
1336    TGet tGet = new TGet(wrap(rowkey));
1337    Map<ByteBuffer, ByteBuffer> attributes = new HashMap<>();
1338    attributes.put(wrap(Bytes.toBytes(ErrorThrowingGetObserver.SHOULD_ERROR_ATTRIBUTE)),
1339      wrap(Bytes.toBytes(errorType.name())));
1340    tGet.setAttributes(attributes);
1341    try {
1342      TResult tResult = handler.get(tTableName, tGet);
1343      fail("Get with error attribute should have thrown an exception");
1344    } catch (TException e) {
1345      LOG.info("Received exception: ", e);
1346      metricsHelper.assertCounter("get_num_ops", preGetCounter + 1, metrics.getSource());
1347      metricsHelper.assertCounter(exceptionKey, preExceptionCounter + 1, metrics.getSource());
1348    }
1349
1350  }
1351
1352  /**
1353   * See HBASE-17611 Latency metrics were capped at ~ 2 seconds due to the use of an int variable to
1354   * capture the duration.
1355   */
1356  @Test
1357  public void testMetricsPrecision() throws Exception {
1358    byte[] rowkey = Bytes.toBytes("row1");
1359    byte[] family = Bytes.toBytes("f");
1360    byte[] col = Bytes.toBytes("c");
1361    // create a table which will throw exceptions for requests
1362    TableName tableName = TableName.valueOf("testMetricsPrecision");
1363    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
1364      .setCoprocessor(DelayingRegionObserver.class.getName())
1365      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build();
1366
1367    Table table = null;
1368    try {
1369      table = UTIL.createTable(tableDescriptor, null);
1370
1371      table.put(new Put(rowkey).addColumn(family, col, Bytes.toBytes("val1")));
1372
1373      ThriftHBaseServiceHandler hbaseHandler = createHandler();
1374      ThriftMetrics metrics = getMetrics(UTIL.getConfiguration());
1375      THBaseService.Iface handler =
1376        HbaseHandlerMetricsProxy.newInstance(hbaseHandler, metrics, null);
1377      ByteBuffer tTableName = wrap(tableName.getName());
1378
1379      // check metrics latency with a successful get
1380      TGet tGet = new TGet(wrap(rowkey));
1381      TResult tResult = handler.get(tTableName, tGet);
1382
1383      List<TColumnValue> expectedColumnValues =
1384        Lists.newArrayList(new TColumnValue(wrap(family), wrap(col), wrap(Bytes.toBytes("val1"))));
1385      assertArrayEquals(rowkey, tResult.getRow());
1386      List<TColumnValue> returnedColumnValues = tResult.getColumnValues();
1387      assertTColumnValuesEqual(expectedColumnValues, returnedColumnValues);
1388
1389      metricsHelper.assertGaugeGt("get_max", 3000L, metrics.getSource());
1390    } finally {
1391      if (table != null) {
1392        try {
1393          table.close();
1394        } catch (IOException ignored) {
1395        }
1396        UTIL.deleteTable(tableName);
1397      }
1398    }
1399  }
1400
1401  @Test
1402  public void testAttribute() throws Exception {
1403    byte[] rowName = Bytes.toBytes("testAttribute");
1404    byte[] attributeKey = Bytes.toBytes("attribute1");
1405    byte[] attributeValue = Bytes.toBytes("value1");
1406    Map<ByteBuffer, ByteBuffer> attributes = new HashMap<>();
1407    attributes.put(wrap(attributeKey), wrap(attributeValue));
1408
1409    TGet tGet = new TGet(wrap(rowName));
1410    tGet.setAttributes(attributes);
1411    Get get = getFromThrift(tGet);
1412    assertArrayEquals(get.getAttribute("attribute1"), attributeValue);
1413
1414    List<TColumnValue> columnValues = new ArrayList<>(1);
1415    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
1416    TPut tPut = new TPut(wrap(rowName), columnValues);
1417    tPut.setAttributes(attributes);
1418    Put put = putFromThrift(tPut);
1419    assertArrayEquals(put.getAttribute("attribute1"), attributeValue);
1420
1421    TScan tScan = new TScan();
1422    tScan.setAttributes(attributes);
1423    Scan scan = scanFromThrift(tScan);
1424    assertArrayEquals(scan.getAttribute("attribute1"), attributeValue);
1425
1426    List<TColumnIncrement> incrementColumns = new ArrayList<>(1);
1427    incrementColumns.add(new TColumnIncrement(wrap(familyAname), wrap(qualifierAname)));
1428    TIncrement tIncrement = new TIncrement(wrap(rowName), incrementColumns);
1429    tIncrement.setAttributes(attributes);
1430    Increment increment = incrementFromThrift(tIncrement);
1431    assertArrayEquals(increment.getAttribute("attribute1"), attributeValue);
1432
1433    TDelete tDelete = new TDelete(wrap(rowName));
1434    tDelete.setAttributes(attributes);
1435    Delete delete = deleteFromThrift(tDelete);
1436    assertArrayEquals(delete.getAttribute("attribute1"), attributeValue);
1437  }
1438
1439  /**
1440   * Put valueA to a row, make sure put has happened, then create a mutation object to put valueB
1441   * and delete ValueA, then check that the row value is only valueB.
1442   */
1443  @Test
1444  public void testMutateRow() throws Exception {
1445    ThriftHBaseServiceHandler handler = createHandler();
1446    byte[] rowName = Bytes.toBytes("testMutateRow");
1447    ByteBuffer table = wrap(tableAname);
1448
1449    List<TColumnValue> columnValuesA = new ArrayList<>(1);
1450    TColumnValue columnValueA =
1451      new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
1452    columnValuesA.add(columnValueA);
1453    TPut putA = new TPut(wrap(rowName), columnValuesA);
1454    putA.setColumnValues(columnValuesA);
1455
1456    handler.put(table, putA);
1457
1458    TGet get = new TGet(wrap(rowName));
1459    TResult result = handler.get(table, get);
1460    assertArrayEquals(rowName, result.getRow());
1461    List<TColumnValue> returnedColumnValues = result.getColumnValues();
1462
1463    List<TColumnValue> expectedColumnValues = new ArrayList<>(1);
1464    expectedColumnValues.add(columnValueA);
1465    assertTColumnValuesEqual(expectedColumnValues, returnedColumnValues);
1466
1467    List<TColumnValue> columnValuesB = new ArrayList<>(1);
1468    TColumnValue columnValueB =
1469      new TColumnValue(wrap(familyAname), wrap(qualifierBname), wrap(valueBname));
1470    columnValuesB.add(columnValueB);
1471    TPut putB = new TPut(wrap(rowName), columnValuesB);
1472    putB.setColumnValues(columnValuesB);
1473
1474    TDelete delete = new TDelete(wrap(rowName));
1475    List<TColumn> deleteColumns = new ArrayList<>(1);
1476    TColumn deleteColumn = new TColumn(wrap(familyAname));
1477    deleteColumn.setQualifier(qualifierAname);
1478    deleteColumns.add(deleteColumn);
1479    delete.setColumns(deleteColumns);
1480
1481    List<TMutation> mutations = new ArrayList<>(2);
1482    TMutation mutationA = TMutation.put(putB);
1483    mutations.add(mutationA);
1484
1485    TMutation mutationB = TMutation.deleteSingle(delete);
1486    mutations.add(mutationB);
1487
1488    TRowMutations tRowMutations = new TRowMutations(wrap(rowName), mutations);
1489    handler.mutateRow(table, tRowMutations);
1490
1491    result = handler.get(table, get);
1492    assertArrayEquals(rowName, result.getRow());
1493    returnedColumnValues = result.getColumnValues();
1494
1495    expectedColumnValues = new ArrayList<>(1);
1496    expectedColumnValues.add(columnValueB);
1497    assertTColumnValuesEqual(expectedColumnValues, returnedColumnValues);
1498  }
1499
1500  /**
1501   * Create TPut, TDelete , TIncrement objects, set durability then call ThriftUtility functions to
1502   * get Put , Delete and Increment respectively. Use getDurability to make sure the returned
1503   * objects have the appropriate durability setting.
1504   */
1505  @Test
1506  public void testDurability() throws Exception {
1507    byte[] rowName = Bytes.toBytes("testDurability");
1508    List<TColumnValue> columnValues = new ArrayList<>(1);
1509    columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
1510
1511    List<TColumnIncrement> incrementColumns = new ArrayList<>(1);
1512    incrementColumns.add(new TColumnIncrement(wrap(familyAname), wrap(qualifierAname)));
1513
1514    TDelete tDelete = new TDelete(wrap(rowName));
1515    tDelete.setDurability(TDurability.SKIP_WAL);
1516    Delete delete = deleteFromThrift(tDelete);
1517    assertEquals(Durability.SKIP_WAL, delete.getDurability());
1518
1519    tDelete.setDurability(TDurability.ASYNC_WAL);
1520    delete = deleteFromThrift(tDelete);
1521    assertEquals(Durability.ASYNC_WAL, delete.getDurability());
1522
1523    tDelete.setDurability(TDurability.SYNC_WAL);
1524    delete = deleteFromThrift(tDelete);
1525    assertEquals(Durability.SYNC_WAL, delete.getDurability());
1526
1527    tDelete.setDurability(TDurability.FSYNC_WAL);
1528    delete = deleteFromThrift(tDelete);
1529    assertEquals(Durability.FSYNC_WAL, delete.getDurability());
1530
1531    TPut tPut = new TPut(wrap(rowName), columnValues);
1532    tPut.setDurability(TDurability.SKIP_WAL);
1533    Put put = putFromThrift(tPut);
1534    assertEquals(Durability.SKIP_WAL, put.getDurability());
1535
1536    tPut.setDurability(TDurability.ASYNC_WAL);
1537    put = putFromThrift(tPut);
1538    assertEquals(Durability.ASYNC_WAL, put.getDurability());
1539
1540    tPut.setDurability(TDurability.SYNC_WAL);
1541    put = putFromThrift(tPut);
1542    assertEquals(Durability.SYNC_WAL, put.getDurability());
1543
1544    tPut.setDurability(TDurability.FSYNC_WAL);
1545    put = putFromThrift(tPut);
1546    assertEquals(Durability.FSYNC_WAL, put.getDurability());
1547
1548    TIncrement tIncrement = new TIncrement(wrap(rowName), incrementColumns);
1549
1550    tIncrement.setDurability(TDurability.SKIP_WAL);
1551    Increment increment = incrementFromThrift(tIncrement);
1552    assertEquals(Durability.SKIP_WAL, increment.getDurability());
1553
1554    tIncrement.setDurability(TDurability.ASYNC_WAL);
1555    increment = incrementFromThrift(tIncrement);
1556    assertEquals(Durability.ASYNC_WAL, increment.getDurability());
1557
1558    tIncrement.setDurability(TDurability.SYNC_WAL);
1559    increment = incrementFromThrift(tIncrement);
1560    assertEquals(Durability.SYNC_WAL, increment.getDurability());
1561
1562    tIncrement.setDurability(TDurability.FSYNC_WAL);
1563    increment = incrementFromThrift(tIncrement);
1564    assertEquals(Durability.FSYNC_WAL, increment.getDurability());
1565  }
1566
1567  @Test
1568  public void testCheckAndMutate() throws Exception {
1569    ThriftHBaseServiceHandler handler = createHandler();
1570    ByteBuffer table = wrap(tableAname);
1571    ByteBuffer row = wrap(Bytes.toBytes("row"));
1572    ByteBuffer family = wrap(familyAname);
1573    ByteBuffer qualifier = wrap(qualifierAname);
1574    ByteBuffer value = wrap(valueAname);
1575
1576    // Create a mutation to write to 'B', our "mutate" of "checkAndMutate"
1577    List<TColumnValue> columnValuesB = new ArrayList<>(1);
1578    TColumnValue columnValueB = new TColumnValue(family, wrap(qualifierBname), wrap(valueBname));
1579    columnValuesB.add(columnValueB);
1580    TPut putB = new TPut(row, columnValuesB);
1581    putB.setColumnValues(columnValuesB);
1582
1583    TRowMutations tRowMutations =
1584      new TRowMutations(row, Arrays.<TMutation> asList(TMutation.put(putB)));
1585
1586    // Empty table when we begin
1587    TResult result = handler.get(table, new TGet(row));
1588    assertEquals(0, result.getColumnValuesSize());
1589
1590    // checkAndMutate -- condition should fail because the value doesn't exist.
1591    assertFalse(handler.checkAndMutate(table, row, family, qualifier, TCompareOperator.EQUAL, value,
1592      tRowMutations), "Expected condition to not pass");
1593
1594    List<TColumnValue> columnValuesA = new ArrayList<>(1);
1595    TColumnValue columnValueA = new TColumnValue(family, qualifier, value);
1596    columnValuesA.add(columnValueA);
1597
1598    // Put an update 'A'
1599    handler.put(table, new TPut(row, columnValuesA));
1600
1601    // Verify that the update is there
1602    result = handler.get(table, new TGet(row));
1603    assertEquals(1, result.getColumnValuesSize());
1604    assertTColumnValueEqual(columnValueA, result.getColumnValues().get(0));
1605
1606    // checkAndMutate -- condition should pass since we added the value
1607    assertTrue(handler.checkAndMutate(table, row, family, qualifier, TCompareOperator.EQUAL, value,
1608      tRowMutations), "Expected condition to pass");
1609
1610    result = handler.get(table, new TGet(row));
1611    assertEquals(2, result.getColumnValuesSize());
1612    assertTColumnValueEqual(columnValueA, result.getColumnValues().get(0));
1613    assertTColumnValueEqual(columnValueB, result.getColumnValues().get(1));
1614  }
1615
1616  @Test
1617  public void testConsistency() throws Exception {
1618    byte[] rowName = Bytes.toBytes("testConsistency");
1619    TGet tGet = new TGet(wrap(rowName));
1620    tGet.setConsistency(TConsistency.STRONG);
1621    Get get = getFromThrift(tGet);
1622    assertEquals(Consistency.STRONG, get.getConsistency());
1623
1624    tGet.setConsistency(TConsistency.TIMELINE);
1625    tGet.setTargetReplicaId(1);
1626    get = getFromThrift(tGet);
1627    assertEquals(Consistency.TIMELINE, get.getConsistency());
1628    assertEquals(1, get.getReplicaId());
1629
1630    TScan tScan = new TScan();
1631    tScan.setConsistency(TConsistency.STRONG);
1632    Scan scan = scanFromThrift(tScan);
1633    assertEquals(Consistency.STRONG, scan.getConsistency());
1634
1635    tScan.setConsistency(TConsistency.TIMELINE);
1636    tScan.setTargetReplicaId(1);
1637    scan = scanFromThrift(tScan);
1638    assertEquals(Consistency.TIMELINE, scan.getConsistency());
1639    assertEquals(1, scan.getReplicaId());
1640
1641    TResult tResult = new TResult();
1642    assertFalse(tResult.isSetStale());
1643    tResult.setStale(true);
1644    assertTrue(tResult.isSetStale());
1645  }
1646
1647  @Test
1648  public void testDDLOpertions() throws Exception {
1649    String namespace = "testDDLOpertionsNamespace";
1650    String table = "testDDLOpertionsTable";
1651    TTableName tTableName = new TTableName();
1652    tTableName.setNs(Bytes.toBytes(namespace));
1653    tTableName.setQualifier(Bytes.toBytes(table));
1654    ThriftHBaseServiceHandler handler = createHandler();
1655    // create name space
1656    TNamespaceDescriptor namespaceDescriptor = new TNamespaceDescriptor();
1657    namespaceDescriptor.setName(namespace);
1658    namespaceDescriptor.putToConfiguration("key1", "value1");
1659    namespaceDescriptor.putToConfiguration("key2", "value2");
1660    handler.createNamespace(namespaceDescriptor);
1661    // list namespace
1662    List<TNamespaceDescriptor> namespaceDescriptors = handler.listNamespaceDescriptors();
1663    // should have 3 namespace, default hbase and testDDLOpertionsNamespace
1664    assertTrue(namespaceDescriptors.size() == 3);
1665    // modify namesapce
1666    namespaceDescriptor.putToConfiguration("kye3", "value3");
1667    handler.modifyNamespace(namespaceDescriptor);
1668    // get namespace
1669    TNamespaceDescriptor namespaceDescriptorReturned = handler.getNamespaceDescriptor(namespace);
1670    assertTrue(namespaceDescriptorReturned.getConfiguration().size() == 3);
1671    // create table
1672    TTableDescriptor tableDescriptor = new TTableDescriptor();
1673    tableDescriptor.setTableName(tTableName);
1674    TColumnFamilyDescriptor columnFamilyDescriptor1 = new TColumnFamilyDescriptor();
1675    columnFamilyDescriptor1.setName(familyAname);
1676    columnFamilyDescriptor1.setDataBlockEncoding(TDataBlockEncoding.DIFF);
1677    tableDescriptor.addToColumns(columnFamilyDescriptor1);
1678    List<ByteBuffer> splitKeys = new ArrayList<>();
1679    splitKeys.add(ByteBuffer.wrap(Bytes.toBytes(5)));
1680    handler.createTable(tableDescriptor, splitKeys);
1681    // modify table
1682    tableDescriptor.setDurability(TDurability.ASYNC_WAL);
1683    handler.modifyTable(tableDescriptor);
1684    // modify column family
1685    columnFamilyDescriptor1.setInMemory(true);
1686    handler.modifyColumnFamily(tTableName, columnFamilyDescriptor1);
1687    // add column family
1688    TColumnFamilyDescriptor columnFamilyDescriptor2 = new TColumnFamilyDescriptor();
1689    columnFamilyDescriptor2.setName(familyBname);
1690    columnFamilyDescriptor2.setDataBlockEncoding(TDataBlockEncoding.PREFIX);
1691    handler.addColumnFamily(tTableName, columnFamilyDescriptor2);
1692    // get table descriptor
1693    TTableDescriptor tableDescriptorReturned = handler.getTableDescriptor(tTableName);
1694    assertTrue(tableDescriptorReturned.getColumns().size() == 2);
1695    assertTrue(tableDescriptorReturned.getDurability() == TDurability.ASYNC_WAL);
1696    TColumnFamilyDescriptor columnFamilyDescriptor1Returned = tableDescriptorReturned.getColumns()
1697      .stream().filter(desc -> Bytes.equals(desc.getName(), familyAname)).findFirst().get();
1698    assertTrue(columnFamilyDescriptor1Returned.isInMemory() == true);
1699    // delete column family
1700    handler.deleteColumnFamily(tTableName, ByteBuffer.wrap(familyBname));
1701    tableDescriptorReturned = handler.getTableDescriptor(tTableName);
1702    assertTrue(tableDescriptorReturned.getColumns().size() == 1);
1703    // disable table
1704    handler.disableTable(tTableName);
1705    assertTrue(handler.isTableDisabled(tTableName));
1706    // enable table
1707    handler.enableTable(tTableName);
1708    assertTrue(handler.isTableEnabled(tTableName));
1709    assertTrue(handler.isTableAvailable(tTableName));
1710    // truncate table
1711    handler.disableTable(tTableName);
1712    handler.truncateTable(tTableName, true);
1713    assertTrue(handler.isTableAvailable(tTableName));
1714    // delete table
1715    handler.disableTable(tTableName);
1716    handler.deleteTable(tTableName);
1717    assertFalse(handler.tableExists(tTableName));
1718    // delete namespace
1719    handler.deleteNamespace(namespace);
1720    namespaceDescriptors = handler.listNamespaceDescriptors();
1721    // should have 2 namespace, default and hbase
1722    assertTrue(namespaceDescriptors.size() == 2);
1723  }
1724
1725  @Test
1726  public void testGetTableDescriptor() throws Exception {
1727    ThriftHBaseServiceHandler handler = createHandler();
1728    TTableDescriptor tableDescriptor =
1729      handler.getTableDescriptor(ThriftUtilities.tableNameFromHBase(TableName.valueOf(tableAname)));
1730    TableDescriptor table = ThriftUtilities.tableDescriptorFromThrift(tableDescriptor);
1731    assertTrue(table.getTableName().equals(TableName.valueOf(tableAname)));
1732    assertTrue(table.getColumnFamilies().length == 2);
1733    assertTrue(table.getColumnFamily(familyAname).getMaxVersions() == 3);
1734    assertTrue(table.getColumnFamily(familyBname).getMaxVersions() == 2);
1735  }
1736
1737  @Test
1738  public void testGetThriftServerType() throws Exception {
1739    ThriftHBaseServiceHandler handler = createHandler();
1740    assertEquals(TThriftServerType.TWO, handler.getThriftServerType());
1741  }
1742
1743  /**
1744   * Verify that thrift2 client calling thrift server can get the thrift server type correctly.
1745   */
1746  @Test
1747  public void testGetThriftServerOneType() throws Exception {
1748
1749    // start a thrift server
1750    HBaseThriftTestingUtility THRIFT_TEST_UTIL = new HBaseThriftTestingUtility();
1751
1752    LOG.info("Starting HBase Thrift server One");
1753    THRIFT_TEST_UTIL.startThriftServer(UTIL.getConfiguration(), ThriftServerType.ONE);
1754    try (TTransport transport =
1755      new TSocket(InetAddress.getLocalHost().getHostName(), THRIFT_TEST_UTIL.getServerPort())) {
1756      TProtocol protocol = new TBinaryProtocol(transport);
1757      // This is our thrift2 client.
1758      THBaseService.Iface client = new THBaseService.Client(protocol);
1759      // open the transport
1760      transport.open();
1761      assertEquals(TThriftServerType.ONE.name(), client.getThriftServerType().name());
1762    } finally {
1763      THRIFT_TEST_UTIL.stopThriftServer();
1764    }
1765  }
1766
1767  @Test
1768  public void testSlowLogResponses() throws Exception {
1769
1770    // start a thrift server
1771    HBaseThriftTestingUtility THRIFT_TEST_UTIL = new HBaseThriftTestingUtility();
1772    Configuration configuration = UTIL.getConfiguration();
1773    configuration.setBoolean("hbase.regionserver.slowlog.buffer.enabled", true);
1774
1775    THRIFT_TEST_UTIL.startThriftServer(configuration, ThriftServerType.ONE);
1776    ThriftHBaseServiceHandler thriftHBaseServiceHandler =
1777      new ThriftHBaseServiceHandler(configuration, UserProvider.instantiate(configuration));
1778    Collection<ServerName> serverNames = UTIL.getAdmin().getRegionServers();
1779    Set<TServerName> tServerNames =
1780      ThriftUtilities.getServerNamesFromHBase(new HashSet<>(serverNames));
1781    List<Boolean> clearedResponses = thriftHBaseServiceHandler.clearSlowLogResponses(tServerNames);
1782    clearedResponses.forEach(Assertions::assertTrue);
1783    TLogQueryFilter tLogQueryFilter = new TLogQueryFilter();
1784    tLogQueryFilter.setLimit(15);
1785    assertEquals(TFilterByOperator.OR, tLogQueryFilter.getFilterByOperator());
1786    LogQueryFilter logQueryFilter = ThriftUtilities.getSlowLogQueryFromThrift(tLogQueryFilter);
1787    assertEquals(LogQueryFilter.FilterByOperator.OR, logQueryFilter.getFilterByOperator());
1788    tLogQueryFilter.setFilterByOperator(TFilterByOperator.AND);
1789    logQueryFilter = ThriftUtilities.getSlowLogQueryFromThrift(tLogQueryFilter);
1790    assertEquals(LogQueryFilter.FilterByOperator.AND, logQueryFilter.getFilterByOperator());
1791    List<TOnlineLogRecord> tLogRecords =
1792      thriftHBaseServiceHandler.getSlowLogResponses(tServerNames, tLogQueryFilter);
1793    assertEquals(0, tLogRecords.size());
1794  }
1795
1796  @Test
1797  public void testPerformTablePermissions() throws Throwable {
1798    // initialize fake objects.
1799    String fakeUser = "user";
1800    TAccessControlEntity tce = new TAccessControlEntity();
1801    tce.setActions("R");
1802    tce.setTableName(Bytes.toString(tableAname));
1803    tce.setScope(TPermissionScope.TABLE);
1804    tce.setUsername(fakeUser);
1805
1806    ThriftHBaseServiceHandler handler = createHandler();
1807    handler.grant(tce);
1808
1809    List<UserPermission> permissionList = AccessControlClient
1810      .getUserPermissions(UTIL.getConnection(), Bytes.toString(tableAname), fakeUser);
1811    // we only grant one R permission
1812    assertEquals(permissionList.size(), 1);
1813
1814    Permission.Action[] actions = permissionList.get(0).getPermission().getActions();
1815    assertEquals(actions.length, 1);
1816    assertEquals(actions[0], Permission.Action.READ);
1817
1818    // than revoke the permission
1819    handler.revoke(tce);
1820    permissionList = AccessControlClient.getUserPermissions(UTIL.getConnection(),
1821      Bytes.toString(tableAname), fakeUser);
1822
1823    // it should return an empty list
1824    assertEquals(0, permissionList.size());
1825  }
1826
1827  @Test
1828  public void testPerformNamespacePermissions() throws Throwable {
1829    // initialize fake objects. We test the permission grant and revoke on default NS.
1830    String fakeUser = "user";
1831    String defaultNameSpace = "default";
1832    TAccessControlEntity tce = new TAccessControlEntity();
1833    tce.setActions("R");
1834    tce.setNsName(defaultNameSpace);
1835    tce.setScope(TPermissionScope.NAMESPACE);
1836    tce.setUsername(fakeUser);
1837
1838    ThriftHBaseServiceHandler handler = createHandler();
1839    handler.grant(tce);
1840
1841    List<UserPermission> permissionList = AccessControlClient
1842      .getUserPermissions(UTIL.getConnection(), "@" + defaultNameSpace, fakeUser);
1843
1844    // we only grant one R permission
1845    assertEquals(permissionList.size(), 1);
1846
1847    Permission.Action[] actions = permissionList.get(0).getPermission().getActions();
1848    assertEquals(actions.length, 1);
1849    assertEquals(actions[0], Permission.Action.READ);
1850
1851    // revoke the permission
1852    handler.revoke(tce);
1853    permissionList = AccessControlClient.getUserPermissions(UTIL.getConnection(),
1854      "@" + defaultNameSpace, fakeUser);
1855
1856    // it should return an empty list
1857    assertEquals(0, permissionList.size());
1858  }
1859
1860  public static class DelayingRegionObserver implements RegionCoprocessor, RegionObserver {
1861    private static final Logger LOG = LoggerFactory.getLogger(DelayingRegionObserver.class);
1862    // sleep time in msec
1863    private long delayMillis;
1864
1865    @Override
1866    public Optional<RegionObserver> getRegionObserver() {
1867      return Optional.of(this);
1868    }
1869
1870    @Override
1871    public void start(CoprocessorEnvironment e) throws IOException {
1872      this.delayMillis = e.getConfiguration().getLong("delayingregionobserver.delay", 3000);
1873    }
1874
1875    @Override
1876    public void preGetOp(ObserverContext<? extends RegionCoprocessorEnvironment> e, Get get,
1877      List<Cell> results) throws IOException {
1878      try {
1879        long start = EnvironmentEdgeManager.currentTime();
1880        TimeUnit.MILLISECONDS.sleep(delayMillis);
1881        if (LOG.isTraceEnabled()) {
1882          LOG.trace("Slept for " + (EnvironmentEdgeManager.currentTime() - start) + " msec");
1883        }
1884      } catch (InterruptedException ie) {
1885        throw new InterruptedIOException("Interrupted while sleeping");
1886      }
1887    }
1888  }
1889}