001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.hamcrest.CoreMatchers.instanceOf;
021import static org.hamcrest.MatcherAssert.assertThat;
022import static org.junit.jupiter.api.Assertions.assertEquals;
023import static org.junit.jupiter.api.Assertions.assertFalse;
024import static org.junit.jupiter.api.Assertions.assertNotNull;
025import static org.junit.jupiter.api.Assertions.assertThrows;
026import static org.junit.jupiter.api.Assertions.assertTrue;
027
028import java.io.IOException;
029import java.util.ArrayList;
030import java.util.Arrays;
031import java.util.HashSet;
032import java.util.List;
033import java.util.NavigableMap;
034import org.apache.hadoop.hbase.Cell;
035import org.apache.hadoop.hbase.CellUtil;
036import org.apache.hadoop.hbase.HConstants;
037import org.apache.hadoop.hbase.KeyValue;
038import org.apache.hadoop.hbase.TableName;
039import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
040import org.apache.hadoop.hbase.util.Bytes;
041import org.apache.hadoop.hbase.util.Threads;
042import org.junit.jupiter.api.TestTemplate;
043import org.slf4j.Logger;
044import org.slf4j.LoggerFactory;
045
046import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
047
048/**
049 * Run tests that use the HBase clients; {@link Table}. Sets up the HBase mini cluster once at start
050 * and runs through all client tests. Each creates a table named for the method and does its stuff
051 * against that. Parameterized to run with different registry implementations.
052 */
053public class FromClientSideTest4 extends FromClientSideTestBase {
054
055  private static final Logger LOG = LoggerFactory.getLogger(FromClientSideTest4.class);
056
057  protected FromClientSideTest4(Class<? extends ConnectionRegistry> registryImpl,
058    int numHedgedReqs) {
059    super(registryImpl, numHedgedReqs);
060  }
061
062  /**
063   * Test batch operations with combination of valid and invalid args
064   */
065  @TestTemplate
066  public void testBatchOperationsWithErrors() throws Exception {
067    TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, 10);
068    try (Connection conn = getConnection(); Table foo = conn.getTable(tableName)) {
069      int NUM_OPS = 100;
070
071      // 1.1 Put with no column families (local validation, runtime exception)
072      List<Put> puts = new ArrayList<>(NUM_OPS);
073      for (int i = 0; i != NUM_OPS; i++) {
074        Put put = new Put(Bytes.toBytes(i));
075        puts.add(put);
076      }
077      assertThrows(IllegalArgumentException.class, () -> foo.put(puts));
078      assertEquals(NUM_OPS, puts.size());
079
080      // 1.2 Put with invalid column family
081      puts.clear();
082      for (int i = 0; i < NUM_OPS; i++) {
083        Put put = new Put(Bytes.toBytes(i));
084        put.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY, Bytes.toBytes(i));
085        puts.add(put);
086      }
087
088      RetriesExhaustedException ree =
089        assertThrows(RetriesExhaustedException.class, () -> foo.put(puts));
090      assertThat(ree.getCause(), instanceOf(NoSuchColumnFamilyException.class));
091
092      // 2.1 Get non-existent rows
093      List<Get> gets = new ArrayList<>(NUM_OPS);
094      for (int i = 0; i < NUM_OPS; i++) {
095        Get get = new Get(Bytes.toBytes(i));
096        gets.add(get);
097      }
098      Result[] getsResult = foo.get(gets);
099      assertNotNull(getsResult);
100      assertEquals(NUM_OPS, getsResult.length);
101      for (int i = 0; i < NUM_OPS; i++) {
102        Result getResult = getsResult[i];
103        if (i % 2 == 0) {
104          assertFalse(getResult.isEmpty());
105        } else {
106          assertTrue(getResult.isEmpty());
107        }
108      }
109
110      // 2.2 Get with invalid column family
111      gets.clear();
112      for (int i = 0; i < NUM_OPS; i++) {
113        Get get = new Get(Bytes.toBytes(i));
114        get.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY);
115        gets.add(get);
116      }
117      ree = assertThrows(RetriesExhaustedException.class, () -> foo.get(gets));
118      assertThat(ree.getCause(), instanceOf(NoSuchColumnFamilyException.class));
119
120      // 3.1 Delete with invalid column family
121      List<Delete> deletes = new ArrayList<>(NUM_OPS);
122      for (int i = 0; i < NUM_OPS; i++) {
123        Delete delete = new Delete(Bytes.toBytes(i));
124        delete.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY);
125        deletes.add(delete);
126      }
127      ree = assertThrows(RetriesExhaustedException.class, () -> foo.delete(deletes));
128      assertThat(ree.getCause(), instanceOf(NoSuchColumnFamilyException.class));
129
130      // all valid rows should have been deleted
131      gets.clear();
132      for (int i = 0; i < NUM_OPS; i++) {
133        Get get = new Get(Bytes.toBytes(i));
134        gets.add(get);
135      }
136      getsResult = foo.get(gets);
137      assertNotNull(getsResult);
138      assertEquals(NUM_OPS, getsResult.length);
139      for (Result getResult : getsResult) {
140        assertTrue(getResult.isEmpty());
141      }
142
143      // 3.2 Delete non-existent rows
144      deletes.clear();
145      for (int i = 0; i < NUM_OPS; i++) {
146        Delete delete = new Delete(Bytes.toBytes(i));
147        deletes.add(delete);
148      }
149      foo.delete(deletes);
150    }
151  }
152
153  //
154  // JIRA Testers
155  //
156
157  /**
158   * HBASE-867 If millions of columns in a column family, hbase scanner won't come up Test will
159   * create numRows rows, each with numColsPerRow columns (1 version each), and attempt to scan them
160   * all. To test at scale, up numColsPerRow to the millions (have not gotten that to work running
161   * as junit though)
162   */
163  @TestTemplate
164  public void testJiraTest867() throws Exception {
165    int numRows = 10;
166    int numColsPerRow = 2000;
167
168    byte[][] ROWS = makeN(ROW, numRows);
169    byte[][] QUALIFIERS = makeN(QUALIFIER, numColsPerRow);
170    TEST_UTIL.createTable(tableName, FAMILY);
171    try (Connection conn = getConnection(); Table ht = conn.getTable(tableName)) {
172      // Insert rows
173      for (int i = 0; i < numRows; i++) {
174        Put put = new Put(ROWS[i]);
175        put.setDurability(Durability.SKIP_WAL);
176        for (int j = 0; j < numColsPerRow; j++) {
177          put.addColumn(FAMILY, QUALIFIERS[j], QUALIFIERS[j]);
178        }
179        assertEquals(put.size(), numColsPerRow, "Put expected to contain " + numColsPerRow
180          + " columns but " + "only contains " + put.size());
181        ht.put(put);
182      }
183
184      // Get a row
185      Get get = new Get(ROWS[numRows - 1]);
186      Result result = ht.get(get);
187      assertNumKeys(result, numColsPerRow);
188      Cell[] keys = result.rawCells();
189      for (int i = 0; i < result.size(); i++) {
190        assertKey(keys[i], ROWS[numRows - 1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
191      }
192
193      // Scan the rows
194      Scan scan = new Scan();
195      try (ResultScanner scanner = ht.getScanner(scan)) {
196        int rowCount = 0;
197        while ((result = scanner.next()) != null) {
198          assertNumKeys(result, numColsPerRow);
199          Cell[] kvs = result.rawCells();
200          for (int i = 0; i < numColsPerRow; i++) {
201            assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
202          }
203          rowCount++;
204        }
205        assertEquals(rowCount, numRows,
206          "Expected to scan " + numRows + " rows but actually scanned " + rowCount + " rows");
207      }
208
209      // flush and try again
210
211      TEST_UTIL.flush();
212
213      // Get a row
214      get = new Get(ROWS[numRows - 1]);
215      result = ht.get(get);
216      assertNumKeys(result, numColsPerRow);
217      keys = result.rawCells();
218      for (int i = 0; i < result.size(); i++) {
219        assertKey(keys[i], ROWS[numRows - 1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
220      }
221
222      // Scan the rows
223      scan = new Scan();
224      try (ResultScanner scanner = ht.getScanner(scan)) {
225        int rowCount = 0;
226        while ((result = scanner.next()) != null) {
227          assertNumKeys(result, numColsPerRow);
228          Cell[] kvs = result.rawCells();
229          for (int i = 0; i < numColsPerRow; i++) {
230            assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
231          }
232          rowCount++;
233        }
234        assertEquals(rowCount, numRows,
235          "Expected to scan " + numRows + " rows but actually scanned " + rowCount + " rows");
236      }
237    }
238  }
239
240  /**
241   * HBASE-861 get with timestamp will return a value if there is a version with an earlier
242   * timestamp
243   */
244  @TestTemplate
245  public void testJiraTest861() throws Exception {
246    byte[][] VALUES = makeNAscii(VALUE, 7);
247    long[] STAMPS = makeStamps(7);
248    TEST_UTIL.createTable(tableName, FAMILY, 10);
249    try (Connection conn = getConnection(); Table ht = conn.getTable(tableName)) {
250      // Insert three versions
251      Put put = new Put(ROW);
252      put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
253      put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
254      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
255      ht.put(put);
256
257      // Get the middle value
258      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
259
260      // Try to get one version before (expect fail)
261      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
262
263      // Try to get one version after (expect fail)
264      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
265
266      // Try same from storefile
267      TEST_UTIL.flush();
268      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
269      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
270      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
271
272      // Insert two more versions surrounding others, into memstore
273      put = new Put(ROW);
274      put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
275      put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
276      ht.put(put);
277
278      // Check we can get everything we should and can't get what we shouldn't
279      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
280      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
281      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
282      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
283      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
284      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
285      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
286
287      // Try same from two storefiles
288      TEST_UTIL.flush();
289      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
290      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
291      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
292      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
293      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
294      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
295      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
296    }
297  }
298
299  /**
300   * HBASE-33 Add a HTable get/obtainScanner method that retrieves all versions of a particular
301   * column and row between two timestamps
302   */
303  @TestTemplate
304  public void testJiraTest33() throws Exception {
305    byte[][] VALUES = makeNAscii(VALUE, 7);
306    long[] STAMPS = makeStamps(7);
307    TEST_UTIL.createTable(tableName, FAMILY, 10);
308    try (Connection conn = getConnection(); Table ht = conn.getTable(tableName)) {
309      // Insert lots versions
310      Put put = new Put(ROW);
311      put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
312      put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
313      put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
314      put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
315      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
316      put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
317      ht.put(put);
318
319      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
320      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
321      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
322      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
323
324      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
325      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
326      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
327      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
328
329      // Try same from storefile
330      TEST_UTIL.flush();
331
332      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
333      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
334      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
335      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
336
337      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
338      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
339      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
340      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
341    }
342  }
343
344  /**
345   * HBASE-1014 commit(BatchUpdate) method should return timestamp
346   */
347  @TestTemplate
348  public void testJiraTest1014() throws Exception {
349    TEST_UTIL.createTable(tableName, FAMILY, 10);
350    try (Connection conn = getConnection(); Table ht = conn.getTable(tableName)) {
351      long manualStamp = 12345;
352
353      // Insert lots versions
354      Put put = new Put(ROW);
355      put.addColumn(FAMILY, QUALIFIER, manualStamp, VALUE);
356      ht.put(put);
357
358      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, manualStamp, VALUE);
359      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp - 1);
360      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp + 1);
361    }
362  }
363
364  /**
365   * HBASE-1182 Scan for columns > some timestamp
366   */
367  @TestTemplate
368  public void testJiraTest1182() throws Exception {
369    byte[][] VALUES = makeNAscii(VALUE, 7);
370    long[] STAMPS = makeStamps(7);
371    TEST_UTIL.createTable(tableName, FAMILY, 10);
372    try (Connection conn = getConnection(); Table ht = conn.getTable(tableName)) {
373      // Insert lots versions
374      Put put = new Put(ROW);
375      put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
376      put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
377      put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
378      put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
379      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
380      put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
381      ht.put(put);
382
383      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
384      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
385      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
386
387      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
388      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
389      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
390
391      // Try same from storefile
392      TEST_UTIL.flush();
393
394      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
395      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
396      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
397
398      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
399      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
400      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
401    }
402  }
403
404  /**
405   * HBASE-52 Add a means of scanning over all versions
406   */
407  @TestTemplate
408  public void testJiraTest52() throws Exception {
409    byte[][] VALUES = makeNAscii(VALUE, 7);
410    long[] STAMPS = makeStamps(7);
411    TEST_UTIL.createTable(tableName, FAMILY, 10);
412    try (Connection conn = getConnection(); Table ht = conn.getTable(tableName)) {
413      // Insert lots versions
414      Put put = new Put(ROW);
415      put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
416      put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
417      put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
418      put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
419      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
420      put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
421      ht.put(put);
422
423      getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
424
425      scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
426
427      // Try same from storefile
428      TEST_UTIL.flush();
429
430      getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
431
432      scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
433    }
434  }
435
436  @TestTemplate
437  @SuppressWarnings("checkstyle:MethodLength")
438  public void testDuplicateVersions() throws Exception {
439    long[] STAMPS = makeStamps(20);
440    byte[][] VALUES = makeNAscii(VALUE, 20);
441    TEST_UTIL.createTable(tableName, FAMILY, 10);
442    try (Connection conn = getConnection(); Table ht = conn.getTable(tableName)) {
443      // Insert 4 versions of same column
444      Put put = new Put(ROW);
445      put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
446      put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
447      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
448      put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
449      ht.put(put);
450
451      // Verify we can get each one properly
452      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
453      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
454      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
455      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
456      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
457      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
458      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
459      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
460
461      // Verify we don't accidentally get others
462      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
463      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
464      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
465      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
466      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
467      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
468
469      // Ensure maxVersions in query is respected
470      Get get = new Get(ROW);
471      get.addColumn(FAMILY, QUALIFIER);
472      get.readVersions(2);
473      Result result = ht.get(get);
474      assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
475        new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
476
477      Scan scan = new Scan().withStartRow(ROW);
478      scan.addColumn(FAMILY, QUALIFIER);
479      scan.readVersions(2);
480      result = getSingleScanResult(ht, scan);
481      assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
482        new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
483
484      // Flush and redo
485
486      TEST_UTIL.flush();
487
488      // Verify we can get each one properly
489      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
490      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
491      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
492      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
493      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
494      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
495      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
496      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
497
498      // Verify we don't accidentally get others
499      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
500      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
501      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
502      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
503      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
504      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
505
506      // Ensure maxVersions in query is respected
507      get = new Get(ROW);
508      get.addColumn(FAMILY, QUALIFIER);
509      get.readVersions(2);
510      result = ht.get(get);
511      assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
512        new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
513
514      scan = new Scan().withStartRow(ROW);
515      scan.addColumn(FAMILY, QUALIFIER);
516      scan.readVersions(2);
517      result = getSingleScanResult(ht, scan);
518      assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
519        new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
520
521      // Add some memstore and retest
522
523      // Insert 4 more versions of same column and a dupe
524      put = new Put(ROW);
525      put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
526      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
527      put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
528      put.addColumn(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
529      put.addColumn(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
530      ht.put(put);
531
532      // Ensure maxVersions in query is respected
533      get = new Get(ROW);
534      get.addColumn(FAMILY, QUALIFIER);
535      get.readVersions(7);
536      result = ht.get(get);
537      assertNResult(result, ROW, FAMILY, QUALIFIER,
538        new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
539        new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
540          VALUES[8] },
541        0, 6);
542
543      scan = new Scan().withStartRow(ROW);
544      scan.addColumn(FAMILY, QUALIFIER);
545      scan.readVersions(7);
546      result = getSingleScanResult(ht, scan);
547      assertNResult(result, ROW, FAMILY, QUALIFIER,
548        new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
549        new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
550          VALUES[8] },
551        0, 6);
552
553      get = new Get(ROW);
554      get.readVersions(7);
555      result = ht.get(get);
556      assertNResult(result, ROW, FAMILY, QUALIFIER,
557        new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
558        new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
559          VALUES[8] },
560        0, 6);
561
562      scan = new Scan().withStartRow(ROW);
563      scan.readVersions(7);
564      result = getSingleScanResult(ht, scan);
565      assertNResult(result, ROW, FAMILY, QUALIFIER,
566        new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
567        new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
568          VALUES[8] },
569        0, 6);
570
571      // Verify we can get each one properly
572      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
573      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
574      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
575      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
576      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
577      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
578      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
579      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
580
581      // Verify we don't accidentally get others
582      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
583      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
584      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
585      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
586
587      // Ensure maxVersions of table is respected
588
589      TEST_UTIL.flush();
590
591      // Insert 4 more versions of same column and a dupe
592      put = new Put(ROW);
593      put.addColumn(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
594      put.addColumn(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
595      put.addColumn(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
596      put.addColumn(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
597      ht.put(put);
598
599      get = new Get(ROW);
600      get.addColumn(FAMILY, QUALIFIER);
601      get.readVersions(Integer.MAX_VALUE);
602      result = ht.get(get);
603      assertNResult(result, ROW, FAMILY, QUALIFIER,
604        new long[] { STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9],
605          STAMPS[11], STAMPS[13], STAMPS[15] },
606        new byte[][] { VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9],
607          VALUES[11], VALUES[13], VALUES[15] },
608        0, 9);
609
610      scan = new Scan().withStartRow(ROW);
611      scan.addColumn(FAMILY, QUALIFIER);
612      scan.readVersions(Integer.MAX_VALUE);
613      result = getSingleScanResult(ht, scan);
614      assertNResult(result, ROW, FAMILY, QUALIFIER,
615        new long[] { STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9],
616          STAMPS[11], STAMPS[13], STAMPS[15] },
617        new byte[][] { VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9],
618          VALUES[11], VALUES[13], VALUES[15] },
619        0, 9);
620
621      // Delete a version in the memstore and a version in a storefile
622      Delete delete = new Delete(ROW);
623      delete.addColumn(FAMILY, QUALIFIER, STAMPS[11]);
624      delete.addColumn(FAMILY, QUALIFIER, STAMPS[7]);
625      ht.delete(delete);
626
627      // Test that it's gone
628      get = new Get(ROW);
629      get.addColumn(FAMILY, QUALIFIER);
630      get.readVersions(Integer.MAX_VALUE);
631      result = ht.get(get);
632      assertNResult(result, ROW, FAMILY, QUALIFIER,
633        new long[] { STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8],
634          STAMPS[9], STAMPS[13], STAMPS[15] },
635        new byte[][] { VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8],
636          VALUES[9], VALUES[13], VALUES[15] },
637        0, 9);
638
639      scan = new Scan().withStartRow(ROW);
640      scan.addColumn(FAMILY, QUALIFIER);
641      scan.readVersions(Integer.MAX_VALUE);
642      result = getSingleScanResult(ht, scan);
643      assertNResult(result, ROW, FAMILY, QUALIFIER,
644        new long[] { STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8],
645          STAMPS[9], STAMPS[13], STAMPS[15] },
646        new byte[][] { VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8],
647          VALUES[9], VALUES[13], VALUES[15] },
648        0, 9);
649    }
650  }
651
652  @TestTemplate
653  public void testUpdates() throws Exception {
654    TEST_UTIL.createTable(tableName, FAMILY, 10);
655    try (Connection conn = getConnection(); Table hTable = conn.getTable(tableName)) {
656      // Write a column with values at timestamp 1, 2 and 3
657      byte[] row = Bytes.toBytes("row1");
658      byte[] qualifier = Bytes.toBytes("myCol");
659      Put put = new Put(row);
660      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
661      hTable.put(put);
662
663      put = new Put(row);
664      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
665      hTable.put(put);
666
667      put = new Put(row);
668      put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
669      hTable.put(put);
670
671      Get get = new Get(row);
672      get.addColumn(FAMILY, qualifier);
673      get.readAllVersions();
674
675      // Check that the column indeed has the right values at timestamps 1 and
676      // 2
677      Result result = hTable.get(get);
678      NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier);
679      assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
680      assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
681
682      // Update the value at timestamp 1
683      put = new Put(row);
684      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
685      hTable.put(put);
686
687      // Update the value at timestamp 2
688      put = new Put(row);
689      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
690      hTable.put(put);
691
692      // Check that the values at timestamp 2 and 1 got updated
693      result = hTable.get(get);
694      navigableMap = result.getMap().get(FAMILY).get(qualifier);
695      assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
696      assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
697    }
698  }
699
700  @TestTemplate
701  public void testUpdatesWithMajorCompaction() throws Exception {
702    TEST_UTIL.createTable(tableName, FAMILY, 10);
703    try (Connection conn = getConnection(); Table hTable = conn.getTable(tableName);
704      Admin admin = conn.getAdmin()) {
705      // Write a column with values at timestamp 1, 2 and 3
706      byte[] row = Bytes.toBytes("row2");
707      byte[] qualifier = Bytes.toBytes("myCol");
708      Put put = new Put(row);
709      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
710      hTable.put(put);
711
712      put = new Put(row);
713      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
714      hTable.put(put);
715
716      put = new Put(row);
717      put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
718      hTable.put(put);
719
720      Get get = new Get(row);
721      get.addColumn(FAMILY, qualifier);
722      get.readAllVersions();
723
724      // Check that the column indeed has the right values at timestamps 1 and
725      // 2
726      Result result = hTable.get(get);
727      NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier);
728      assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
729      assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
730
731      // Trigger a major compaction
732      admin.flush(tableName);
733      admin.majorCompact(tableName);
734      Thread.sleep(6000);
735
736      // Update the value at timestamp 1
737      put = new Put(row);
738      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
739      hTable.put(put);
740
741      // Update the value at timestamp 2
742      put = new Put(row);
743      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
744      hTable.put(put);
745
746      // Trigger a major compaction
747      admin.flush(tableName);
748      admin.majorCompact(tableName);
749      Thread.sleep(6000);
750
751      // Check that the values at timestamp 2 and 1 got updated
752      result = hTable.get(get);
753      navigableMap = result.getMap().get(FAMILY).get(qualifier);
754      assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
755      assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
756    }
757  }
758
759  @TestTemplate
760  public void testMajorCompactionBetweenTwoUpdates() throws Exception {
761    TEST_UTIL.createTable(tableName, FAMILY, 10);
762    try (Connection conn = getConnection(); Table hTable = conn.getTable(tableName);
763      Admin admin = conn.getAdmin()) {
764      // Write a column with values at timestamp 1, 2 and 3
765      byte[] row = Bytes.toBytes("row3");
766      byte[] qualifier = Bytes.toBytes("myCol");
767      Put put = new Put(row);
768      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
769      hTable.put(put);
770
771      put = new Put(row);
772      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
773      hTable.put(put);
774
775      put = new Put(row);
776      put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
777      hTable.put(put);
778
779      Get get = new Get(row);
780      get.addColumn(FAMILY, qualifier);
781      get.readAllVersions();
782
783      // Check that the column indeed has the right values at timestamps 1 and
784      // 2
785      Result result = hTable.get(get);
786      NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier);
787      assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
788      assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
789
790      // Trigger a major compaction
791      admin.flush(tableName);
792      admin.majorCompact(tableName);
793      Thread.sleep(6000);
794
795      // Update the value at timestamp 1
796      put = new Put(row);
797      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
798      hTable.put(put);
799
800      // Trigger a major compaction
801      admin.flush(tableName);
802      admin.majorCompact(tableName);
803      Thread.sleep(6000);
804
805      // Update the value at timestamp 2
806      put = new Put(row);
807      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
808      hTable.put(put);
809
810      // Trigger a major compaction
811      admin.flush(tableName);
812      admin.majorCompact(tableName);
813      Thread.sleep(6000);
814
815      // Check that the values at timestamp 2 and 1 got updated
816      result = hTable.get(get);
817      navigableMap = result.getMap().get(FAMILY).get(qualifier);
818
819      assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
820      assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
821    }
822  }
823
824  @TestTemplate
825  public void testGetEmptyTable() throws IOException {
826    TEST_UTIL.createTable(tableName, FAMILY);
827    try (Connection conn = getConnection(); Table table = conn.getTable(tableName)) {
828      Get get = new Get(ROW);
829      get.addFamily(FAMILY);
830      Result r = table.get(get);
831      assertTrue(r.isEmpty());
832    }
833  }
834
835  @TestTemplate
836  public void testGetNullQualifier() throws IOException {
837    TEST_UTIL.createTable(tableName, FAMILY);
838    try (Connection conn = getConnection(); Table table = conn.getTable(tableName)) {
839      Put put = new Put(ROW);
840      put.addColumn(FAMILY, QUALIFIER, VALUE);
841      table.put(put);
842
843      put = new Put(ROW);
844      put.addColumn(FAMILY, null, VALUE);
845      table.put(put);
846      LOG.info("Row put");
847
848      Get get = new Get(ROW);
849      get.addColumn(FAMILY, null);
850      Result r = table.get(get);
851      assertEquals(1, r.size());
852
853      get = new Get(ROW);
854      get.addFamily(FAMILY);
855      r = table.get(get);
856      assertEquals(2, r.size());
857    }
858  }
859
860  @TestTemplate
861  public void testGetNonExistentRow() throws IOException {
862    TEST_UTIL.createTable(tableName, FAMILY);
863    try (Connection conn = getConnection(); Table table = conn.getTable(tableName)) {
864      Put put = new Put(ROW);
865      put.addColumn(FAMILY, QUALIFIER, VALUE);
866      table.put(put);
867      LOG.info("Row put");
868
869      Get get = new Get(ROW);
870      get.addFamily(FAMILY);
871      Result r = table.get(get);
872      assertFalse(r.isEmpty());
873      System.out.println("Row retrieved successfully");
874
875      byte[] missingrow = Bytes.toBytes("missingrow");
876      get = new Get(missingrow);
877      get.addFamily(FAMILY);
878      r = table.get(get);
879      assertTrue(r.isEmpty());
880      LOG.info("Row missing as it should be");
881    }
882  }
883
884  @TestTemplate
885  public void testPut() throws IOException {
886    final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
887    final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
888    final byte[] row1 = Bytes.toBytes("row1");
889    final byte[] row2 = Bytes.toBytes("row2");
890    final byte[] value = Bytes.toBytes("abcd");
891    TEST_UTIL.createTable(tableName, new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY });
892    try (Connection conn = getConnection(); Table table = conn.getTable(tableName)) {
893      Put put = new Put(row1);
894      put.addColumn(CONTENTS_FAMILY, null, value);
895      table.put(put);
896
897      put = new Put(row2);
898      put.addColumn(CONTENTS_FAMILY, null, value);
899
900      assertEquals(1, put.size());
901      assertEquals(1, put.getFamilyCellMap().get(CONTENTS_FAMILY).size());
902
903      // KeyValue v1 expectation. Cast for now until we go all Cell all the time. TODO
904      KeyValue kv = (KeyValue) put.getFamilyCellMap().get(CONTENTS_FAMILY).get(0);
905
906      assertTrue(Bytes.equals(CellUtil.cloneFamily(kv), CONTENTS_FAMILY));
907      // will it return null or an empty byte array?
908      assertTrue(Bytes.equals(CellUtil.cloneQualifier(kv), new byte[0]));
909
910      assertTrue(Bytes.equals(CellUtil.cloneValue(kv), value));
911
912      table.put(put);
913
914      Scan scan = new Scan();
915      scan.addColumn(CONTENTS_FAMILY, null);
916      try (ResultScanner scanner = table.getScanner(scan)) {
917        for (Result r : scanner) {
918          for (Cell key : r.rawCells()) {
919            System.out.println(Bytes.toString(r.getRow()) + ": " + key.toString());
920          }
921        }
922      }
923    }
924  }
925
926  @TestTemplate
927  public void testPutNoCF() throws IOException {
928    final byte[] BAD_FAM = Bytes.toBytes("BAD_CF");
929    final byte[] VAL = Bytes.toBytes(100);
930    TEST_UTIL.createTable(tableName, FAMILY);
931    try (Connection conn = getConnection(); Table table = conn.getTable(tableName)) {
932      assertThrows(NoSuchColumnFamilyException.class,
933        () -> table.put(new Put(ROW).addColumn(BAD_FAM, QUALIFIER, VAL)),
934        "Should throw NoSuchColumnFamilyException");
935    }
936  }
937
938  @TestTemplate
939  public void testRowsPut() throws IOException {
940    final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
941    final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
942    final int NB_BATCH_ROWS = 10;
943    final byte[] value = Bytes.toBytes("abcd");
944    TEST_UTIL.createTable(tableName, new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY });
945    try (Connection conn = getConnection(); Table table = conn.getTable(tableName)) {
946      ArrayList<Put> rowsUpdate = new ArrayList<>();
947      for (int i = 0; i < NB_BATCH_ROWS; i++) {
948        byte[] row = Bytes.toBytes("row" + i);
949        Put put = new Put(row);
950        put.setDurability(Durability.SKIP_WAL);
951        put.addColumn(CONTENTS_FAMILY, null, value);
952        rowsUpdate.add(put);
953      }
954      table.put(rowsUpdate);
955      Scan scan = new Scan();
956      scan.addFamily(CONTENTS_FAMILY);
957      try (ResultScanner scanner = table.getScanner(scan)) {
958        int nbRows = Iterables.size(scanner);
959        assertEquals(NB_BATCH_ROWS, nbRows);
960      }
961    }
962  }
963
964  @TestTemplate
965  public void testRowsPutBufferedManyManyFlushes() throws IOException {
966    final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
967    final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
968    final byte[] value = Bytes.toBytes("abcd");
969    final int NB_BATCH_ROWS = 10;
970    TEST_UTIL.createTable(tableName, new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY });
971    try (Connection conn = getConnection(); Table table = conn.getTable(tableName)) {
972      ArrayList<Put> rowsUpdate = new ArrayList<>();
973      for (int i = 0; i < NB_BATCH_ROWS * 10; i++) {
974        byte[] row = Bytes.toBytes("row" + i);
975        Put put = new Put(row);
976        put.setDurability(Durability.SKIP_WAL);
977        put.addColumn(CONTENTS_FAMILY, null, value);
978        rowsUpdate.add(put);
979      }
980      table.put(rowsUpdate);
981
982      Scan scan = new Scan();
983      scan.addFamily(CONTENTS_FAMILY);
984      try (ResultScanner scanner = table.getScanner(scan)) {
985        int nbRows = Iterables.size(scanner);
986        assertEquals(NB_BATCH_ROWS * 10, nbRows);
987      }
988    }
989  }
990
991  /**
992   * test for HBASE-737
993   */
994  @TestTemplate
995  public void testHBase737() throws IOException {
996    final byte[] FAM1 = Bytes.toBytes("fam1");
997    final byte[] FAM2 = Bytes.toBytes("fam2");
998    TEST_UTIL.createTable(tableName, new byte[][] { FAM1, FAM2 });
999    // Open table
1000    try (Connection conn = getConnection(); Table table = conn.getTable(tableName)) {
1001      // Insert some values
1002      Put put = new Put(ROW);
1003      put.addColumn(FAM1, Bytes.toBytes("letters"), Bytes.toBytes("abcdefg"));
1004      table.put(put);
1005      Threads.sleepWithoutInterrupt(1000);
1006
1007      put = new Put(ROW);
1008      put.addColumn(FAM1, Bytes.toBytes("numbers"), Bytes.toBytes("123456"));
1009      table.put(put);
1010
1011      Threads.sleepWithoutInterrupt(1000);
1012
1013      put = new Put(ROW);
1014      put.addColumn(FAM2, Bytes.toBytes("letters"), Bytes.toBytes("hijklmnop"));
1015      table.put(put);
1016
1017      long[] times = new long[3];
1018
1019      // First scan the memstore
1020
1021      Scan scan = new Scan();
1022      scan.addFamily(FAM1);
1023      scan.addFamily(FAM2);
1024      try (ResultScanner s = table.getScanner(scan)) {
1025        int index = 0;
1026        Result r;
1027        while ((r = s.next()) != null) {
1028          for (Cell key : r.rawCells()) {
1029            times[index++] = key.getTimestamp();
1030          }
1031        }
1032      }
1033      for (int i = 0; i < times.length - 1; i++) {
1034        for (int j = i + 1; j < times.length; j++) {
1035          assertTrue(times[j] > times[i]);
1036        }
1037      }
1038
1039      // Flush data to disk and try again
1040      TEST_UTIL.flush();
1041
1042      // Reset times
1043      Arrays.fill(times, 0);
1044
1045      Threads.sleepWithoutInterrupt(1000);
1046
1047      scan = new Scan();
1048      scan.addFamily(FAM1);
1049      scan.addFamily(FAM2);
1050      try (ResultScanner s = table.getScanner(scan)) {
1051        int index = 0;
1052        Result r = null;
1053        while ((r = s.next()) != null) {
1054          for (Cell key : r.rawCells()) {
1055            times[index++] = key.getTimestamp();
1056          }
1057        }
1058        for (int i = 0; i < times.length - 1; i++) {
1059          for (int j = i + 1; j < times.length; j++) {
1060            assertTrue(times[j] > times[i]);
1061          }
1062        }
1063      }
1064    }
1065  }
1066
1067  @TestTemplate
1068  public void testListTables() throws IOException {
1069    final String testTableName = tableName.toString();
1070    final TableName tableName1 = TableName.valueOf(testTableName + "1");
1071    final TableName tableName2 = TableName.valueOf(testTableName + "2");
1072    final TableName tableName3 = TableName.valueOf(testTableName + "3");
1073    TableName[] tables = new TableName[] { tableName1, tableName2, tableName3 };
1074    for (TableName table : tables) {
1075      TEST_UTIL.createTable(table, FAMILY);
1076    }
1077    try (Connection conn = getConnection(); Admin admin = conn.getAdmin()) {
1078      List<TableDescriptor> ts = admin.listTableDescriptors();
1079      HashSet<TableDescriptor> result = new HashSet<>(ts);
1080      int size = result.size();
1081      assertTrue(size >= tables.length);
1082      for (TableName table : tables) {
1083        boolean found = false;
1084        for (TableDescriptor t : ts) {
1085          if (t.getTableName().equals(table)) {
1086            found = true;
1087            break;
1088          }
1089        }
1090        assertTrue(found, "Not found: " + table);
1091      }
1092    }
1093  }
1094
1095  @TestTemplate
1096  public void testMiscHTableStuff() throws IOException {
1097    final String testTableName = tableName.toString();
1098    final TableName tableAname = TableName.valueOf(testTableName + "A");
1099    final TableName tableBname = TableName.valueOf(testTableName + "B");
1100    final byte[] attrName = Bytes.toBytes("TESTATTR");
1101    final byte[] attrValue = Bytes.toBytes("somevalue");
1102    byte[] value = Bytes.toBytes("value");
1103    TEST_UTIL.createTable(tableAname, HConstants.CATALOG_FAMILY);
1104    TEST_UTIL.createTable(tableBname, HConstants.CATALOG_FAMILY);
1105    try (Connection conn = getConnection(); Table a = conn.getTable(tableAname);
1106      Table b = conn.getTable(tableBname)) {
1107      Put put = new Put(ROW);
1108      put.addColumn(HConstants.CATALOG_FAMILY, null, value);
1109      a.put(put);
1110
1111      // open a new connection to A and a connection to b
1112      try (Connection c = getConnection(); Table newA = c.getTable(tableAname)) {
1113        // copy data from A to B
1114        Scan scan = new Scan();
1115        scan.addFamily(HConstants.CATALOG_FAMILY);
1116        try (ResultScanner s = newA.getScanner(scan)) {
1117          for (Result r : s) {
1118            put = new Put(r.getRow());
1119            put.setDurability(Durability.SKIP_WAL);
1120            for (Cell kv : r.rawCells()) {
1121              put.add(kv);
1122            }
1123            b.put(put);
1124          }
1125        }
1126      }
1127
1128      // Opening a new connection to A will cause the tables to be reloaded
1129      try (Connection c = getConnection(); Table anotherA = c.getTable(tableAname)) {
1130        Get get = new Get(ROW);
1131        get.addFamily(HConstants.CATALOG_FAMILY);
1132        anotherA.get(get);
1133      }
1134
1135      // We can still access A through newA because it has the table information
1136      // cached. And if it needs to recalibrate, that will cause the information
1137      // to be reloaded.
1138
1139      // Test user metadata
1140      Admin admin = TEST_UTIL.getAdmin();
1141      // make a modifiable descriptor
1142      TableDescriptor desc = a.getDescriptor();
1143      // offline the table
1144      admin.disableTable(tableAname);
1145      // add a user attribute to HTD
1146      TableDescriptorBuilder builder =
1147        TableDescriptorBuilder.newBuilder(desc).setValue(attrName, attrValue);
1148      // add a user attribute to HCD
1149      for (ColumnFamilyDescriptor c : desc.getColumnFamilies()) {
1150        builder.modifyColumnFamily(
1151          ColumnFamilyDescriptorBuilder.newBuilder(c).setValue(attrName, attrValue).build());
1152      }
1153      // update metadata for all regions of this table
1154      admin.modifyTable(builder.build());
1155      // enable the table
1156      admin.enableTable(tableAname);
1157
1158      // Test that attribute changes were applied
1159      desc = a.getDescriptor();
1160      assertEquals(desc.getTableName(), tableAname, "wrong table descriptor returned");
1161      // check HTD attribute
1162      value = desc.getValue(attrName);
1163      assertNotNull(value, "missing HTD attribute value");
1164      assertFalse(Bytes.compareTo(value, attrValue) != 0, "HTD attribute value is incorrect");
1165      // check HCD attribute
1166      for (ColumnFamilyDescriptor c : desc.getColumnFamilies()) {
1167        value = c.getValue(attrName);
1168        assertNotNull(value, "missing HCD attribute value");
1169        assertFalse(Bytes.compareTo(value, attrValue) != 0, "HCD attribute value is incorrect");
1170      }
1171    }
1172  }
1173}