Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.hamcrest.CoreMatchers.instanceOf;
021import static org.hamcrest.MatcherAssert.assertThat;
022import static org.junit.Assert.assertEquals;
023import static org.junit.Assert.assertFalse;
024import static org.junit.Assert.assertNotNull;
025import static org.junit.Assert.assertTrue;
026import static org.junit.Assert.fail;
027
028import java.io.IOException;
029import java.util.ArrayList;
030import java.util.Arrays;
031import java.util.Collection;
032import java.util.EnumSet;
033import java.util.HashSet;
034import java.util.List;
035import java.util.NavigableMap;
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.hbase.Cell;
038import org.apache.hadoop.hbase.CellUtil;
039import org.apache.hadoop.hbase.ClusterMetrics.Option;
040import org.apache.hadoop.hbase.HBaseClassTestRule;
041import org.apache.hadoop.hbase.HColumnDescriptor;
042import org.apache.hadoop.hbase.HConstants;
043import org.apache.hadoop.hbase.HTableDescriptor;
044import org.apache.hadoop.hbase.KeyValue;
045import org.apache.hadoop.hbase.MiniHBaseCluster;
046import org.apache.hadoop.hbase.TableName;
047import org.apache.hadoop.hbase.TableNameTestRule;
048import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
049import org.apache.hadoop.hbase.master.LoadBalancer;
050import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
051import org.apache.hadoop.hbase.testclassification.ClientTests;
052import org.apache.hadoop.hbase.testclassification.LargeTests;
053import org.apache.hadoop.hbase.util.Bytes;
054import org.junit.AfterClass;
055import org.junit.Assume;
056import org.junit.ClassRule;
057import org.junit.Rule;
058import org.junit.Test;
059import org.junit.experimental.categories.Category;
060import org.junit.runner.RunWith;
061import org.junit.runners.Parameterized;
062import org.slf4j.Logger;
063import org.slf4j.LoggerFactory;
064
065import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
066
067/**
068 * Run tests that use the HBase clients; {@link Table}. Sets up the HBase mini cluster once at start
069 * and runs through all client tests. Each creates a table named for the method and does its stuff
070 * against that. Parameterized to run with different registry implementations.
071 */
072@Category({ LargeTests.class, ClientTests.class })
073@SuppressWarnings("deprecation")
074@RunWith(Parameterized.class)
075public class TestFromClientSide4 extends FromClientSideBase {
076  private static final Logger LOG = LoggerFactory.getLogger(TestFromClientSide4.class);
077  @ClassRule
078  public static final HBaseClassTestRule CLASS_RULE =
079    HBaseClassTestRule.forClass(TestFromClientSide4.class);
080  @Rule
081  public TableNameTestRule name = new TableNameTestRule();
082
083  // To keep the child classes happy.
084  TestFromClientSide4() {
085  }
086
087  public TestFromClientSide4(Class registry, int numHedgedReqs) throws Exception {
088    initialize(registry, numHedgedReqs, MultiRowMutationEndpoint.class);
089  }
090
091  @Parameterized.Parameters
092  public static Collection parameters() {
093    return Arrays.asList(new Object[][] { { MasterRegistry.class, 1 }, { MasterRegistry.class, 2 },
094      { ZKConnectionRegistry.class, 1 } });
095  }
096
097  @AfterClass
098  public static void tearDownAfterClass() throws Exception {
099    afterClass();
100  }
101
102  /**
103   * Test batch operations with combination of valid and invalid args
104   */
105  @Test
106  public void testBatchOperationsWithErrors() throws Exception {
107    final TableName tableName = name.getTableName();
108    try (Table foo = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, 10)) {
109
110      int NUM_OPS = 100;
111
112      // 1.1 Put with no column families (local validation, runtime exception)
113      List<Put> puts = new ArrayList<>(NUM_OPS);
114      for (int i = 0; i != NUM_OPS; i++) {
115        Put put = new Put(Bytes.toBytes(i));
116        puts.add(put);
117      }
118
119      try {
120        foo.put(puts);
121        fail();
122      } catch (IllegalArgumentException e) {
123        // expected
124        assertEquals(NUM_OPS, puts.size());
125      }
126
127      // 1.2 Put with invalid column family
128      puts.clear();
129      for (int i = 0; i < NUM_OPS; i++) {
130        Put put = new Put(Bytes.toBytes(i));
131        put.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY, Bytes.toBytes(i));
132        puts.add(put);
133      }
134
135      try {
136        foo.put(puts);
137        fail();
138      } catch (RetriesExhaustedException e) {
139        if (e instanceof RetriesExhaustedWithDetailsException) {
140          assertThat(((RetriesExhaustedWithDetailsException) e).exceptions.get(0),
141            instanceOf(NoSuchColumnFamilyException.class));
142        } else {
143          assertThat(e.getCause(), instanceOf(NoSuchColumnFamilyException.class));
144        }
145      }
146
147      // 2.1 Get non-existent rows
148      List<Get> gets = new ArrayList<>(NUM_OPS);
149      for (int i = 0; i < NUM_OPS; i++) {
150        Get get = new Get(Bytes.toBytes(i));
151        gets.add(get);
152      }
153      Result[] getsResult = foo.get(gets);
154      assertNotNull(getsResult);
155      assertEquals(NUM_OPS, getsResult.length);
156      for (int i = 0; i < NUM_OPS; i++) {
157        Result getResult = getsResult[i];
158        if (i % 2 == 0) {
159          assertFalse(getResult.isEmpty());
160        } else {
161          assertTrue(getResult.isEmpty());
162        }
163      }
164
165      // 2.2 Get with invalid column family
166      gets.clear();
167      for (int i = 0; i < NUM_OPS; i++) {
168        Get get = new Get(Bytes.toBytes(i));
169        get.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY);
170        gets.add(get);
171      }
172      try {
173        foo.get(gets);
174        fail();
175      } catch (RetriesExhaustedException e) {
176        if (e instanceof RetriesExhaustedWithDetailsException) {
177          assertThat(((RetriesExhaustedWithDetailsException) e).exceptions.get(0),
178            instanceOf(NoSuchColumnFamilyException.class));
179        } else {
180          assertThat(e.getCause(), instanceOf(NoSuchColumnFamilyException.class));
181        }
182      }
183
184      // 3.1 Delete with invalid column family
185      List<Delete> deletes = new ArrayList<>(NUM_OPS);
186      for (int i = 0; i < NUM_OPS; i++) {
187        Delete delete = new Delete(Bytes.toBytes(i));
188        delete.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY);
189        deletes.add(delete);
190      }
191      try {
192        foo.delete(deletes);
193        fail();
194      } catch (RetriesExhaustedException e) {
195        if (e instanceof RetriesExhaustedWithDetailsException) {
196          assertThat(((RetriesExhaustedWithDetailsException) e).exceptions.get(0),
197            instanceOf(NoSuchColumnFamilyException.class));
198        } else {
199          assertThat(e.getCause(), instanceOf(NoSuchColumnFamilyException.class));
200        }
201      }
202
203      // all valid rows should have been deleted
204      gets.clear();
205      for (int i = 0; i < NUM_OPS; i++) {
206        Get get = new Get(Bytes.toBytes(i));
207        gets.add(get);
208      }
209      getsResult = foo.get(gets);
210      assertNotNull(getsResult);
211      assertEquals(NUM_OPS, getsResult.length);
212      for (Result getResult : getsResult) {
213        assertTrue(getResult.isEmpty());
214      }
215
216      // 3.2 Delete non-existent rows
217      deletes.clear();
218      for (int i = 0; i < NUM_OPS; i++) {
219        Delete delete = new Delete(Bytes.toBytes(i));
220        deletes.add(delete);
221      }
222      foo.delete(deletes);
223    }
224  }
225
226  //
227  // JIRA Testers
228  //
229
230  /**
231   * HBASE-867 If millions of columns in a column family, hbase scanner won't come up Test will
232   * create numRows rows, each with numColsPerRow columns (1 version each), and attempt to scan them
233   * all. To test at scale, up numColsPerRow to the millions (have not gotten that to work running
234   * as junit though)
235   */
236  @Test
237  public void testJiraTest867() throws Exception {
238    int numRows = 10;
239    int numColsPerRow = 2000;
240
241    final TableName tableName = name.getTableName();
242
243    byte[][] ROWS = makeN(ROW, numRows);
244    byte[][] QUALIFIERS = makeN(QUALIFIER, numColsPerRow);
245
246    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
247
248      // Insert rows
249
250      for (int i = 0; i < numRows; i++) {
251        Put put = new Put(ROWS[i]);
252        put.setDurability(Durability.SKIP_WAL);
253        for (int j = 0; j < numColsPerRow; j++) {
254          put.addColumn(FAMILY, QUALIFIERS[j], QUALIFIERS[j]);
255        }
256        assertEquals("Put expected to contain " + numColsPerRow + " columns but " + "only contains "
257          + put.size(), put.size(), numColsPerRow);
258        ht.put(put);
259      }
260
261      // Get a row
262      Get get = new Get(ROWS[numRows - 1]);
263      Result result = ht.get(get);
264      assertNumKeys(result, numColsPerRow);
265      Cell[] keys = result.rawCells();
266      for (int i = 0; i < result.size(); i++) {
267        assertKey(keys[i], ROWS[numRows - 1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
268      }
269
270      // Scan the rows
271      Scan scan = new Scan();
272      try (ResultScanner scanner = ht.getScanner(scan)) {
273        int rowCount = 0;
274        while ((result = scanner.next()) != null) {
275          assertNumKeys(result, numColsPerRow);
276          Cell[] kvs = result.rawCells();
277          for (int i = 0; i < numColsPerRow; i++) {
278            assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
279          }
280          rowCount++;
281        }
282        assertEquals(
283          "Expected to scan " + numRows + " rows but actually scanned " + rowCount + " rows",
284          rowCount, numRows);
285      }
286
287      // flush and try again
288
289      TEST_UTIL.flush();
290
291      // Get a row
292      get = new Get(ROWS[numRows - 1]);
293      result = ht.get(get);
294      assertNumKeys(result, numColsPerRow);
295      keys = result.rawCells();
296      for (int i = 0; i < result.size(); i++) {
297        assertKey(keys[i], ROWS[numRows - 1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
298      }
299
300      // Scan the rows
301      scan = new Scan();
302      try (ResultScanner scanner = ht.getScanner(scan)) {
303        int rowCount = 0;
304        while ((result = scanner.next()) != null) {
305          assertNumKeys(result, numColsPerRow);
306          Cell[] kvs = result.rawCells();
307          for (int i = 0; i < numColsPerRow; i++) {
308            assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
309          }
310          rowCount++;
311        }
312        assertEquals(
313          "Expected to scan " + numRows + " rows but actually scanned " + rowCount + " rows",
314          rowCount, numRows);
315      }
316    }
317  }
318
319  /**
320   * HBASE-861 get with timestamp will return a value if there is a version with an earlier
321   * timestamp
322   */
323  @Test
324  public void testJiraTest861() throws Exception {
325    final TableName tableName = name.getTableName();
326    byte[][] VALUES = makeNAscii(VALUE, 7);
327    long[] STAMPS = makeStamps(7);
328
329    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
330
331      // Insert three versions
332
333      Put put = new Put(ROW);
334      put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
335      put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
336      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
337      ht.put(put);
338
339      // Get the middle value
340      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
341
342      // Try to get one version before (expect fail)
343      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
344
345      // Try to get one version after (expect fail)
346      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
347
348      // Try same from storefile
349      TEST_UTIL.flush();
350      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
351      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
352      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
353
354      // Insert two more versions surrounding others, into memstore
355      put = new Put(ROW);
356      put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
357      put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
358      ht.put(put);
359
360      // Check we can get everything we should and can't get what we shouldn't
361      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
362      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
363      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
364      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
365      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
366      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
367      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
368
369      // Try same from two storefiles
370      TEST_UTIL.flush();
371      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
372      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
373      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
374      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
375      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
376      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
377      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
378    }
379  }
380
381  /**
382   * HBASE-33 Add a HTable get/obtainScanner method that retrieves all versions of a particular
383   * column and row between two timestamps
384   */
385  @Test
386  public void testJiraTest33() throws Exception {
387    final TableName tableName = name.getTableName();
388    byte[][] VALUES = makeNAscii(VALUE, 7);
389    long[] STAMPS = makeStamps(7);
390
391    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
392
393      // Insert lots versions
394
395      Put put = new Put(ROW);
396      put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
397      put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
398      put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
399      put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
400      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
401      put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
402      ht.put(put);
403
404      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
405      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
406      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
407      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
408
409      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
410      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
411      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
412      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
413
414      // Try same from storefile
415      TEST_UTIL.flush();
416
417      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
418      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
419      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
420      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
421
422      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
423      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
424      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
425      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
426    }
427  }
428
429  /**
430   * HBASE-1014 commit(BatchUpdate) method should return timestamp
431   */
432  @Test
433  public void testJiraTest1014() throws Exception {
434    final TableName tableName = name.getTableName();
435
436    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
437
438      long manualStamp = 12345;
439
440      // Insert lots versions
441
442      Put put = new Put(ROW);
443      put.addColumn(FAMILY, QUALIFIER, manualStamp, VALUE);
444      ht.put(put);
445
446      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, manualStamp, VALUE);
447      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp - 1);
448      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp + 1);
449    }
450  }
451
452  /**
453   * HBASE-1182 Scan for columns > some timestamp
454   */
455  @Test
456  public void testJiraTest1182() throws Exception {
457    final TableName tableName = name.getTableName();
458    byte[][] VALUES = makeNAscii(VALUE, 7);
459    long[] STAMPS = makeStamps(7);
460
461    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
462
463      // Insert lots versions
464
465      Put put = new Put(ROW);
466      put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
467      put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
468      put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
469      put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
470      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
471      put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
472      ht.put(put);
473
474      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
475      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
476      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
477
478      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
479      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
480      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
481
482      // Try same from storefile
483      TEST_UTIL.flush();
484
485      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
486      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
487      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
488
489      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
490      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
491      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
492    }
493  }
494
495  /**
496   * HBASE-52 Add a means of scanning over all versions
497   */
498  @Test
499  public void testJiraTest52() throws Exception {
500    final TableName tableName = name.getTableName();
501    byte[][] VALUES = makeNAscii(VALUE, 7);
502    long[] STAMPS = makeStamps(7);
503
504    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
505
506      // Insert lots versions
507
508      Put put = new Put(ROW);
509      put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
510      put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
511      put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
512      put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
513      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
514      put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
515      ht.put(put);
516
517      getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
518
519      scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
520
521      // Try same from storefile
522      TEST_UTIL.flush();
523
524      getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
525
526      scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
527    }
528  }
529
530  @Test
531  @SuppressWarnings("checkstyle:MethodLength")
532  public void testDuplicateVersions() throws Exception {
533    final TableName tableName = name.getTableName();
534
535    long[] STAMPS = makeStamps(20);
536    byte[][] VALUES = makeNAscii(VALUE, 20);
537
538    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
539
540      // Insert 4 versions of same column
541      Put put = new Put(ROW);
542      put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
543      put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
544      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
545      put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
546      ht.put(put);
547
548      // Verify we can get each one properly
549      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
550      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
551      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
552      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
553      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
554      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
555      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
556      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
557
558      // Verify we don't accidentally get others
559      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
560      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
561      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
562      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
563      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
564      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
565
566      // Ensure maxVersions in query is respected
567      Get get = new Get(ROW);
568      get.addColumn(FAMILY, QUALIFIER);
569      get.readVersions(2);
570      Result result = ht.get(get);
571      assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
572        new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
573
574      Scan scan = new Scan(ROW);
575      scan.addColumn(FAMILY, QUALIFIER);
576      scan.setMaxVersions(2);
577      result = getSingleScanResult(ht, scan);
578      assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
579        new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
580
581      // Flush and redo
582
583      TEST_UTIL.flush();
584
585      // Verify we can get each one properly
586      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
587      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
588      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
589      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
590      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
591      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
592      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
593      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
594
595      // Verify we don't accidentally get others
596      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
597      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
598      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
599      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
600      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
601      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
602
603      // Ensure maxVersions in query is respected
604      get = new Get(ROW);
605      get.addColumn(FAMILY, QUALIFIER);
606      get.readVersions(2);
607      result = ht.get(get);
608      assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
609        new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
610
611      scan = new Scan(ROW);
612      scan.addColumn(FAMILY, QUALIFIER);
613      scan.setMaxVersions(2);
614      result = getSingleScanResult(ht, scan);
615      assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
616        new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
617
618      // Add some memstore and retest
619
620      // Insert 4 more versions of same column and a dupe
621      put = new Put(ROW);
622      put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
623      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
624      put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
625      put.addColumn(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
626      put.addColumn(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
627      ht.put(put);
628
629      // Ensure maxVersions in query is respected
630      get = new Get(ROW);
631      get.addColumn(FAMILY, QUALIFIER);
632      get.readVersions(7);
633      result = ht.get(get);
634      assertNResult(result, ROW, FAMILY, QUALIFIER,
635        new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
636        new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
637          VALUES[8] },
638        0, 6);
639
640      scan = new Scan(ROW);
641      scan.addColumn(FAMILY, QUALIFIER);
642      scan.setMaxVersions(7);
643      result = getSingleScanResult(ht, scan);
644      assertNResult(result, ROW, FAMILY, QUALIFIER,
645        new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
646        new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
647          VALUES[8] },
648        0, 6);
649
650      get = new Get(ROW);
651      get.readVersions(7);
652      result = ht.get(get);
653      assertNResult(result, ROW, FAMILY, QUALIFIER,
654        new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
655        new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
656          VALUES[8] },
657        0, 6);
658
659      scan = new Scan(ROW);
660      scan.setMaxVersions(7);
661      result = getSingleScanResult(ht, scan);
662      assertNResult(result, ROW, FAMILY, QUALIFIER,
663        new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
664        new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
665          VALUES[8] },
666        0, 6);
667
668      // Verify we can get each one properly
669      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
670      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
671      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
672      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
673      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
674      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
675      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
676      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
677
678      // Verify we don't accidentally get others
679      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
680      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
681      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
682      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
683
684      // Ensure maxVersions of table is respected
685
686      TEST_UTIL.flush();
687
688      // Insert 4 more versions of same column and a dupe
689      put = new Put(ROW);
690      put.addColumn(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
691      put.addColumn(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
692      put.addColumn(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
693      put.addColumn(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
694      ht.put(put);
695
696      get = new Get(ROW);
697      get.addColumn(FAMILY, QUALIFIER);
698      get.readVersions(Integer.MAX_VALUE);
699      result = ht.get(get);
700      assertNResult(result, ROW, FAMILY, QUALIFIER,
701        new long[] { STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9],
702          STAMPS[11], STAMPS[13], STAMPS[15] },
703        new byte[][] { VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9],
704          VALUES[11], VALUES[13], VALUES[15] },
705        0, 9);
706
707      scan = new Scan(ROW);
708      scan.addColumn(FAMILY, QUALIFIER);
709      scan.setMaxVersions(Integer.MAX_VALUE);
710      result = getSingleScanResult(ht, scan);
711      assertNResult(result, ROW, FAMILY, QUALIFIER,
712        new long[] { STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9],
713          STAMPS[11], STAMPS[13], STAMPS[15] },
714        new byte[][] { VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9],
715          VALUES[11], VALUES[13], VALUES[15] },
716        0, 9);
717
718      // Delete a version in the memstore and a version in a storefile
719      Delete delete = new Delete(ROW);
720      delete.addColumn(FAMILY, QUALIFIER, STAMPS[11]);
721      delete.addColumn(FAMILY, QUALIFIER, STAMPS[7]);
722      ht.delete(delete);
723
724      // Test that it's gone
725      get = new Get(ROW);
726      get.addColumn(FAMILY, QUALIFIER);
727      get.readVersions(Integer.MAX_VALUE);
728      result = ht.get(get);
729      assertNResult(result, ROW, FAMILY, QUALIFIER,
730        new long[] { STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8],
731          STAMPS[9], STAMPS[13], STAMPS[15] },
732        new byte[][] { VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8],
733          VALUES[9], VALUES[13], VALUES[15] },
734        0, 9);
735
736      scan = new Scan(ROW);
737      scan.addColumn(FAMILY, QUALIFIER);
738      scan.setMaxVersions(Integer.MAX_VALUE);
739      result = getSingleScanResult(ht, scan);
740      assertNResult(result, ROW, FAMILY, QUALIFIER,
741        new long[] { STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8],
742          STAMPS[9], STAMPS[13], STAMPS[15] },
743        new byte[][] { VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8],
744          VALUES[9], VALUES[13], VALUES[15] },
745        0, 9);
746    }
747  }
748
749  @Test
750  public void testUpdates() throws Exception {
751    final TableName tableName = name.getTableName();
752    try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
753
754      // Write a column with values at timestamp 1, 2 and 3
755      byte[] row = Bytes.toBytes("row1");
756      byte[] qualifier = Bytes.toBytes("myCol");
757      Put put = new Put(row);
758      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
759      hTable.put(put);
760
761      put = new Put(row);
762      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
763      hTable.put(put);
764
765      put = new Put(row);
766      put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
767      hTable.put(put);
768
769      Get get = new Get(row);
770      get.addColumn(FAMILY, qualifier);
771      get.readAllVersions();
772
773      // Check that the column indeed has the right values at timestamps 1 and
774      // 2
775      Result result = hTable.get(get);
776      NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier);
777      assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
778      assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
779
780      // Update the value at timestamp 1
781      put = new Put(row);
782      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
783      hTable.put(put);
784
785      // Update the value at timestamp 2
786      put = new Put(row);
787      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
788      hTable.put(put);
789
790      // Check that the values at timestamp 2 and 1 got updated
791      result = hTable.get(get);
792      navigableMap = result.getMap().get(FAMILY).get(qualifier);
793      assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
794      assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
795    }
796  }
797
798  @Test
799  public void testUpdatesWithMajorCompaction() throws Exception {
800    final TableName tableName = name.getTableName();
801    try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10);
802      Admin admin = TEST_UTIL.getAdmin()) {
803
804      // Write a column with values at timestamp 1, 2 and 3
805      byte[] row = Bytes.toBytes("row2");
806      byte[] qualifier = Bytes.toBytes("myCol");
807      Put put = new Put(row);
808      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
809      hTable.put(put);
810
811      put = new Put(row);
812      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
813      hTable.put(put);
814
815      put = new Put(row);
816      put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
817      hTable.put(put);
818
819      Get get = new Get(row);
820      get.addColumn(FAMILY, qualifier);
821      get.readAllVersions();
822
823      // Check that the column indeed has the right values at timestamps 1 and
824      // 2
825      Result result = hTable.get(get);
826      NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier);
827      assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
828      assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
829
830      // Trigger a major compaction
831      admin.flush(tableName);
832      admin.majorCompact(tableName);
833      Thread.sleep(6000);
834
835      // Update the value at timestamp 1
836      put = new Put(row);
837      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
838      hTable.put(put);
839
840      // Update the value at timestamp 2
841      put = new Put(row);
842      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
843      hTable.put(put);
844
845      // Trigger a major compaction
846      admin.flush(tableName);
847      admin.majorCompact(tableName);
848      Thread.sleep(6000);
849
850      // Check that the values at timestamp 2 and 1 got updated
851      result = hTable.get(get);
852      navigableMap = result.getMap().get(FAMILY).get(qualifier);
853      assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
854      assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
855    }
856  }
857
858  @Test
859  public void testMajorCompactionBetweenTwoUpdates() throws Exception {
860    final TableName tableName = name.getTableName();
861    try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10);
862      Admin admin = TEST_UTIL.getAdmin()) {
863
864      // Write a column with values at timestamp 1, 2 and 3
865      byte[] row = Bytes.toBytes("row3");
866      byte[] qualifier = Bytes.toBytes("myCol");
867      Put put = new Put(row);
868      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
869      hTable.put(put);
870
871      put = new Put(row);
872      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
873      hTable.put(put);
874
875      put = new Put(row);
876      put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
877      hTable.put(put);
878
879      Get get = new Get(row);
880      get.addColumn(FAMILY, qualifier);
881      get.readAllVersions();
882
883      // Check that the column indeed has the right values at timestamps 1 and
884      // 2
885      Result result = hTable.get(get);
886      NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier);
887      assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
888      assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
889
890      // Trigger a major compaction
891      admin.flush(tableName);
892      admin.majorCompact(tableName);
893      Thread.sleep(6000);
894
895      // Update the value at timestamp 1
896      put = new Put(row);
897      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
898      hTable.put(put);
899
900      // Trigger a major compaction
901      admin.flush(tableName);
902      admin.majorCompact(tableName);
903      Thread.sleep(6000);
904
905      // Update the value at timestamp 2
906      put = new Put(row);
907      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
908      hTable.put(put);
909
910      // Trigger a major compaction
911      admin.flush(tableName);
912      admin.majorCompact(tableName);
913      Thread.sleep(6000);
914
915      // Check that the values at timestamp 2 and 1 got updated
916      result = hTable.get(get);
917      navigableMap = result.getMap().get(FAMILY).get(qualifier);
918
919      assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
920      assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
921    }
922  }
923
924  @Test
925  public void testGet_EmptyTable() throws IOException {
926    try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
927      Get get = new Get(ROW);
928      get.addFamily(FAMILY);
929      Result r = table.get(get);
930      assertTrue(r.isEmpty());
931    }
932  }
933
934  @Test
935  public void testGet_NullQualifier() throws IOException {
936    try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
937      Put put = new Put(ROW);
938      put.addColumn(FAMILY, QUALIFIER, VALUE);
939      table.put(put);
940
941      put = new Put(ROW);
942      put.addColumn(FAMILY, null, VALUE);
943      table.put(put);
944      LOG.info("Row put");
945
946      Get get = new Get(ROW);
947      get.addColumn(FAMILY, null);
948      Result r = table.get(get);
949      assertEquals(1, r.size());
950
951      get = new Get(ROW);
952      get.addFamily(FAMILY);
953      r = table.get(get);
954      assertEquals(2, r.size());
955    }
956  }
957
958  @Test
959  public void testGet_NonExistentRow() throws IOException {
960    try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
961      Put put = new Put(ROW);
962      put.addColumn(FAMILY, QUALIFIER, VALUE);
963      table.put(put);
964      LOG.info("Row put");
965
966      Get get = new Get(ROW);
967      get.addFamily(FAMILY);
968      Result r = table.get(get);
969      assertFalse(r.isEmpty());
970      System.out.println("Row retrieved successfully");
971
972      byte[] missingrow = Bytes.toBytes("missingrow");
973      get = new Get(missingrow);
974      get.addFamily(FAMILY);
975      r = table.get(get);
976      assertTrue(r.isEmpty());
977      LOG.info("Row missing as it should be");
978    }
979  }
980
981  @Test
982  public void testPut() throws IOException {
983    final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
984    final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
985    final byte[] row1 = Bytes.toBytes("row1");
986    final byte[] row2 = Bytes.toBytes("row2");
987    final byte[] value = Bytes.toBytes("abcd");
988    try (Table table =
989      TEST_UTIL.createTable(name.getTableName(), new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY })) {
990      Put put = new Put(row1);
991      put.addColumn(CONTENTS_FAMILY, null, value);
992      table.put(put);
993
994      put = new Put(row2);
995      put.addColumn(CONTENTS_FAMILY, null, value);
996
997      assertEquals(1, put.size());
998      assertEquals(1, put.getFamilyCellMap().get(CONTENTS_FAMILY).size());
999
1000      // KeyValue v1 expectation. Cast for now until we go all Cell all the time. TODO
1001      KeyValue kv = (KeyValue) put.getFamilyCellMap().get(CONTENTS_FAMILY).get(0);
1002
1003      assertTrue(Bytes.equals(CellUtil.cloneFamily(kv), CONTENTS_FAMILY));
1004      // will it return null or an empty byte array?
1005      assertTrue(Bytes.equals(CellUtil.cloneQualifier(kv), new byte[0]));
1006
1007      assertTrue(Bytes.equals(CellUtil.cloneValue(kv), value));
1008
1009      table.put(put);
1010
1011      Scan scan = new Scan();
1012      scan.addColumn(CONTENTS_FAMILY, null);
1013      try (ResultScanner scanner = table.getScanner(scan)) {
1014        for (Result r : scanner) {
1015          for (Cell key : r.rawCells()) {
1016            System.out.println(Bytes.toString(r.getRow()) + ": " + key.toString());
1017          }
1018        }
1019      }
1020    }
1021  }
1022
1023  @Test
1024  public void testPutNoCF() throws IOException {
1025    final byte[] BAD_FAM = Bytes.toBytes("BAD_CF");
1026    final byte[] VAL = Bytes.toBytes(100);
1027    try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
1028      boolean caughtNSCFE = false;
1029
1030      try {
1031        Put p = new Put(ROW);
1032        p.addColumn(BAD_FAM, QUALIFIER, VAL);
1033        table.put(p);
1034      } catch (Exception e) {
1035        caughtNSCFE = e instanceof NoSuchColumnFamilyException;
1036      }
1037      assertTrue("Should throw NoSuchColumnFamilyException", caughtNSCFE);
1038    }
1039  }
1040
1041  @Test
1042  public void testRowsPut() throws IOException {
1043    final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
1044    final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
1045    final int NB_BATCH_ROWS = 10;
1046    final byte[] value = Bytes.toBytes("abcd");
1047    try (Table table =
1048      TEST_UTIL.createTable(name.getTableName(), new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY })) {
1049      ArrayList<Put> rowsUpdate = new ArrayList<>();
1050      for (int i = 0; i < NB_BATCH_ROWS; i++) {
1051        byte[] row = Bytes.toBytes("row" + i);
1052        Put put = new Put(row);
1053        put.setDurability(Durability.SKIP_WAL);
1054        put.addColumn(CONTENTS_FAMILY, null, value);
1055        rowsUpdate.add(put);
1056      }
1057      table.put(rowsUpdate);
1058      Scan scan = new Scan();
1059      scan.addFamily(CONTENTS_FAMILY);
1060      try (ResultScanner scanner = table.getScanner(scan)) {
1061        int nbRows = Iterables.size(scanner);
1062        assertEquals(NB_BATCH_ROWS, nbRows);
1063      }
1064    }
1065  }
1066
1067  @Test
1068  public void testRowsPutBufferedManyManyFlushes() throws IOException {
1069    final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
1070    final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
1071    final byte[] value = Bytes.toBytes("abcd");
1072    final int NB_BATCH_ROWS = 10;
1073    try (Table table =
1074      TEST_UTIL.createTable(name.getTableName(), new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY })) {
1075      ArrayList<Put> rowsUpdate = new ArrayList<>();
1076      for (int i = 0; i < NB_BATCH_ROWS * 10; i++) {
1077        byte[] row = Bytes.toBytes("row" + i);
1078        Put put = new Put(row);
1079        put.setDurability(Durability.SKIP_WAL);
1080        put.addColumn(CONTENTS_FAMILY, null, value);
1081        rowsUpdate.add(put);
1082      }
1083      table.put(rowsUpdate);
1084
1085      Scan scan = new Scan();
1086      scan.addFamily(CONTENTS_FAMILY);
1087      try (ResultScanner scanner = table.getScanner(scan)) {
1088        int nbRows = Iterables.size(scanner);
1089        assertEquals(NB_BATCH_ROWS * 10, nbRows);
1090      }
1091    }
1092  }
1093
1094  @Test
1095  public void testAddKeyValue() {
1096    final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
1097    final byte[] value = Bytes.toBytes("abcd");
1098    final byte[] row1 = Bytes.toBytes("row1");
1099    final byte[] row2 = Bytes.toBytes("row2");
1100    byte[] qualifier = Bytes.toBytes("qf1");
1101    Put put = new Put(row1);
1102
1103    // Adding KeyValue with the same row
1104    KeyValue kv = new KeyValue(row1, CONTENTS_FAMILY, qualifier, value);
1105    boolean ok = true;
1106    try {
1107      put.add(kv);
1108    } catch (IOException e) {
1109      ok = false;
1110    }
1111    assertTrue(ok);
1112
1113    // Adding KeyValue with the different row
1114    kv = new KeyValue(row2, CONTENTS_FAMILY, qualifier, value);
1115    ok = false;
1116    try {
1117      put.add(kv);
1118    } catch (IOException e) {
1119      ok = true;
1120    }
1121    assertTrue(ok);
1122  }
1123
1124  /**
1125   * test for HBASE-737
1126   */
1127  @Test
1128  public void testHBase737() throws IOException {
1129    final byte[] FAM1 = Bytes.toBytes("fam1");
1130    final byte[] FAM2 = Bytes.toBytes("fam2");
1131    // Open table
1132    try (Table table = TEST_UTIL.createTable(name.getTableName(), new byte[][] { FAM1, FAM2 })) {
1133      // Insert some values
1134      Put put = new Put(ROW);
1135      put.addColumn(FAM1, Bytes.toBytes("letters"), Bytes.toBytes("abcdefg"));
1136      table.put(put);
1137      try {
1138        Thread.sleep(1000);
1139      } catch (InterruptedException i) {
1140        // ignore
1141      }
1142
1143      put = new Put(ROW);
1144      put.addColumn(FAM1, Bytes.toBytes("numbers"), Bytes.toBytes("123456"));
1145      table.put(put);
1146
1147      try {
1148        Thread.sleep(1000);
1149      } catch (InterruptedException i) {
1150        // ignore
1151      }
1152
1153      put = new Put(ROW);
1154      put.addColumn(FAM2, Bytes.toBytes("letters"), Bytes.toBytes("hijklmnop"));
1155      table.put(put);
1156
1157      long[] times = new long[3];
1158
1159      // First scan the memstore
1160
1161      Scan scan = new Scan();
1162      scan.addFamily(FAM1);
1163      scan.addFamily(FAM2);
1164      try (ResultScanner s = table.getScanner(scan)) {
1165        int index = 0;
1166        Result r;
1167        while ((r = s.next()) != null) {
1168          for (Cell key : r.rawCells()) {
1169            times[index++] = key.getTimestamp();
1170          }
1171        }
1172      }
1173      for (int i = 0; i < times.length - 1; i++) {
1174        for (int j = i + 1; j < times.length; j++) {
1175          assertTrue(times[j] > times[i]);
1176        }
1177      }
1178
1179      // Flush data to disk and try again
1180      TEST_UTIL.flush();
1181
1182      // Reset times
1183      Arrays.fill(times, 0);
1184
1185      try {
1186        Thread.sleep(1000);
1187      } catch (InterruptedException i) {
1188        // ignore
1189      }
1190      scan = new Scan();
1191      scan.addFamily(FAM1);
1192      scan.addFamily(FAM2);
1193      try (ResultScanner s = table.getScanner(scan)) {
1194        int index = 0;
1195        Result r = null;
1196        while ((r = s.next()) != null) {
1197          for (Cell key : r.rawCells()) {
1198            times[index++] = key.getTimestamp();
1199          }
1200        }
1201        for (int i = 0; i < times.length - 1; i++) {
1202          for (int j = i + 1; j < times.length; j++) {
1203            assertTrue(times[j] > times[i]);
1204          }
1205        }
1206      }
1207    }
1208  }
1209
1210  @Test
1211  public void testListTables() throws IOException {
1212    final String testTableName = name.getTableName().toString();
1213    final TableName tableName1 = TableName.valueOf(testTableName + "1");
1214    final TableName tableName2 = TableName.valueOf(testTableName + "2");
1215    final TableName tableName3 = TableName.valueOf(testTableName + "3");
1216    TableName[] tables = new TableName[] { tableName1, tableName2, tableName3 };
1217    for (TableName table : tables) {
1218      TEST_UTIL.createTable(table, FAMILY);
1219    }
1220    try (Admin admin = TEST_UTIL.getAdmin()) {
1221      List<TableDescriptor> ts = admin.listTableDescriptors();
1222      HashSet<TableDescriptor> result = new HashSet<>(ts);
1223      int size = result.size();
1224      assertTrue(size >= tables.length);
1225      for (TableName table : tables) {
1226        boolean found = false;
1227        for (TableDescriptor t : ts) {
1228          if (t.getTableName().equals(table)) {
1229            found = true;
1230            break;
1231          }
1232        }
1233        assertTrue("Not found: " + table, found);
1234      }
1235    }
1236  }
1237
1238  /**
1239   * simple test that just executes parts of the client API that accept a pre-created Connection
1240   * instance
1241   */
1242  @Test
1243  public void testUnmanagedHConnection() throws IOException {
1244    final TableName tableName = name.getTableName();
1245    TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
1246    try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
1247      Table t = conn.getTable(tableName); Admin admin = conn.getAdmin()) {
1248      assertTrue(admin.tableExists(tableName));
1249      assertTrue(t.get(new Get(ROW)).isEmpty());
1250    }
1251  }
1252
1253  /**
1254   * test of that unmanaged HConnections are able to reconnect properly (see HBASE-5058)
1255   */
1256  @Test
1257  public void testUnmanagedHConnectionReconnect() throws Exception {
1258    Configuration conf = TEST_UTIL.getConfiguration();
1259    Class registryImpl = conf.getClass(HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY,
1260      ZKConnectionRegistry.class);
1261    // This test does not make sense for MasterRegistry since it stops the only master in the
1262    // cluster and starts a new master without populating the underlying config for the connection.
1263    Assume.assumeFalse(registryImpl.equals(MasterRegistry.class));
1264    final TableName tableName = name.getTableName();
1265    TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
1266    try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) {
1267      try (Table t = conn.getTable(tableName); Admin admin = conn.getAdmin()) {
1268        assertTrue(admin.tableExists(tableName));
1269        assertTrue(t.get(new Get(ROW)).isEmpty());
1270      }
1271
1272      // stop the master
1273      MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1274      cluster.stopMaster(0, false);
1275      cluster.waitOnMaster(0);
1276
1277      // start up a new master
1278      cluster.startMaster();
1279      assertTrue(cluster.waitForActiveAndReadyMaster());
1280
1281      // test that the same unmanaged connection works with a new
1282      // Admin and can connect to the new master;
1283      boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration());
1284      try (Admin admin = conn.getAdmin()) {
1285        assertTrue(admin.tableExists(tableName));
1286        assertEquals(
1287          admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().size(),
1288          SLAVES + (tablesOnMaster ? 1 : 0));
1289      }
1290    }
1291  }
1292
1293  @Test
1294  public void testMiscHTableStuff() throws IOException {
1295    final String testTableName = name.getTableName().toString();
1296    final TableName tableAname = TableName.valueOf(testTableName + "A");
1297    final TableName tableBname = TableName.valueOf(testTableName + "B");
1298    final byte[] attrName = Bytes.toBytes("TESTATTR");
1299    final byte[] attrValue = Bytes.toBytes("somevalue");
1300    byte[] value = Bytes.toBytes("value");
1301
1302    try (Table a = TEST_UTIL.createTable(tableAname, HConstants.CATALOG_FAMILY);
1303      Table b = TEST_UTIL.createTable(tableBname, HConstants.CATALOG_FAMILY)) {
1304      Put put = new Put(ROW);
1305      put.addColumn(HConstants.CATALOG_FAMILY, null, value);
1306      a.put(put);
1307
1308      // open a new connection to A and a connection to b
1309      try (Table newA = TEST_UTIL.getConnection().getTable(tableAname)) {
1310
1311        // copy data from A to B
1312        Scan scan = new Scan();
1313        scan.addFamily(HConstants.CATALOG_FAMILY);
1314        try (ResultScanner s = newA.getScanner(scan)) {
1315          for (Result r : s) {
1316            put = new Put(r.getRow());
1317            put.setDurability(Durability.SKIP_WAL);
1318            for (Cell kv : r.rawCells()) {
1319              put.add(kv);
1320            }
1321            b.put(put);
1322          }
1323        }
1324      }
1325
1326      // Opening a new connection to A will cause the tables to be reloaded
1327      try (Table anotherA = TEST_UTIL.getConnection().getTable(tableAname)) {
1328        Get get = new Get(ROW);
1329        get.addFamily(HConstants.CATALOG_FAMILY);
1330        anotherA.get(get);
1331      }
1332
1333      // We can still access A through newA because it has the table information
1334      // cached. And if it needs to recalibrate, that will cause the information
1335      // to be reloaded.
1336
1337      // Test user metadata
1338      Admin admin = TEST_UTIL.getAdmin();
1339      // make a modifiable descriptor
1340      HTableDescriptor desc = new HTableDescriptor(a.getDescriptor());
1341      // offline the table
1342      admin.disableTable(tableAname);
1343      // add a user attribute to HTD
1344      desc.setValue(attrName, attrValue);
1345      // add a user attribute to HCD
1346      for (HColumnDescriptor c : desc.getFamilies()) {
1347        c.setValue(attrName, attrValue);
1348      }
1349      // update metadata for all regions of this table
1350      admin.modifyTable(desc);
1351      // enable the table
1352      admin.enableTable(tableAname);
1353
1354      // Test that attribute changes were applied
1355      desc = new HTableDescriptor(a.getDescriptor());
1356      assertEquals("wrong table descriptor returned", desc.getTableName(), tableAname);
1357      // check HTD attribute
1358      value = desc.getValue(attrName);
1359      assertNotNull("missing HTD attribute value", value);
1360      assertFalse("HTD attribute value is incorrect", Bytes.compareTo(value, attrValue) != 0);
1361      // check HCD attribute
1362      for (HColumnDescriptor c : desc.getFamilies()) {
1363        value = c.getValue(attrName);
1364        assertNotNull("missing HCD attribute value", value);
1365        assertFalse("HCD attribute value is incorrect", Bytes.compareTo(value, attrValue) != 0);
1366      }
1367    }
1368  }
1369}