001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.hamcrest.CoreMatchers.instanceOf;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertNotNull;
024import static org.junit.Assert.assertThat;
025import static org.junit.Assert.assertTrue;
026import static org.junit.Assert.fail;
027import java.io.IOException;
028import java.util.ArrayList;
029import java.util.Arrays;
030import java.util.Collection;
031import java.util.EnumSet;
032import java.util.HashSet;
033import java.util.List;
034import java.util.NavigableMap;
035import org.apache.hadoop.conf.Configuration;
036import org.apache.hadoop.hbase.Cell;
037import org.apache.hadoop.hbase.CellUtil;
038import org.apache.hadoop.hbase.ClusterMetrics.Option;
039import org.apache.hadoop.hbase.HBaseClassTestRule;
040import org.apache.hadoop.hbase.HColumnDescriptor;
041import org.apache.hadoop.hbase.HConstants;
042import org.apache.hadoop.hbase.HTableDescriptor;
043import org.apache.hadoop.hbase.KeyValue;
044import org.apache.hadoop.hbase.MiniHBaseCluster;
045import org.apache.hadoop.hbase.TableName;
046import org.apache.hadoop.hbase.TableNameTestRule;
047import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
048import org.apache.hadoop.hbase.master.LoadBalancer;
049import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
050import org.apache.hadoop.hbase.testclassification.ClientTests;
051import org.apache.hadoop.hbase.testclassification.LargeTests;
052import org.apache.hadoop.hbase.util.Bytes;
053import org.junit.AfterClass;
054import org.junit.Assume;
055import org.junit.ClassRule;
056import org.junit.Rule;
057import org.junit.Test;
058import org.junit.experimental.categories.Category;
059import org.junit.runner.RunWith;
060import org.junit.runners.Parameterized;
061import org.slf4j.Logger;
062import org.slf4j.LoggerFactory;
063
064/**
065 * Run tests that use the HBase clients; {@link Table}.
066 * Sets up the HBase mini cluster once at start and runs through all client tests.
067 * Each creates a table named for the method and does its stuff against that.
068 *
069 * Parameterized to run with different registry implementations.
070 */
071@Category({LargeTests.class, ClientTests.class})
072@SuppressWarnings ("deprecation")
073@RunWith(Parameterized.class)
074public class TestFromClientSide4 extends FromClientSideBase {
075  private static final Logger LOG = LoggerFactory.getLogger(TestFromClientSide4.class);
076  @ClassRule
077  public static final HBaseClassTestRule CLASS_RULE =
078    HBaseClassTestRule.forClass(TestFromClientSide4.class);
079  @Rule
080  public TableNameTestRule name = new TableNameTestRule();
081
082  // To keep the child classes happy.
083  TestFromClientSide4() {
084  }
085
086  public TestFromClientSide4(Class registry, int numHedgedReqs) throws Exception {
087    initialize(registry, numHedgedReqs, MultiRowMutationEndpoint.class);
088  }
089
090  @Parameterized.Parameters
091  public static Collection parameters() {
092    return Arrays.asList(new Object[][] { { MasterRegistry.class, 1 }, { MasterRegistry.class, 2 },
093      { ZKConnectionRegistry.class, 1 } });
094  }
095
096  @AfterClass
097  public static void tearDownAfterClass() throws Exception {
098    afterClass();
099  }
100
101  /**
102   * Test batch operations with combination of valid and invalid args
103   */
104  @Test public void testBatchOperationsWithErrors() throws Exception {
105    final TableName tableName = name.getTableName();
106    try (Table foo = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, 10)) {
107
108      int NUM_OPS = 100;
109
110      // 1.1 Put with no column families (local validation, runtime exception)
111      List<Put> puts = new ArrayList<>(NUM_OPS);
112      for (int i = 0; i != NUM_OPS; i++) {
113        Put put = new Put(Bytes.toBytes(i));
114        puts.add(put);
115      }
116
117      try {
118        foo.put(puts);
119        fail();
120      } catch (IllegalArgumentException e) {
121        // expected
122        assertEquals(NUM_OPS, puts.size());
123      }
124
125      // 1.2 Put with invalid column family
126      puts.clear();
127      for (int i = 0; i < NUM_OPS; i++) {
128        Put put = new Put(Bytes.toBytes(i));
129        put.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY, Bytes.toBytes(i));
130        puts.add(put);
131      }
132
133      try {
134        foo.put(puts);
135        fail();
136      } catch (RetriesExhaustedException e) {
137        if (e instanceof RetriesExhaustedWithDetailsException) {
138          assertThat(((RetriesExhaustedWithDetailsException)e).exceptions.get(0),
139            instanceOf(NoSuchColumnFamilyException.class));
140        } else {
141          assertThat(e.getCause(), instanceOf(NoSuchColumnFamilyException.class));
142        }
143      }
144
145      // 2.1 Get non-existent rows
146      List<Get> gets = new ArrayList<>(NUM_OPS);
147      for (int i = 0; i < NUM_OPS; i++) {
148        Get get = new Get(Bytes.toBytes(i));
149        gets.add(get);
150      }
151      Result[] getsResult = foo.get(gets);
152      assertNotNull(getsResult);
153      assertEquals(NUM_OPS, getsResult.length);
154      for (int i = 0; i < NUM_OPS; i++) {
155        Result getResult = getsResult[i];
156        if (i % 2 == 0) {
157          assertFalse(getResult.isEmpty());
158        } else {
159          assertTrue(getResult.isEmpty());
160        }
161      }
162
163      // 2.2 Get with invalid column family
164      gets.clear();
165      for (int i = 0; i < NUM_OPS; i++) {
166        Get get = new Get(Bytes.toBytes(i));
167        get.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY);
168        gets.add(get);
169      }
170      try {
171        foo.get(gets);
172        fail();
173      } catch (RetriesExhaustedException e) {
174        if (e instanceof RetriesExhaustedWithDetailsException) {
175          assertThat(((RetriesExhaustedWithDetailsException)e).exceptions.get(0),
176            instanceOf(NoSuchColumnFamilyException.class));
177        } else {
178          assertThat(e.getCause(), instanceOf(NoSuchColumnFamilyException.class));
179        }
180      }
181
182      // 3.1 Delete with invalid column family
183      List<Delete> deletes = new ArrayList<>(NUM_OPS);
184      for (int i = 0; i < NUM_OPS; i++) {
185        Delete delete = new Delete(Bytes.toBytes(i));
186        delete.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY);
187        deletes.add(delete);
188      }
189      try {
190        foo.delete(deletes);
191        fail();
192      } catch (RetriesExhaustedException e) {
193        if (e instanceof RetriesExhaustedWithDetailsException) {
194          assertThat(((RetriesExhaustedWithDetailsException)e).exceptions.get(0),
195            instanceOf(NoSuchColumnFamilyException.class));
196        } else {
197          assertThat(e.getCause(), instanceOf(NoSuchColumnFamilyException.class));
198        }
199      }
200
201      // all valid rows should have been deleted
202      gets.clear();
203      for (int i = 0; i < NUM_OPS; i++) {
204        Get get = new Get(Bytes.toBytes(i));
205        gets.add(get);
206      }
207      getsResult = foo.get(gets);
208      assertNotNull(getsResult);
209      assertEquals(NUM_OPS, getsResult.length);
210      for (Result getResult : getsResult) {
211        assertTrue(getResult.isEmpty());
212      }
213
214      // 3.2 Delete non-existent rows
215      deletes.clear();
216      for (int i = 0; i < NUM_OPS; i++) {
217        Delete delete = new Delete(Bytes.toBytes(i));
218        deletes.add(delete);
219      }
220      foo.delete(deletes);
221    }
222  }
223
224  //
225  // JIRA Testers
226  //
227
228  /**
229   * HBASE-867
230   * If millions of columns in a column family, hbase scanner won't come up
231   * Test will create numRows rows, each with numColsPerRow columns
232   * (1 version each), and attempt to scan them all.
233   * To test at scale, up numColsPerRow to the millions
234   * (have not gotten that to work running as junit though)
235   */
236  @Test public void testJiraTest867() throws Exception {
237    int numRows = 10;
238    int numColsPerRow = 2000;
239
240    final TableName tableName = name.getTableName();
241
242    byte[][] ROWS = makeN(ROW, numRows);
243    byte[][] QUALIFIERS = makeN(QUALIFIER, numColsPerRow);
244
245    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
246
247      // Insert rows
248
249      for (int i = 0; i < numRows; i++) {
250        Put put = new Put(ROWS[i]);
251        put.setDurability(Durability.SKIP_WAL);
252        for (int j = 0; j < numColsPerRow; j++) {
253          put.addColumn(FAMILY, QUALIFIERS[j], QUALIFIERS[j]);
254        }
255        assertEquals(
256          "Put expected to contain " + numColsPerRow + " columns but " + "only contains " + put
257            .size(), put.size(), numColsPerRow);
258        ht.put(put);
259      }
260
261      // Get a row
262      Get get = new Get(ROWS[numRows - 1]);
263      Result result = ht.get(get);
264      assertNumKeys(result, numColsPerRow);
265      Cell[] keys = result.rawCells();
266      for (int i = 0; i < result.size(); i++) {
267        assertKey(keys[i], ROWS[numRows - 1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
268      }
269
270      // Scan the rows
271      Scan scan = new Scan();
272      try (ResultScanner scanner = ht.getScanner(scan)) {
273        int rowCount = 0;
274        while ((result = scanner.next()) != null) {
275          assertNumKeys(result, numColsPerRow);
276          Cell[] kvs = result.rawCells();
277          for (int i = 0; i < numColsPerRow; i++) {
278            assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
279          }
280          rowCount++;
281        }
282        assertEquals(
283          "Expected to scan " + numRows + " rows but actually scanned " + rowCount + " rows",
284          rowCount, numRows);
285      }
286
287      // flush and try again
288
289      TEST_UTIL.flush();
290
291      // Get a row
292      get = new Get(ROWS[numRows - 1]);
293      result = ht.get(get);
294      assertNumKeys(result, numColsPerRow);
295      keys = result.rawCells();
296      for (int i = 0; i < result.size(); i++) {
297        assertKey(keys[i], ROWS[numRows - 1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
298      }
299
300      // Scan the rows
301      scan = new Scan();
302      try (ResultScanner scanner = ht.getScanner(scan)) {
303        int rowCount = 0;
304        while ((result = scanner.next()) != null) {
305          assertNumKeys(result, numColsPerRow);
306          Cell[] kvs = result.rawCells();
307          for (int i = 0; i < numColsPerRow; i++) {
308            assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
309          }
310          rowCount++;
311        }
312        assertEquals(
313          "Expected to scan " + numRows + " rows but actually scanned " + rowCount + " rows",
314          rowCount, numRows);
315      }
316    }
317  }
318
319  /**
320   * HBASE-861
321   * get with timestamp will return a value if there is a version with an
322   * earlier timestamp
323   */
324  @Test public void testJiraTest861() throws Exception {
325    final TableName tableName = name.getTableName();
326    byte[][] VALUES = makeNAscii(VALUE, 7);
327    long[] STAMPS = makeStamps(7);
328
329    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
330
331      // Insert three versions
332
333      Put put = new Put(ROW);
334      put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
335      put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
336      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
337      ht.put(put);
338
339      // Get the middle value
340      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
341
342      // Try to get one version before (expect fail)
343      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
344
345      // Try to get one version after (expect fail)
346      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
347
348      // Try same from storefile
349      TEST_UTIL.flush();
350      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
351      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
352      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
353
354      // Insert two more versions surrounding others, into memstore
355      put = new Put(ROW);
356      put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
357      put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
358      ht.put(put);
359
360      // Check we can get everything we should and can't get what we shouldn't
361      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
362      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
363      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
364      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
365      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
366      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
367      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
368
369      // Try same from two storefiles
370      TEST_UTIL.flush();
371      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
372      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
373      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
374      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
375      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
376      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
377      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
378    }
379  }
380
381  /**
382   * HBASE-33
383   * Add a HTable get/obtainScanner method that retrieves all versions of a
384   * particular column and row between two timestamps
385   */
386  @Test public void testJiraTest33() throws Exception {
387    final TableName tableName = name.getTableName();
388    byte[][] VALUES = makeNAscii(VALUE, 7);
389    long[] STAMPS = makeStamps(7);
390
391    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
392
393      // Insert lots versions
394
395      Put put = new Put(ROW);
396      put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
397      put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
398      put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
399      put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
400      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
401      put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
402      ht.put(put);
403
404      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
405      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
406      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
407      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
408
409      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
410      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
411      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
412      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
413
414      // Try same from storefile
415      TEST_UTIL.flush();
416
417      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
418      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
419      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
420      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
421
422      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
423      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
424      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
425      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
426    }
427  }
428
429  /**
430   * HBASE-1014
431   * commit(BatchUpdate) method should return timestamp
432   */
433  @Test public void testJiraTest1014() throws Exception {
434    final TableName tableName = name.getTableName();
435
436    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
437
438      long manualStamp = 12345;
439
440      // Insert lots versions
441
442      Put put = new Put(ROW);
443      put.addColumn(FAMILY, QUALIFIER, manualStamp, VALUE);
444      ht.put(put);
445
446      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, manualStamp, VALUE);
447      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp - 1);
448      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp + 1);
449    }
450  }
451
452  /**
453   * HBASE-1182
454   * Scan for columns > some timestamp
455   */
456  @Test public void testJiraTest1182() throws Exception {
457    final TableName tableName = name.getTableName();
458    byte[][] VALUES = makeNAscii(VALUE, 7);
459    long[] STAMPS = makeStamps(7);
460
461    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
462
463      // Insert lots versions
464
465      Put put = new Put(ROW);
466      put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
467      put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
468      put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
469      put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
470      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
471      put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
472      ht.put(put);
473
474      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
475      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
476      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
477
478      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
479      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
480      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
481
482      // Try same from storefile
483      TEST_UTIL.flush();
484
485      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
486      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
487      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
488
489      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
490      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
491      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
492    }
493  }
494
495  /**
496   * HBASE-52
497   * Add a means of scanning over all versions
498   */
499  @Test public void testJiraTest52() throws Exception {
500    final TableName tableName = name.getTableName();
501    byte[][] VALUES = makeNAscii(VALUE, 7);
502    long[] STAMPS = makeStamps(7);
503
504    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
505
506      // Insert lots versions
507
508      Put put = new Put(ROW);
509      put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
510      put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
511      put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
512      put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
513      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
514      put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
515      ht.put(put);
516
517      getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
518
519      scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
520
521      // Try same from storefile
522      TEST_UTIL.flush();
523
524      getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
525
526      scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
527    }
528  }
529
530  @Test
531  @SuppressWarnings("checkstyle:MethodLength")
532  public void testDuplicateVersions() throws Exception {
533    final TableName tableName = name.getTableName();
534
535    long[] STAMPS = makeStamps(20);
536    byte[][] VALUES = makeNAscii(VALUE, 20);
537
538    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
539
540      // Insert 4 versions of same column
541      Put put = new Put(ROW);
542      put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
543      put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
544      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
545      put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
546      ht.put(put);
547
548      // Verify we can get each one properly
549      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
550      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
551      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
552      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
553      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
554      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
555      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
556      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
557
558      // Verify we don't accidentally get others
559      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
560      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
561      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
562      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
563      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
564      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
565
566      // Ensure maxVersions in query is respected
567      Get get = new Get(ROW);
568      get.addColumn(FAMILY, QUALIFIER);
569      get.readVersions(2);
570      Result result = ht.get(get);
571      assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
572        new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
573
574      Scan scan = new Scan(ROW);
575      scan.addColumn(FAMILY, QUALIFIER);
576      scan.setMaxVersions(2);
577      result = getSingleScanResult(ht, scan);
578      assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
579        new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
580
581      // Flush and redo
582
583      TEST_UTIL.flush();
584
585      // Verify we can get each one properly
586      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
587      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
588      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
589      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
590      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
591      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
592      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
593      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
594
595      // Verify we don't accidentally get others
596      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
597      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
598      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
599      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
600      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
601      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
602
603      // Ensure maxVersions in query is respected
604      get = new Get(ROW);
605      get.addColumn(FAMILY, QUALIFIER);
606      get.readVersions(2);
607      result = ht.get(get);
608      assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
609        new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
610
611      scan = new Scan(ROW);
612      scan.addColumn(FAMILY, QUALIFIER);
613      scan.setMaxVersions(2);
614      result = getSingleScanResult(ht, scan);
615      assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
616        new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
617
618      // Add some memstore and retest
619
620      // Insert 4 more versions of same column and a dupe
621      put = new Put(ROW);
622      put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
623      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
624      put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
625      put.addColumn(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
626      put.addColumn(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
627      ht.put(put);
628
629      // Ensure maxVersions in query is respected
630      get = new Get(ROW);
631      get.addColumn(FAMILY, QUALIFIER);
632      get.readVersions(7);
633      result = ht.get(get);
634      assertNResult(result, ROW, FAMILY, QUALIFIER,
635        new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
636        new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
637          VALUES[8] }, 0, 6);
638
639      scan = new Scan(ROW);
640      scan.addColumn(FAMILY, QUALIFIER);
641      scan.setMaxVersions(7);
642      result = getSingleScanResult(ht, scan);
643      assertNResult(result, ROW, FAMILY, QUALIFIER,
644        new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
645        new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
646          VALUES[8] }, 0, 6);
647
648      get = new Get(ROW);
649      get.readVersions(7);
650      result = ht.get(get);
651      assertNResult(result, ROW, FAMILY, QUALIFIER,
652        new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
653        new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
654          VALUES[8] }, 0, 6);
655
656      scan = new Scan(ROW);
657      scan.setMaxVersions(7);
658      result = getSingleScanResult(ht, scan);
659      assertNResult(result, ROW, FAMILY, QUALIFIER,
660        new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
661        new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
662          VALUES[8] }, 0, 6);
663
664      // Verify we can get each one properly
665      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
666      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
667      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
668      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
669      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
670      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
671      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
672      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
673
674      // Verify we don't accidentally get others
675      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
676      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
677      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
678      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
679
680      // Ensure maxVersions of table is respected
681
682      TEST_UTIL.flush();
683
684      // Insert 4 more versions of same column and a dupe
685      put = new Put(ROW);
686      put.addColumn(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
687      put.addColumn(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
688      put.addColumn(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
689      put.addColumn(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
690      ht.put(put);
691
692      get = new Get(ROW);
693      get.addColumn(FAMILY, QUALIFIER);
694      get.readVersions(Integer.MAX_VALUE);
695      result = ht.get(get);
696      assertNResult(result, ROW, FAMILY, QUALIFIER,
697        new long[] { STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9],
698          STAMPS[11], STAMPS[13], STAMPS[15] },
699        new byte[][] { VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9],
700          VALUES[11], VALUES[13], VALUES[15] }, 0, 9);
701
702      scan = new Scan(ROW);
703      scan.addColumn(FAMILY, QUALIFIER);
704      scan.setMaxVersions(Integer.MAX_VALUE);
705      result = getSingleScanResult(ht, scan);
706      assertNResult(result, ROW, FAMILY, QUALIFIER,
707        new long[] { STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9],
708          STAMPS[11], STAMPS[13], STAMPS[15] },
709        new byte[][] { VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9],
710          VALUES[11], VALUES[13], VALUES[15] }, 0, 9);
711
712      // Delete a version in the memstore and a version in a storefile
713      Delete delete = new Delete(ROW);
714      delete.addColumn(FAMILY, QUALIFIER, STAMPS[11]);
715      delete.addColumn(FAMILY, QUALIFIER, STAMPS[7]);
716      ht.delete(delete);
717
718      // Test that it's gone
719      get = new Get(ROW);
720      get.addColumn(FAMILY, QUALIFIER);
721      get.readVersions(Integer.MAX_VALUE);
722      result = ht.get(get);
723      assertNResult(result, ROW, FAMILY, QUALIFIER,
724        new long[] { STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8],
725          STAMPS[9], STAMPS[13], STAMPS[15] },
726        new byte[][] { VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8],
727          VALUES[9], VALUES[13], VALUES[15] }, 0, 9);
728
729      scan = new Scan(ROW);
730      scan.addColumn(FAMILY, QUALIFIER);
731      scan.setMaxVersions(Integer.MAX_VALUE);
732      result = getSingleScanResult(ht, scan);
733      assertNResult(result, ROW, FAMILY, QUALIFIER,
734        new long[] { STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8],
735          STAMPS[9], STAMPS[13], STAMPS[15] },
736        new byte[][] { VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8],
737          VALUES[9], VALUES[13], VALUES[15] }, 0, 9);
738    }
739  }
740
741  @Test public void testUpdates() throws Exception {
742    final TableName tableName = name.getTableName();
743    try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
744
745      // Write a column with values at timestamp 1, 2 and 3
746      byte[] row = Bytes.toBytes("row1");
747      byte[] qualifier = Bytes.toBytes("myCol");
748      Put put = new Put(row);
749      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
750      hTable.put(put);
751
752      put = new Put(row);
753      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
754      hTable.put(put);
755
756      put = new Put(row);
757      put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
758      hTable.put(put);
759
760      Get get = new Get(row);
761      get.addColumn(FAMILY, qualifier);
762      get.readAllVersions();
763
764      // Check that the column indeed has the right values at timestamps 1 and
765      // 2
766      Result result = hTable.get(get);
767      NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier);
768      assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
769      assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
770
771      // Update the value at timestamp 1
772      put = new Put(row);
773      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
774      hTable.put(put);
775
776      // Update the value at timestamp 2
777      put = new Put(row);
778      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
779      hTable.put(put);
780
781      // Check that the values at timestamp 2 and 1 got updated
782      result = hTable.get(get);
783      navigableMap = result.getMap().get(FAMILY).get(qualifier);
784      assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
785      assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
786    }
787  }
788
789  @Test public void testUpdatesWithMajorCompaction() throws Exception {
790    final TableName tableName = name.getTableName();
791    try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10);
792        Admin admin = TEST_UTIL.getAdmin()) {
793
794      // Write a column with values at timestamp 1, 2 and 3
795      byte[] row = Bytes.toBytes("row2");
796      byte[] qualifier = Bytes.toBytes("myCol");
797      Put put = new Put(row);
798      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
799      hTable.put(put);
800
801      put = new Put(row);
802      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
803      hTable.put(put);
804
805      put = new Put(row);
806      put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
807      hTable.put(put);
808
809      Get get = new Get(row);
810      get.addColumn(FAMILY, qualifier);
811      get.readAllVersions();
812
813      // Check that the column indeed has the right values at timestamps 1 and
814      // 2
815      Result result = hTable.get(get);
816      NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier);
817      assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
818      assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
819
820      // Trigger a major compaction
821      admin.flush(tableName);
822      admin.majorCompact(tableName);
823      Thread.sleep(6000);
824
825      // Update the value at timestamp 1
826      put = new Put(row);
827      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
828      hTable.put(put);
829
830      // Update the value at timestamp 2
831      put = new Put(row);
832      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
833      hTable.put(put);
834
835      // Trigger a major compaction
836      admin.flush(tableName);
837      admin.majorCompact(tableName);
838      Thread.sleep(6000);
839
840      // Check that the values at timestamp 2 and 1 got updated
841      result = hTable.get(get);
842      navigableMap = result.getMap().get(FAMILY).get(qualifier);
843      assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
844      assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
845    }
846  }
847
848  @Test public void testMajorCompactionBetweenTwoUpdates() throws Exception {
849    final TableName tableName = name.getTableName();
850    try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10);
851        Admin admin = TEST_UTIL.getAdmin()) {
852
853      // Write a column with values at timestamp 1, 2 and 3
854      byte[] row = Bytes.toBytes("row3");
855      byte[] qualifier = Bytes.toBytes("myCol");
856      Put put = new Put(row);
857      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
858      hTable.put(put);
859
860      put = new Put(row);
861      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
862      hTable.put(put);
863
864      put = new Put(row);
865      put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
866      hTable.put(put);
867
868      Get get = new Get(row);
869      get.addColumn(FAMILY, qualifier);
870      get.readAllVersions();
871
872      // Check that the column indeed has the right values at timestamps 1 and
873      // 2
874      Result result = hTable.get(get);
875      NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier);
876      assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
877      assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
878
879      // Trigger a major compaction
880      admin.flush(tableName);
881      admin.majorCompact(tableName);
882      Thread.sleep(6000);
883
884      // Update the value at timestamp 1
885      put = new Put(row);
886      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
887      hTable.put(put);
888
889      // Trigger a major compaction
890      admin.flush(tableName);
891      admin.majorCompact(tableName);
892      Thread.sleep(6000);
893
894      // Update the value at timestamp 2
895      put = new Put(row);
896      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
897      hTable.put(put);
898
899      // Trigger a major compaction
900      admin.flush(tableName);
901      admin.majorCompact(tableName);
902      Thread.sleep(6000);
903
904      // Check that the values at timestamp 2 and 1 got updated
905      result = hTable.get(get);
906      navigableMap = result.getMap().get(FAMILY).get(qualifier);
907
908      assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
909      assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
910    }
911  }
912
913  @Test public void testGet_EmptyTable() throws IOException {
914    try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
915      Get get = new Get(ROW);
916      get.addFamily(FAMILY);
917      Result r = table.get(get);
918      assertTrue(r.isEmpty());
919    }
920  }
921
922  @Test public void testGet_NullQualifier() throws IOException {
923    try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
924      Put put = new Put(ROW);
925      put.addColumn(FAMILY, QUALIFIER, VALUE);
926      table.put(put);
927
928      put = new Put(ROW);
929      put.addColumn(FAMILY, null, VALUE);
930      table.put(put);
931      LOG.info("Row put");
932
933      Get get = new Get(ROW);
934      get.addColumn(FAMILY, null);
935      Result r = table.get(get);
936      assertEquals(1, r.size());
937
938      get = new Get(ROW);
939      get.addFamily(FAMILY);
940      r = table.get(get);
941      assertEquals(2, r.size());
942    }
943  }
944
945  @Test public void testGet_NonExistentRow() throws IOException {
946    try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
947      Put put = new Put(ROW);
948      put.addColumn(FAMILY, QUALIFIER, VALUE);
949      table.put(put);
950      LOG.info("Row put");
951
952      Get get = new Get(ROW);
953      get.addFamily(FAMILY);
954      Result r = table.get(get);
955      assertFalse(r.isEmpty());
956      System.out.println("Row retrieved successfully");
957
958      byte[] missingrow = Bytes.toBytes("missingrow");
959      get = new Get(missingrow);
960      get.addFamily(FAMILY);
961      r = table.get(get);
962      assertTrue(r.isEmpty());
963      LOG.info("Row missing as it should be");
964    }
965  }
966
967  @Test public void testPut() throws IOException {
968    final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
969    final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
970    final byte[] row1 = Bytes.toBytes("row1");
971    final byte[] row2 = Bytes.toBytes("row2");
972    final byte[] value = Bytes.toBytes("abcd");
973    try (Table table = TEST_UTIL
974      .createTable(name.getTableName(), new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY })) {
975      Put put = new Put(row1);
976      put.addColumn(CONTENTS_FAMILY, null, value);
977      table.put(put);
978
979      put = new Put(row2);
980      put.addColumn(CONTENTS_FAMILY, null, value);
981
982      assertEquals(1, put.size());
983      assertEquals(1, put.getFamilyCellMap().get(CONTENTS_FAMILY).size());
984
985      // KeyValue v1 expectation.  Cast for now until we go all Cell all the time. TODO
986      KeyValue kv = (KeyValue) put.getFamilyCellMap().get(CONTENTS_FAMILY).get(0);
987
988      assertTrue(Bytes.equals(CellUtil.cloneFamily(kv), CONTENTS_FAMILY));
989      // will it return null or an empty byte array?
990      assertTrue(Bytes.equals(CellUtil.cloneQualifier(kv), new byte[0]));
991
992      assertTrue(Bytes.equals(CellUtil.cloneValue(kv), value));
993
994      table.put(put);
995
996      Scan scan = new Scan();
997      scan.addColumn(CONTENTS_FAMILY, null);
998      try (ResultScanner scanner = table.getScanner(scan)) {
999        for (Result r : scanner) {
1000          for (Cell key : r.rawCells()) {
1001            System.out.println(Bytes.toString(r.getRow()) + ": " + key.toString());
1002          }
1003        }
1004      }
1005    }
1006  }
1007
1008  @Test public void testPutNoCF() throws IOException {
1009    final byte[] BAD_FAM = Bytes.toBytes("BAD_CF");
1010    final byte[] VAL = Bytes.toBytes(100);
1011    try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
1012      boolean caughtNSCFE = false;
1013
1014      try {
1015        Put p = new Put(ROW);
1016        p.addColumn(BAD_FAM, QUALIFIER, VAL);
1017        table.put(p);
1018      } catch (Exception e) {
1019        caughtNSCFE = e instanceof NoSuchColumnFamilyException;
1020      }
1021      assertTrue("Should throw NoSuchColumnFamilyException", caughtNSCFE);
1022    }
1023  }
1024
1025  @Test public void testRowsPut() throws IOException {
1026    final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
1027    final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
1028    final int NB_BATCH_ROWS = 10;
1029    final byte[] value = Bytes.toBytes("abcd");
1030    try (Table table = TEST_UTIL
1031      .createTable(name.getTableName(), new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY })) {
1032      ArrayList<Put> rowsUpdate = new ArrayList<>();
1033      for (int i = 0; i < NB_BATCH_ROWS; i++) {
1034        byte[] row = Bytes.toBytes("row" + i);
1035        Put put = new Put(row);
1036        put.setDurability(Durability.SKIP_WAL);
1037        put.addColumn(CONTENTS_FAMILY, null, value);
1038        rowsUpdate.add(put);
1039      }
1040      table.put(rowsUpdate);
1041      Scan scan = new Scan();
1042      scan.addFamily(CONTENTS_FAMILY);
1043      try (ResultScanner scanner = table.getScanner(scan)) {
1044        int nbRows = 0;
1045        for (@SuppressWarnings("unused") Result row : scanner) {
1046          nbRows++;
1047        }
1048        assertEquals(NB_BATCH_ROWS, nbRows);
1049      }
1050    }
1051  }
1052
1053  @Test public void testRowsPutBufferedManyManyFlushes() throws IOException {
1054    final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
1055    final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
1056    final byte[] value = Bytes.toBytes("abcd");
1057    final int NB_BATCH_ROWS = 10;
1058    try (Table table = TEST_UTIL
1059      .createTable(name.getTableName(), new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY })) {
1060      ArrayList<Put> rowsUpdate = new ArrayList<>();
1061      for (int i = 0; i < NB_BATCH_ROWS * 10; i++) {
1062        byte[] row = Bytes.toBytes("row" + i);
1063        Put put = new Put(row);
1064        put.setDurability(Durability.SKIP_WAL);
1065        put.addColumn(CONTENTS_FAMILY, null, value);
1066        rowsUpdate.add(put);
1067      }
1068      table.put(rowsUpdate);
1069
1070      Scan scan = new Scan();
1071      scan.addFamily(CONTENTS_FAMILY);
1072      try (ResultScanner scanner = table.getScanner(scan)) {
1073        int nbRows = 0;
1074        for (@SuppressWarnings("unused") Result row : scanner) {
1075          nbRows++;
1076        }
1077        assertEquals(NB_BATCH_ROWS * 10, nbRows);
1078      }
1079    }
1080  }
1081
1082  @Test public void testAddKeyValue() {
1083    final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
1084    final byte[] value = Bytes.toBytes("abcd");
1085    final byte[] row1 = Bytes.toBytes("row1");
1086    final byte[] row2 = Bytes.toBytes("row2");
1087    byte[] qualifier = Bytes.toBytes("qf1");
1088    Put put = new Put(row1);
1089
1090    // Adding KeyValue with the same row
1091    KeyValue kv = new KeyValue(row1, CONTENTS_FAMILY, qualifier, value);
1092    boolean ok = true;
1093    try {
1094      put.add(kv);
1095    } catch (IOException e) {
1096      ok = false;
1097    }
1098    assertTrue(ok);
1099
1100    // Adding KeyValue with the different row
1101    kv = new KeyValue(row2, CONTENTS_FAMILY, qualifier, value);
1102    ok = false;
1103    try {
1104      put.add(kv);
1105    } catch (IOException e) {
1106      ok = true;
1107    }
1108    assertTrue(ok);
1109  }
1110
1111  /**
1112   * test for HBASE-737
1113   */
1114  @Test public void testHBase737() throws IOException {
1115    final byte[] FAM1 = Bytes.toBytes("fam1");
1116    final byte[] FAM2 = Bytes.toBytes("fam2");
1117    // Open table
1118    try (Table table = TEST_UTIL.createTable(name.getTableName(), new byte[][] { FAM1, FAM2 })) {
1119      // Insert some values
1120      Put put = new Put(ROW);
1121      put.addColumn(FAM1, Bytes.toBytes("letters"), Bytes.toBytes("abcdefg"));
1122      table.put(put);
1123      try {
1124        Thread.sleep(1000);
1125      } catch (InterruptedException i) {
1126        //ignore
1127      }
1128
1129      put = new Put(ROW);
1130      put.addColumn(FAM1, Bytes.toBytes("numbers"), Bytes.toBytes("123456"));
1131      table.put(put);
1132
1133      try {
1134        Thread.sleep(1000);
1135      } catch (InterruptedException i) {
1136        //ignore
1137      }
1138
1139      put = new Put(ROW);
1140      put.addColumn(FAM2, Bytes.toBytes("letters"), Bytes.toBytes("hijklmnop"));
1141      table.put(put);
1142
1143      long[] times = new long[3];
1144
1145      // First scan the memstore
1146
1147      Scan scan = new Scan();
1148      scan.addFamily(FAM1);
1149      scan.addFamily(FAM2);
1150      try (ResultScanner s = table.getScanner(scan)) {
1151        int index = 0;
1152        Result r;
1153        while ((r = s.next()) != null) {
1154          for (Cell key : r.rawCells()) {
1155            times[index++] = key.getTimestamp();
1156          }
1157        }
1158      }
1159      for (int i = 0; i < times.length - 1; i++) {
1160        for (int j = i + 1; j < times.length; j++) {
1161          assertTrue(times[j] > times[i]);
1162        }
1163      }
1164
1165      // Flush data to disk and try again
1166      TEST_UTIL.flush();
1167
1168      // Reset times
1169      Arrays.fill(times, 0);
1170
1171      try {
1172        Thread.sleep(1000);
1173      } catch (InterruptedException i) {
1174        //ignore
1175      }
1176      scan = new Scan();
1177      scan.addFamily(FAM1);
1178      scan.addFamily(FAM2);
1179      try (ResultScanner s = table.getScanner(scan)) {
1180        int index = 0;
1181        Result r = null;
1182        while ((r = s.next()) != null) {
1183          for (Cell key : r.rawCells()) {
1184            times[index++] = key.getTimestamp();
1185          }
1186        }
1187        for (int i = 0; i < times.length - 1; i++) {
1188          for (int j = i + 1; j < times.length; j++) {
1189            assertTrue(times[j] > times[i]);
1190          }
1191        }
1192      }
1193    }
1194  }
1195
1196  @Test public void testListTables() throws IOException {
1197    final String testTableName = name.getTableName().toString();
1198    final TableName tableName1 = TableName.valueOf(testTableName + "1");
1199    final TableName tableName2 = TableName.valueOf(testTableName + "2");
1200    final TableName tableName3 = TableName.valueOf(testTableName + "3");
1201    TableName[] tables = new TableName[] { tableName1, tableName2, tableName3 };
1202    for (TableName table : tables) {
1203      TEST_UTIL.createTable(table, FAMILY);
1204    }
1205    try (Admin admin = TEST_UTIL.getAdmin()) {
1206      List<TableDescriptor> ts = admin.listTableDescriptors();
1207      HashSet<TableDescriptor> result = new HashSet<>(ts);
1208      int size = result.size();
1209      assertTrue(size >= tables.length);
1210      for (TableName table : tables) {
1211        boolean found = false;
1212        for (TableDescriptor t : ts) {
1213          if (t.getTableName().equals(table)) {
1214            found = true;
1215            break;
1216          }
1217        }
1218        assertTrue("Not found: " + table, found);
1219      }
1220    }
1221  }
1222
1223  /**
1224   * simple test that just executes parts of the client
1225   * API that accept a pre-created Connection instance
1226   */
1227  @Test public void testUnmanagedHConnection() throws IOException {
1228    final TableName tableName = name.getTableName();
1229    TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
1230    try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
1231      Table t = conn.getTable(tableName);
1232      Admin admin = conn.getAdmin()) {
1233      assertTrue(admin.tableExists(tableName));
1234      assertTrue(t.get(new Get(ROW)).isEmpty());
1235    }
1236  }
1237
1238  /**
1239   * test of that unmanaged HConnections are able to reconnect
1240   * properly (see HBASE-5058)
1241   */
1242  @Test public void testUnmanagedHConnectionReconnect() throws Exception {
1243    Configuration conf = TEST_UTIL.getConfiguration();
1244    Class registryImpl = conf
1245      .getClass(HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY, ZKConnectionRegistry.class);
1246    // This test does not make sense for MasterRegistry since it stops the only master in the
1247    // cluster and starts a new master without populating the underlying config for the connection.
1248    Assume.assumeFalse(registryImpl.equals(MasterRegistry.class));
1249    final TableName tableName = name.getTableName();
1250    TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
1251    try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) {
1252      try (Table t = conn.getTable(tableName); Admin admin = conn.getAdmin()) {
1253        assertTrue(admin.tableExists(tableName));
1254        assertTrue(t.get(new Get(ROW)).isEmpty());
1255      }
1256
1257      // stop the master
1258      MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1259      cluster.stopMaster(0, false);
1260      cluster.waitOnMaster(0);
1261
1262      // start up a new master
1263      cluster.startMaster();
1264      assertTrue(cluster.waitForActiveAndReadyMaster());
1265
1266      // test that the same unmanaged connection works with a new
1267      // Admin and can connect to the new master;
1268      boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration());
1269      try (Admin admin = conn.getAdmin()) {
1270        assertTrue(admin.tableExists(tableName));
1271        assertEquals(
1272          admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().size(),
1273          SLAVES + (tablesOnMaster ? 1 : 0));
1274      }
1275    }
1276  }
1277
1278  @Test public void testMiscHTableStuff() throws IOException {
1279    final String testTableName = name.getTableName().toString();
1280    final TableName tableAname = TableName.valueOf(testTableName + "A");
1281    final TableName tableBname = TableName.valueOf(testTableName + "B");
1282    final byte[] attrName = Bytes.toBytes("TESTATTR");
1283    final byte[] attrValue = Bytes.toBytes("somevalue");
1284    byte[] value = Bytes.toBytes("value");
1285
1286    try (Table a = TEST_UTIL.createTable(tableAname, HConstants.CATALOG_FAMILY);
1287      Table b = TEST_UTIL.createTable(tableBname, HConstants.CATALOG_FAMILY)) {
1288      Put put = new Put(ROW);
1289      put.addColumn(HConstants.CATALOG_FAMILY, null, value);
1290      a.put(put);
1291
1292      // open a new connection to A and a connection to b
1293      try (Table newA = TEST_UTIL.getConnection().getTable(tableAname)) {
1294
1295        // copy data from A to B
1296        Scan scan = new Scan();
1297        scan.addFamily(HConstants.CATALOG_FAMILY);
1298        try (ResultScanner s = newA.getScanner(scan)) {
1299          for (Result r : s) {
1300            put = new Put(r.getRow());
1301            put.setDurability(Durability.SKIP_WAL);
1302            for (Cell kv : r.rawCells()) {
1303              put.add(kv);
1304            }
1305            b.put(put);
1306          }
1307        }
1308      }
1309
1310      // Opening a new connection to A will cause the tables to be reloaded
1311      try (Table anotherA = TEST_UTIL.getConnection().getTable(tableAname)) {
1312        Get get = new Get(ROW);
1313        get.addFamily(HConstants.CATALOG_FAMILY);
1314        anotherA.get(get);
1315      }
1316
1317      // We can still access A through newA because it has the table information
1318      // cached. And if it needs to recalibrate, that will cause the information
1319      // to be reloaded.
1320
1321      // Test user metadata
1322      Admin admin = TEST_UTIL.getAdmin();
1323      // make a modifiable descriptor
1324      HTableDescriptor desc = new HTableDescriptor(a.getDescriptor());
1325      // offline the table
1326      admin.disableTable(tableAname);
1327      // add a user attribute to HTD
1328      desc.setValue(attrName, attrValue);
1329      // add a user attribute to HCD
1330      for (HColumnDescriptor c : desc.getFamilies()) {
1331        c.setValue(attrName, attrValue);
1332      }
1333      // update metadata for all regions of this table
1334      admin.modifyTable(desc);
1335      // enable the table
1336      admin.enableTable(tableAname);
1337
1338      // Test that attribute changes were applied
1339      desc = new HTableDescriptor(a.getDescriptor());
1340      assertEquals("wrong table descriptor returned", desc.getTableName(), tableAname);
1341      // check HTD attribute
1342      value = desc.getValue(attrName);
1343      assertNotNull("missing HTD attribute value", value);
1344      assertFalse("HTD attribute value is incorrect", Bytes.compareTo(value, attrValue) != 0);
1345      // check HCD attribute
1346      for (HColumnDescriptor c : desc.getFamilies()) {
1347        value = c.getValue(attrName);
1348        assertNotNull("missing HCD attribute value", value);
1349        assertFalse("HCD attribute value is incorrect", Bytes.compareTo(value, attrValue) != 0);
1350      }
1351    }
1352  }
1353}
1354