001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.hamcrest.CoreMatchers.instanceOf;
021import static org.hamcrest.MatcherAssert.assertThat;
022import static org.junit.Assert.assertEquals;
023import static org.junit.Assert.assertFalse;
024import static org.junit.Assert.assertNotNull;
025import static org.junit.Assert.assertTrue;
026import static org.junit.Assert.fail;
027
028import java.io.IOException;
029import java.util.ArrayList;
030import java.util.Arrays;
031import java.util.Collection;
032import java.util.EnumSet;
033import java.util.HashSet;
034import java.util.List;
035import java.util.NavigableMap;
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.hbase.Cell;
038import org.apache.hadoop.hbase.CellUtil;
039import org.apache.hadoop.hbase.ClusterMetrics.Option;
040import org.apache.hadoop.hbase.HBaseClassTestRule;
041import org.apache.hadoop.hbase.HConstants;
042import org.apache.hadoop.hbase.KeyValue;
043import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
044import org.apache.hadoop.hbase.TableName;
045import org.apache.hadoop.hbase.TableNameTestRule;
046import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
047import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
048import org.apache.hadoop.hbase.testclassification.ClientTests;
049import org.apache.hadoop.hbase.testclassification.LargeTests;
050import org.apache.hadoop.hbase.util.Bytes;
051import org.junit.AfterClass;
052import org.junit.Assume;
053import org.junit.ClassRule;
054import org.junit.Rule;
055import org.junit.Test;
056import org.junit.experimental.categories.Category;
057import org.junit.runner.RunWith;
058import org.junit.runners.Parameterized;
059import org.slf4j.Logger;
060import org.slf4j.LoggerFactory;
061
062/**
063 * Run tests that use the HBase clients; {@link Table}.
064 * Sets up the HBase mini cluster once at start and runs through all client tests.
065 * Each creates a table named for the method and does its stuff against that.
066 *
067 * Parameterized to run with different registry implementations.
068 */
069@Category({LargeTests.class, ClientTests.class})
070@SuppressWarnings ("deprecation")
071@RunWith(Parameterized.class)
072public class TestFromClientSide4 extends FromClientSideBase {
073  private static final Logger LOG = LoggerFactory.getLogger(TestFromClientSide4.class);
074  @ClassRule
075  public static final HBaseClassTestRule CLASS_RULE =
076    HBaseClassTestRule.forClass(TestFromClientSide4.class);
077  @Rule
078  public TableNameTestRule name = new TableNameTestRule();
079
080  // To keep the child classes happy.
081  TestFromClientSide4() {
082  }
083
084  public TestFromClientSide4(Class registry, int numHedgedReqs) throws Exception {
085    initialize(registry, numHedgedReqs, MultiRowMutationEndpoint.class);
086  }
087
088  @Parameterized.Parameters
089  public static Collection parameters() {
090    return Arrays.asList(new Object[][] { { MasterRegistry.class, 1 }, { MasterRegistry.class, 2 },
091      { ZKConnectionRegistry.class, 1 } });
092  }
093
094  @AfterClass
095  public static void tearDownAfterClass() throws Exception {
096    afterClass();
097  }
098
099  /**
100   * Test batch operations with combination of valid and invalid args
101   */
102  @Test public void testBatchOperationsWithErrors() throws Exception {
103    final TableName tableName = name.getTableName();
104    try (Table foo = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, 10)) {
105
106      int NUM_OPS = 100;
107
108      // 1.1 Put with no column families (local validation, runtime exception)
109      List<Put> puts = new ArrayList<>(NUM_OPS);
110      for (int i = 0; i != NUM_OPS; i++) {
111        Put put = new Put(Bytes.toBytes(i));
112        puts.add(put);
113      }
114
115      try {
116        foo.put(puts);
117        fail();
118      } catch (IllegalArgumentException e) {
119        // expected
120        assertEquals(NUM_OPS, puts.size());
121      }
122
123      // 1.2 Put with invalid column family
124      puts.clear();
125      for (int i = 0; i < NUM_OPS; i++) {
126        Put put = new Put(Bytes.toBytes(i));
127        put.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY, Bytes.toBytes(i));
128        puts.add(put);
129      }
130
131      try {
132        foo.put(puts);
133        fail();
134      } catch (RetriesExhaustedException e) {
135        // expected
136        assertThat(e.getCause(), instanceOf(NoSuchColumnFamilyException.class));
137      }
138
139      // 2.1 Get non-existent rows
140      List<Get> gets = new ArrayList<>(NUM_OPS);
141      for (int i = 0; i < NUM_OPS; i++) {
142        Get get = new Get(Bytes.toBytes(i));
143        gets.add(get);
144      }
145      Result[] getsResult = foo.get(gets);
146      assertNotNull(getsResult);
147      assertEquals(NUM_OPS, getsResult.length);
148      for (int i = 0; i < NUM_OPS; i++) {
149        Result getResult = getsResult[i];
150        if (i % 2 == 0) {
151          assertFalse(getResult.isEmpty());
152        } else {
153          assertTrue(getResult.isEmpty());
154        }
155      }
156
157      // 2.2 Get with invalid column family
158      gets.clear();
159      for (int i = 0; i < NUM_OPS; i++) {
160        Get get = new Get(Bytes.toBytes(i));
161        get.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY);
162        gets.add(get);
163      }
164      try {
165        foo.get(gets);
166        fail();
167      } catch (RetriesExhaustedException e) {
168        // expected
169        assertThat(e.getCause(), instanceOf(NoSuchColumnFamilyException.class));
170      }
171
172      // 3.1 Delete with invalid column family
173      List<Delete> deletes = new ArrayList<>(NUM_OPS);
174      for (int i = 0; i < NUM_OPS; i++) {
175        Delete delete = new Delete(Bytes.toBytes(i));
176        delete.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY);
177        deletes.add(delete);
178      }
179      try {
180        foo.delete(deletes);
181        fail();
182      } catch (RetriesExhaustedException e) {
183        // expected
184        assertThat(e.getCause(), instanceOf(NoSuchColumnFamilyException.class));
185      }
186
187      // all valid rows should have been deleted
188      gets.clear();
189      for (int i = 0; i < NUM_OPS; i++) {
190        Get get = new Get(Bytes.toBytes(i));
191        gets.add(get);
192      }
193      getsResult = foo.get(gets);
194      assertNotNull(getsResult);
195      assertEquals(NUM_OPS, getsResult.length);
196      for (Result getResult : getsResult) {
197        assertTrue(getResult.isEmpty());
198      }
199
200      // 3.2 Delete non-existent rows
201      deletes.clear();
202      for (int i = 0; i < NUM_OPS; i++) {
203        Delete delete = new Delete(Bytes.toBytes(i));
204        deletes.add(delete);
205      }
206      foo.delete(deletes);
207    }
208  }
209
210  //
211  // JIRA Testers
212  //
213
214  /**
215   * HBASE-867
216   * If millions of columns in a column family, hbase scanner won't come up
217   * Test will create numRows rows, each with numColsPerRow columns
218   * (1 version each), and attempt to scan them all.
219   * To test at scale, up numColsPerRow to the millions
220   * (have not gotten that to work running as junit though)
221   */
222  @Test public void testJiraTest867() throws Exception {
223    int numRows = 10;
224    int numColsPerRow = 2000;
225
226    final TableName tableName = name.getTableName();
227
228    byte[][] ROWS = makeN(ROW, numRows);
229    byte[][] QUALIFIERS = makeN(QUALIFIER, numColsPerRow);
230
231    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
232
233      // Insert rows
234
235      for (int i = 0; i < numRows; i++) {
236        Put put = new Put(ROWS[i]);
237        put.setDurability(Durability.SKIP_WAL);
238        for (int j = 0; j < numColsPerRow; j++) {
239          put.addColumn(FAMILY, QUALIFIERS[j], QUALIFIERS[j]);
240        }
241        assertEquals(
242          "Put expected to contain " + numColsPerRow + " columns but " + "only contains " + put
243            .size(), put.size(), numColsPerRow);
244        ht.put(put);
245      }
246
247      // Get a row
248      Get get = new Get(ROWS[numRows - 1]);
249      Result result = ht.get(get);
250      assertNumKeys(result, numColsPerRow);
251      Cell[] keys = result.rawCells();
252      for (int i = 0; i < result.size(); i++) {
253        assertKey(keys[i], ROWS[numRows - 1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
254      }
255
256      // Scan the rows
257      Scan scan = new Scan();
258      try (ResultScanner scanner = ht.getScanner(scan)) {
259        int rowCount = 0;
260        while ((result = scanner.next()) != null) {
261          assertNumKeys(result, numColsPerRow);
262          Cell[] kvs = result.rawCells();
263          for (int i = 0; i < numColsPerRow; i++) {
264            assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
265          }
266          rowCount++;
267        }
268        assertEquals(
269          "Expected to scan " + numRows + " rows but actually scanned " + rowCount + " rows",
270          rowCount, numRows);
271      }
272
273      // flush and try again
274
275      TEST_UTIL.flush();
276
277      // Get a row
278      get = new Get(ROWS[numRows - 1]);
279      result = ht.get(get);
280      assertNumKeys(result, numColsPerRow);
281      keys = result.rawCells();
282      for (int i = 0; i < result.size(); i++) {
283        assertKey(keys[i], ROWS[numRows - 1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
284      }
285
286      // Scan the rows
287      scan = new Scan();
288      try (ResultScanner scanner = ht.getScanner(scan)) {
289        int rowCount = 0;
290        while ((result = scanner.next()) != null) {
291          assertNumKeys(result, numColsPerRow);
292          Cell[] kvs = result.rawCells();
293          for (int i = 0; i < numColsPerRow; i++) {
294            assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
295          }
296          rowCount++;
297        }
298        assertEquals(
299          "Expected to scan " + numRows + " rows but actually scanned " + rowCount + " rows",
300          rowCount, numRows);
301      }
302    }
303  }
304
305  /**
306   * HBASE-861
307   * get with timestamp will return a value if there is a version with an
308   * earlier timestamp
309   */
310  @Test public void testJiraTest861() throws Exception {
311    final TableName tableName = name.getTableName();
312    byte[][] VALUES = makeNAscii(VALUE, 7);
313    long[] STAMPS = makeStamps(7);
314
315    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
316
317      // Insert three versions
318
319      Put put = new Put(ROW);
320      put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
321      put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
322      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
323      ht.put(put);
324
325      // Get the middle value
326      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
327
328      // Try to get one version before (expect fail)
329      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
330
331      // Try to get one version after (expect fail)
332      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
333
334      // Try same from storefile
335      TEST_UTIL.flush();
336      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
337      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
338      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
339
340      // Insert two more versions surrounding others, into memstore
341      put = new Put(ROW);
342      put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
343      put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
344      ht.put(put);
345
346      // Check we can get everything we should and can't get what we shouldn't
347      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
348      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
349      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
350      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
351      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
352      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
353      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
354
355      // Try same from two storefiles
356      TEST_UTIL.flush();
357      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
358      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
359      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
360      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
361      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
362      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
363      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
364    }
365  }
366
367  /**
368   * HBASE-33
369   * Add a HTable get/obtainScanner method that retrieves all versions of a
370   * particular column and row between two timestamps
371   */
372  @Test public void testJiraTest33() throws Exception {
373    final TableName tableName = name.getTableName();
374    byte[][] VALUES = makeNAscii(VALUE, 7);
375    long[] STAMPS = makeStamps(7);
376
377    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
378
379      // Insert lots versions
380
381      Put put = new Put(ROW);
382      put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
383      put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
384      put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
385      put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
386      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
387      put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
388      ht.put(put);
389
390      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
391      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
392      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
393      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
394
395      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
396      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
397      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
398      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
399
400      // Try same from storefile
401      TEST_UTIL.flush();
402
403      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
404      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
405      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
406      getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
407
408      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
409      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
410      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
411      scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
412    }
413  }
414
415  /**
416   * HBASE-1014
417   * commit(BatchUpdate) method should return timestamp
418   */
419  @Test public void testJiraTest1014() throws Exception {
420    final TableName tableName = name.getTableName();
421
422    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
423
424      long manualStamp = 12345;
425
426      // Insert lots versions
427
428      Put put = new Put(ROW);
429      put.addColumn(FAMILY, QUALIFIER, manualStamp, VALUE);
430      ht.put(put);
431
432      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, manualStamp, VALUE);
433      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp - 1);
434      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp + 1);
435    }
436  }
437
438  /**
439   * HBASE-1182
440   * Scan for columns > some timestamp
441   */
442  @Test public void testJiraTest1182() throws Exception {
443    final TableName tableName = name.getTableName();
444    byte[][] VALUES = makeNAscii(VALUE, 7);
445    long[] STAMPS = makeStamps(7);
446
447    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
448
449      // Insert lots versions
450
451      Put put = new Put(ROW);
452      put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
453      put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
454      put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
455      put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
456      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
457      put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
458      ht.put(put);
459
460      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
461      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
462      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
463
464      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
465      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
466      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
467
468      // Try same from storefile
469      TEST_UTIL.flush();
470
471      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
472      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
473      getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
474
475      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
476      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
477      scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
478    }
479  }
480
481  /**
482   * HBASE-52
483   * Add a means of scanning over all versions
484   */
485  @Test public void testJiraTest52() throws Exception {
486    final TableName tableName = name.getTableName();
487    byte[][] VALUES = makeNAscii(VALUE, 7);
488    long[] STAMPS = makeStamps(7);
489
490    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
491
492      // Insert lots versions
493
494      Put put = new Put(ROW);
495      put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
496      put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
497      put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
498      put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
499      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
500      put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
501      ht.put(put);
502
503      getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
504
505      scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
506
507      // Try same from storefile
508      TEST_UTIL.flush();
509
510      getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
511
512      scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
513    }
514  }
515
516  @Test
517  @SuppressWarnings("checkstyle:MethodLength")
518  public void testDuplicateVersions() throws Exception {
519    final TableName tableName = name.getTableName();
520
521    long[] STAMPS = makeStamps(20);
522    byte[][] VALUES = makeNAscii(VALUE, 20);
523
524    try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
525
526      // Insert 4 versions of same column
527      Put put = new Put(ROW);
528      put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
529      put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
530      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
531      put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
532      ht.put(put);
533
534      // Verify we can get each one properly
535      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
536      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
537      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
538      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
539      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
540      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
541      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
542      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
543
544      // Verify we don't accidentally get others
545      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
546      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
547      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
548      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
549      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
550      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
551
552      // Ensure maxVersions in query is respected
553      Get get = new Get(ROW);
554      get.addColumn(FAMILY, QUALIFIER);
555      get.readVersions(2);
556      Result result = ht.get(get);
557      assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
558        new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
559
560      Scan scan = new Scan().withStartRow(ROW);
561      scan.addColumn(FAMILY, QUALIFIER);
562      scan.readVersions(2);
563      result = getSingleScanResult(ht, scan);
564      assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
565        new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
566
567      // Flush and redo
568
569      TEST_UTIL.flush();
570
571      // Verify we can get each one properly
572      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
573      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
574      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
575      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
576      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
577      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
578      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
579      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
580
581      // Verify we don't accidentally get others
582      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
583      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
584      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
585      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
586      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
587      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
588
589      // Ensure maxVersions in query is respected
590      get = new Get(ROW);
591      get.addColumn(FAMILY, QUALIFIER);
592      get.readVersions(2);
593      result = ht.get(get);
594      assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
595        new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
596
597      scan = new Scan().withStartRow(ROW);
598      scan.addColumn(FAMILY, QUALIFIER);
599      scan.readVersions(2);
600      result = getSingleScanResult(ht, scan);
601      assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
602        new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
603
604      // Add some memstore and retest
605
606      // Insert 4 more versions of same column and a dupe
607      put = new Put(ROW);
608      put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
609      put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
610      put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
611      put.addColumn(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
612      put.addColumn(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
613      ht.put(put);
614
615      // Ensure maxVersions in query is respected
616      get = new Get(ROW);
617      get.addColumn(FAMILY, QUALIFIER);
618      get.readVersions(7);
619      result = ht.get(get);
620      assertNResult(result, ROW, FAMILY, QUALIFIER,
621        new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
622        new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
623          VALUES[8] }, 0, 6);
624
625      scan = new Scan().withStartRow(ROW);
626      scan.addColumn(FAMILY, QUALIFIER);
627      scan.readVersions(7);
628      result = getSingleScanResult(ht, scan);
629      assertNResult(result, ROW, FAMILY, QUALIFIER,
630        new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
631        new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
632          VALUES[8] }, 0, 6);
633
634      get = new Get(ROW);
635      get.readVersions(7);
636      result = ht.get(get);
637      assertNResult(result, ROW, FAMILY, QUALIFIER,
638        new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
639        new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
640          VALUES[8] }, 0, 6);
641
642      scan = new Scan().withStartRow(ROW);
643      scan.readVersions(7);
644      result = getSingleScanResult(ht, scan);
645      assertNResult(result, ROW, FAMILY, QUALIFIER,
646        new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
647        new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
648          VALUES[8] }, 0, 6);
649
650      // Verify we can get each one properly
651      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
652      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
653      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
654      getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
655      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
656      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
657      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
658      scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
659
660      // Verify we don't accidentally get others
661      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
662      getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
663      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
664      scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
665
666      // Ensure maxVersions of table is respected
667
668      TEST_UTIL.flush();
669
670      // Insert 4 more versions of same column and a dupe
671      put = new Put(ROW);
672      put.addColumn(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
673      put.addColumn(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
674      put.addColumn(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
675      put.addColumn(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
676      ht.put(put);
677
678      get = new Get(ROW);
679      get.addColumn(FAMILY, QUALIFIER);
680      get.readVersions(Integer.MAX_VALUE);
681      result = ht.get(get);
682      assertNResult(result, ROW, FAMILY, QUALIFIER,
683        new long[] { STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9],
684          STAMPS[11], STAMPS[13], STAMPS[15] },
685        new byte[][] { VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9],
686          VALUES[11], VALUES[13], VALUES[15] }, 0, 9);
687
688      scan = new Scan().withStartRow(ROW);
689      scan.addColumn(FAMILY, QUALIFIER);
690      scan.readVersions(Integer.MAX_VALUE);
691      result = getSingleScanResult(ht, scan);
692      assertNResult(result, ROW, FAMILY, QUALIFIER,
693        new long[] { STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9],
694          STAMPS[11], STAMPS[13], STAMPS[15] },
695        new byte[][] { VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9],
696          VALUES[11], VALUES[13], VALUES[15] }, 0, 9);
697
698      // Delete a version in the memstore and a version in a storefile
699      Delete delete = new Delete(ROW);
700      delete.addColumn(FAMILY, QUALIFIER, STAMPS[11]);
701      delete.addColumn(FAMILY, QUALIFIER, STAMPS[7]);
702      ht.delete(delete);
703
704      // Test that it's gone
705      get = new Get(ROW);
706      get.addColumn(FAMILY, QUALIFIER);
707      get.readVersions(Integer.MAX_VALUE);
708      result = ht.get(get);
709      assertNResult(result, ROW, FAMILY, QUALIFIER,
710        new long[] { STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8],
711          STAMPS[9], STAMPS[13], STAMPS[15] },
712        new byte[][] { VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8],
713          VALUES[9], VALUES[13], VALUES[15] }, 0, 9);
714
715      scan = new Scan().withStartRow(ROW);
716      scan.addColumn(FAMILY, QUALIFIER);
717      scan.readVersions(Integer.MAX_VALUE);
718      result = getSingleScanResult(ht, scan);
719      assertNResult(result, ROW, FAMILY, QUALIFIER,
720        new long[] { STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8],
721          STAMPS[9], STAMPS[13], STAMPS[15] },
722        new byte[][] { VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8],
723          VALUES[9], VALUES[13], VALUES[15] }, 0, 9);
724    }
725  }
726
727  @Test public void testUpdates() throws Exception {
728    final TableName tableName = name.getTableName();
729    try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
730
731      // Write a column with values at timestamp 1, 2 and 3
732      byte[] row = Bytes.toBytes("row1");
733      byte[] qualifier = Bytes.toBytes("myCol");
734      Put put = new Put(row);
735      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
736      hTable.put(put);
737
738      put = new Put(row);
739      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
740      hTable.put(put);
741
742      put = new Put(row);
743      put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
744      hTable.put(put);
745
746      Get get = new Get(row);
747      get.addColumn(FAMILY, qualifier);
748      get.readAllVersions();
749
750      // Check that the column indeed has the right values at timestamps 1 and
751      // 2
752      Result result = hTable.get(get);
753      NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier);
754      assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
755      assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
756
757      // Update the value at timestamp 1
758      put = new Put(row);
759      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
760      hTable.put(put);
761
762      // Update the value at timestamp 2
763      put = new Put(row);
764      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
765      hTable.put(put);
766
767      // Check that the values at timestamp 2 and 1 got updated
768      result = hTable.get(get);
769      navigableMap = result.getMap().get(FAMILY).get(qualifier);
770      assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
771      assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
772    }
773  }
774
775  @Test public void testUpdatesWithMajorCompaction() throws Exception {
776    final TableName tableName = name.getTableName();
777    try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10);
778        Admin admin = TEST_UTIL.getAdmin()) {
779
780      // Write a column with values at timestamp 1, 2 and 3
781      byte[] row = Bytes.toBytes("row2");
782      byte[] qualifier = Bytes.toBytes("myCol");
783      Put put = new Put(row);
784      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
785      hTable.put(put);
786
787      put = new Put(row);
788      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
789      hTable.put(put);
790
791      put = new Put(row);
792      put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
793      hTable.put(put);
794
795      Get get = new Get(row);
796      get.addColumn(FAMILY, qualifier);
797      get.readAllVersions();
798
799      // Check that the column indeed has the right values at timestamps 1 and
800      // 2
801      Result result = hTable.get(get);
802      NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier);
803      assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
804      assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
805
806      // Trigger a major compaction
807      admin.flush(tableName);
808      admin.majorCompact(tableName);
809      Thread.sleep(6000);
810
811      // Update the value at timestamp 1
812      put = new Put(row);
813      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
814      hTable.put(put);
815
816      // Update the value at timestamp 2
817      put = new Put(row);
818      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
819      hTable.put(put);
820
821      // Trigger a major compaction
822      admin.flush(tableName);
823      admin.majorCompact(tableName);
824      Thread.sleep(6000);
825
826      // Check that the values at timestamp 2 and 1 got updated
827      result = hTable.get(get);
828      navigableMap = result.getMap().get(FAMILY).get(qualifier);
829      assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
830      assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
831    }
832  }
833
834  @Test public void testMajorCompactionBetweenTwoUpdates() throws Exception {
835    final TableName tableName = name.getTableName();
836    try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10);
837        Admin admin = TEST_UTIL.getAdmin()) {
838
839      // Write a column with values at timestamp 1, 2 and 3
840      byte[] row = Bytes.toBytes("row3");
841      byte[] qualifier = Bytes.toBytes("myCol");
842      Put put = new Put(row);
843      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
844      hTable.put(put);
845
846      put = new Put(row);
847      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
848      hTable.put(put);
849
850      put = new Put(row);
851      put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
852      hTable.put(put);
853
854      Get get = new Get(row);
855      get.addColumn(FAMILY, qualifier);
856      get.readAllVersions();
857
858      // Check that the column indeed has the right values at timestamps 1 and
859      // 2
860      Result result = hTable.get(get);
861      NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier);
862      assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
863      assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
864
865      // Trigger a major compaction
866      admin.flush(tableName);
867      admin.majorCompact(tableName);
868      Thread.sleep(6000);
869
870      // Update the value at timestamp 1
871      put = new Put(row);
872      put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
873      hTable.put(put);
874
875      // Trigger a major compaction
876      admin.flush(tableName);
877      admin.majorCompact(tableName);
878      Thread.sleep(6000);
879
880      // Update the value at timestamp 2
881      put = new Put(row);
882      put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
883      hTable.put(put);
884
885      // Trigger a major compaction
886      admin.flush(tableName);
887      admin.majorCompact(tableName);
888      Thread.sleep(6000);
889
890      // Check that the values at timestamp 2 and 1 got updated
891      result = hTable.get(get);
892      navigableMap = result.getMap().get(FAMILY).get(qualifier);
893
894      assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
895      assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
896    }
897  }
898
899  @Test public void testGet_EmptyTable() throws IOException {
900    try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
901      Get get = new Get(ROW);
902      get.addFamily(FAMILY);
903      Result r = table.get(get);
904      assertTrue(r.isEmpty());
905    }
906  }
907
908  @Test public void testGet_NullQualifier() throws IOException {
909    try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
910      Put put = new Put(ROW);
911      put.addColumn(FAMILY, QUALIFIER, VALUE);
912      table.put(put);
913
914      put = new Put(ROW);
915      put.addColumn(FAMILY, null, VALUE);
916      table.put(put);
917      LOG.info("Row put");
918
919      Get get = new Get(ROW);
920      get.addColumn(FAMILY, null);
921      Result r = table.get(get);
922      assertEquals(1, r.size());
923
924      get = new Get(ROW);
925      get.addFamily(FAMILY);
926      r = table.get(get);
927      assertEquals(2, r.size());
928    }
929  }
930
931  @Test public void testGet_NonExistentRow() throws IOException {
932    try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
933      Put put = new Put(ROW);
934      put.addColumn(FAMILY, QUALIFIER, VALUE);
935      table.put(put);
936      LOG.info("Row put");
937
938      Get get = new Get(ROW);
939      get.addFamily(FAMILY);
940      Result r = table.get(get);
941      assertFalse(r.isEmpty());
942      System.out.println("Row retrieved successfully");
943
944      byte[] missingrow = Bytes.toBytes("missingrow");
945      get = new Get(missingrow);
946      get.addFamily(FAMILY);
947      r = table.get(get);
948      assertTrue(r.isEmpty());
949      LOG.info("Row missing as it should be");
950    }
951  }
952
953  @Test public void testPut() throws IOException {
954    final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
955    final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
956    final byte[] row1 = Bytes.toBytes("row1");
957    final byte[] row2 = Bytes.toBytes("row2");
958    final byte[] value = Bytes.toBytes("abcd");
959    try (Table table = TEST_UTIL
960      .createTable(name.getTableName(), new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY })) {
961      Put put = new Put(row1);
962      put.addColumn(CONTENTS_FAMILY, null, value);
963      table.put(put);
964
965      put = new Put(row2);
966      put.addColumn(CONTENTS_FAMILY, null, value);
967
968      assertEquals(1, put.size());
969      assertEquals(1, put.getFamilyCellMap().get(CONTENTS_FAMILY).size());
970
971      // KeyValue v1 expectation.  Cast for now until we go all Cell all the time. TODO
972      KeyValue kv = (KeyValue) put.getFamilyCellMap().get(CONTENTS_FAMILY).get(0);
973
974      assertTrue(Bytes.equals(CellUtil.cloneFamily(kv), CONTENTS_FAMILY));
975      // will it return null or an empty byte array?
976      assertTrue(Bytes.equals(CellUtil.cloneQualifier(kv), new byte[0]));
977
978      assertTrue(Bytes.equals(CellUtil.cloneValue(kv), value));
979
980      table.put(put);
981
982      Scan scan = new Scan();
983      scan.addColumn(CONTENTS_FAMILY, null);
984      try (ResultScanner scanner = table.getScanner(scan)) {
985        for (Result r : scanner) {
986          for (Cell key : r.rawCells()) {
987            System.out.println(Bytes.toString(r.getRow()) + ": " + key.toString());
988          }
989        }
990      }
991    }
992  }
993
994  @Test public void testPutNoCF() throws IOException {
995    final byte[] BAD_FAM = Bytes.toBytes("BAD_CF");
996    final byte[] VAL = Bytes.toBytes(100);
997    try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
998      boolean caughtNSCFE = false;
999
1000      try {
1001        Put p = new Put(ROW);
1002        p.addColumn(BAD_FAM, QUALIFIER, VAL);
1003        table.put(p);
1004      } catch (Exception e) {
1005        caughtNSCFE = e instanceof NoSuchColumnFamilyException;
1006      }
1007      assertTrue("Should throw NoSuchColumnFamilyException", caughtNSCFE);
1008    }
1009  }
1010
1011  @Test public void testRowsPut() throws IOException {
1012    final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
1013    final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
1014    final int NB_BATCH_ROWS = 10;
1015    final byte[] value = Bytes.toBytes("abcd");
1016    try (Table table = TEST_UTIL
1017      .createTable(name.getTableName(), new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY })) {
1018      ArrayList<Put> rowsUpdate = new ArrayList<>();
1019      for (int i = 0; i < NB_BATCH_ROWS; i++) {
1020        byte[] row = Bytes.toBytes("row" + i);
1021        Put put = new Put(row);
1022        put.setDurability(Durability.SKIP_WAL);
1023        put.addColumn(CONTENTS_FAMILY, null, value);
1024        rowsUpdate.add(put);
1025      }
1026      table.put(rowsUpdate);
1027      Scan scan = new Scan();
1028      scan.addFamily(CONTENTS_FAMILY);
1029      try (ResultScanner scanner = table.getScanner(scan)) {
1030        int nbRows = 0;
1031        for (@SuppressWarnings("unused") Result row : scanner) {
1032          nbRows++;
1033        }
1034        assertEquals(NB_BATCH_ROWS, nbRows);
1035      }
1036    }
1037  }
1038
1039  @Test public void testRowsPutBufferedManyManyFlushes() throws IOException {
1040    final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
1041    final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
1042    final byte[] value = Bytes.toBytes("abcd");
1043    final int NB_BATCH_ROWS = 10;
1044    try (Table table = TEST_UTIL
1045      .createTable(name.getTableName(), new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY })) {
1046      ArrayList<Put> rowsUpdate = new ArrayList<>();
1047      for (int i = 0; i < NB_BATCH_ROWS * 10; i++) {
1048        byte[] row = Bytes.toBytes("row" + i);
1049        Put put = new Put(row);
1050        put.setDurability(Durability.SKIP_WAL);
1051        put.addColumn(CONTENTS_FAMILY, null, value);
1052        rowsUpdate.add(put);
1053      }
1054      table.put(rowsUpdate);
1055
1056      Scan scan = new Scan();
1057      scan.addFamily(CONTENTS_FAMILY);
1058      try (ResultScanner scanner = table.getScanner(scan)) {
1059        int nbRows = 0;
1060        for (@SuppressWarnings("unused") Result row : scanner) {
1061          nbRows++;
1062        }
1063        assertEquals(NB_BATCH_ROWS * 10, nbRows);
1064      }
1065    }
1066  }
1067
1068  @Test public void testAddKeyValue() {
1069    final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
1070    final byte[] value = Bytes.toBytes("abcd");
1071    final byte[] row1 = Bytes.toBytes("row1");
1072    final byte[] row2 = Bytes.toBytes("row2");
1073    byte[] qualifier = Bytes.toBytes("qf1");
1074    Put put = new Put(row1);
1075
1076    // Adding KeyValue with the same row
1077    KeyValue kv = new KeyValue(row1, CONTENTS_FAMILY, qualifier, value);
1078    boolean ok = true;
1079    try {
1080      put.add(kv);
1081    } catch (IOException e) {
1082      ok = false;
1083    }
1084    assertTrue(ok);
1085
1086    // Adding KeyValue with the different row
1087    kv = new KeyValue(row2, CONTENTS_FAMILY, qualifier, value);
1088    ok = false;
1089    try {
1090      put.add(kv);
1091    } catch (IOException e) {
1092      ok = true;
1093    }
1094    assertTrue(ok);
1095  }
1096
1097  /**
1098   * test for HBASE-737
1099   */
1100  @Test public void testHBase737() throws IOException {
1101    final byte[] FAM1 = Bytes.toBytes("fam1");
1102    final byte[] FAM2 = Bytes.toBytes("fam2");
1103    // Open table
1104    try (Table table = TEST_UTIL.createTable(name.getTableName(), new byte[][] { FAM1, FAM2 })) {
1105      // Insert some values
1106      Put put = new Put(ROW);
1107      put.addColumn(FAM1, Bytes.toBytes("letters"), Bytes.toBytes("abcdefg"));
1108      table.put(put);
1109      try {
1110        Thread.sleep(1000);
1111      } catch (InterruptedException i) {
1112        //ignore
1113      }
1114
1115      put = new Put(ROW);
1116      put.addColumn(FAM1, Bytes.toBytes("numbers"), Bytes.toBytes("123456"));
1117      table.put(put);
1118
1119      try {
1120        Thread.sleep(1000);
1121      } catch (InterruptedException i) {
1122        //ignore
1123      }
1124
1125      put = new Put(ROW);
1126      put.addColumn(FAM2, Bytes.toBytes("letters"), Bytes.toBytes("hijklmnop"));
1127      table.put(put);
1128
1129      long[] times = new long[3];
1130
1131      // First scan the memstore
1132
1133      Scan scan = new Scan();
1134      scan.addFamily(FAM1);
1135      scan.addFamily(FAM2);
1136      try (ResultScanner s = table.getScanner(scan)) {
1137        int index = 0;
1138        Result r;
1139        while ((r = s.next()) != null) {
1140          for (Cell key : r.rawCells()) {
1141            times[index++] = key.getTimestamp();
1142          }
1143        }
1144      }
1145      for (int i = 0; i < times.length - 1; i++) {
1146        for (int j = i + 1; j < times.length; j++) {
1147          assertTrue(times[j] > times[i]);
1148        }
1149      }
1150
1151      // Flush data to disk and try again
1152      TEST_UTIL.flush();
1153
1154      // Reset times
1155      Arrays.fill(times, 0);
1156
1157      try {
1158        Thread.sleep(1000);
1159      } catch (InterruptedException i) {
1160        //ignore
1161      }
1162      scan = new Scan();
1163      scan.addFamily(FAM1);
1164      scan.addFamily(FAM2);
1165      try (ResultScanner s = table.getScanner(scan)) {
1166        int index = 0;
1167        Result r = null;
1168        while ((r = s.next()) != null) {
1169          for (Cell key : r.rawCells()) {
1170            times[index++] = key.getTimestamp();
1171          }
1172        }
1173        for (int i = 0; i < times.length - 1; i++) {
1174          for (int j = i + 1; j < times.length; j++) {
1175            assertTrue(times[j] > times[i]);
1176          }
1177        }
1178      }
1179    }
1180  }
1181
1182  @Test public void testListTables() throws IOException {
1183    final String testTableName = name.getTableName().toString();
1184    final TableName tableName1 = TableName.valueOf(testTableName + "1");
1185    final TableName tableName2 = TableName.valueOf(testTableName + "2");
1186    final TableName tableName3 = TableName.valueOf(testTableName + "3");
1187    TableName[] tables = new TableName[] { tableName1, tableName2, tableName3 };
1188    for (TableName table : tables) {
1189      TEST_UTIL.createTable(table, FAMILY);
1190    }
1191    try (Admin admin = TEST_UTIL.getAdmin()) {
1192      List<TableDescriptor> ts = admin.listTableDescriptors();
1193      HashSet<TableDescriptor> result = new HashSet<>(ts);
1194      int size = result.size();
1195      assertTrue(size >= tables.length);
1196      for (TableName table : tables) {
1197        boolean found = false;
1198        for (TableDescriptor t : ts) {
1199          if (t.getTableName().equals(table)) {
1200            found = true;
1201            break;
1202          }
1203        }
1204        assertTrue("Not found: " + table, found);
1205      }
1206    }
1207  }
1208
1209  /**
1210   * simple test that just executes parts of the client
1211   * API that accept a pre-created Connection instance
1212   */
1213  @Test public void testUnmanagedHConnection() throws IOException {
1214    final TableName tableName = name.getTableName();
1215    TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
1216    try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
1217      Table t = conn.getTable(tableName);
1218      Admin admin = conn.getAdmin()) {
1219      assertTrue(admin.tableExists(tableName));
1220      assertTrue(t.get(new Get(ROW)).isEmpty());
1221    }
1222  }
1223
1224  /**
1225   * test of that unmanaged HConnections are able to reconnect
1226   * properly (see HBASE-5058)
1227   */
1228  @Test public void testUnmanagedHConnectionReconnect() throws Exception {
1229    Configuration conf = TEST_UTIL.getConfiguration();
1230    Class registryImpl = conf
1231      .getClass(HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY, ZKConnectionRegistry.class);
1232    // This test does not make sense for MasterRegistry since it stops the only master in the
1233    // cluster and starts a new master without populating the underlying config for the connection.
1234    Assume.assumeFalse(registryImpl.equals(MasterRegistry.class));
1235    final TableName tableName = name.getTableName();
1236    TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
1237    try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) {
1238      try (Table t = conn.getTable(tableName); Admin admin = conn.getAdmin()) {
1239        assertTrue(admin.tableExists(tableName));
1240        assertTrue(t.get(new Get(ROW)).isEmpty());
1241      }
1242
1243      // stop the master
1244      SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1245      cluster.stopMaster(0, false);
1246      cluster.waitOnMaster(0);
1247
1248      // start up a new master
1249      cluster.startMaster();
1250      assertTrue(cluster.waitForActiveAndReadyMaster());
1251
1252      // test that the same unmanaged connection works with a new
1253      // Admin and can connect to the new master;
1254      try (Admin admin = conn.getAdmin()) {
1255        assertTrue(admin.tableExists(tableName));
1256        assertEquals(
1257          admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().size(),
1258          SLAVES);
1259      }
1260    }
1261  }
1262
1263  @Test public void testMiscHTableStuff() throws IOException {
1264    final String testTableName = name.getTableName().toString();
1265    final TableName tableAname = TableName.valueOf(testTableName + "A");
1266    final TableName tableBname = TableName.valueOf(testTableName + "B");
1267    final byte[] attrName = Bytes.toBytes("TESTATTR");
1268    final byte[] attrValue = Bytes.toBytes("somevalue");
1269    byte[] value = Bytes.toBytes("value");
1270
1271    try (Table a = TEST_UTIL.createTable(tableAname, HConstants.CATALOG_FAMILY);
1272      Table b = TEST_UTIL.createTable(tableBname, HConstants.CATALOG_FAMILY)) {
1273      Put put = new Put(ROW);
1274      put.addColumn(HConstants.CATALOG_FAMILY, null, value);
1275      a.put(put);
1276
1277      // open a new connection to A and a connection to b
1278      try (Table newA = TEST_UTIL.getConnection().getTable(tableAname)) {
1279
1280        // copy data from A to B
1281        Scan scan = new Scan();
1282        scan.addFamily(HConstants.CATALOG_FAMILY);
1283        try (ResultScanner s = newA.getScanner(scan)) {
1284          for (Result r : s) {
1285            put = new Put(r.getRow());
1286            put.setDurability(Durability.SKIP_WAL);
1287            for (Cell kv : r.rawCells()) {
1288              put.add(kv);
1289            }
1290            b.put(put);
1291          }
1292        }
1293      }
1294
1295      // Opening a new connection to A will cause the tables to be reloaded
1296      try (Table anotherA = TEST_UTIL.getConnection().getTable(tableAname)) {
1297        Get get = new Get(ROW);
1298        get.addFamily(HConstants.CATALOG_FAMILY);
1299        anotherA.get(get);
1300      }
1301
1302      // We can still access A through newA because it has the table information
1303      // cached. And if it needs to recalibrate, that will cause the information
1304      // to be reloaded.
1305
1306      // Test user metadata
1307      Admin admin = TEST_UTIL.getAdmin();
1308      // make a modifiable descriptor
1309      TableDescriptor desc = a.getDescriptor();
1310      // offline the table
1311      admin.disableTable(tableAname);
1312      // add a user attribute to HTD
1313      TableDescriptorBuilder builder =
1314        TableDescriptorBuilder.newBuilder(desc).setValue(attrName, attrValue);
1315      // add a user attribute to HCD
1316      for (ColumnFamilyDescriptor c : desc.getColumnFamilies()) {
1317        builder.modifyColumnFamily(
1318          ColumnFamilyDescriptorBuilder.newBuilder(c).setValue(attrName, attrValue).build());
1319      }
1320      // update metadata for all regions of this table
1321      admin.modifyTable(builder.build());
1322      // enable the table
1323      admin.enableTable(tableAname);
1324
1325      // Test that attribute changes were applied
1326      desc = a.getDescriptor();
1327      assertEquals("wrong table descriptor returned", desc.getTableName(), tableAname);
1328      // check HTD attribute
1329      value = desc.getValue(attrName);
1330      assertNotNull("missing HTD attribute value", value);
1331      assertFalse("HTD attribute value is incorrect", Bytes.compareTo(value, attrValue) != 0);
1332      // check HCD attribute
1333      for (ColumnFamilyDescriptor c : desc.getColumnFamilies()) {
1334        value = c.getValue(attrName);
1335        assertNotNull("missing HCD attribute value", value);
1336        assertFalse("HCD attribute value is incorrect", Bytes.compareTo(value, attrValue) != 0);
1337      }
1338    }
1339  }
1340}
1341