001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.hamcrest.MatcherAssert.assertThat;
021import static org.hamcrest.Matchers.containsString;
022import static org.junit.jupiter.api.Assertions.assertEquals;
023import static org.junit.jupiter.api.Assertions.assertFalse;
024import static org.junit.jupiter.api.Assertions.assertNull;
025import static org.junit.jupiter.api.Assertions.assertThrows;
026import static org.junit.jupiter.api.Assertions.assertTrue;
027import static org.junit.jupiter.api.Assertions.fail;
028
029import java.io.IOException;
030import java.util.ArrayList;
031import java.util.Arrays;
032import java.util.Collections;
033import java.util.List;
034import java.util.Random;
035import java.util.concurrent.ThreadLocalRandom;
036import org.apache.hadoop.hbase.HRegionLocation;
037import org.apache.hadoop.hbase.RegionMetrics;
038import org.apache.hadoop.hbase.ServerName;
039import org.apache.hadoop.hbase.ipc.RpcClient;
040import org.apache.hadoop.hbase.ipc.RpcClientFactory;
041import org.apache.hadoop.hbase.util.Bytes;
042import org.junit.jupiter.api.TestTemplate;
043import org.slf4j.Logger;
044import org.slf4j.LoggerFactory;
045
046public class FromClientSideTest2 extends FromClientSideTestBase {
047
048  protected FromClientSideTest2(Class<? extends ConnectionRegistry> registryImpl,
049    int numHedgedReqs) {
050    super(registryImpl, numHedgedReqs);
051  }
052
053  private static final Logger LOG = LoggerFactory.getLogger(FromClientSideTest2.class);
054
055  private static int WAITTABLE_MILLIS;
056  private static byte[] ANOTHERROW;
057  private static byte[] COL_QUAL;
058  private static byte[] VAL_BYTES;
059  private static byte[] ROW_BYTES;
060
061  protected static void startCluster(Class<?>... cps) throws Exception {
062    WAITTABLE_MILLIS = 10000;
063    SLAVES = 3;
064    ANOTHERROW = Bytes.toBytes("anotherrow");
065    COL_QUAL = Bytes.toBytes("f1");
066    VAL_BYTES = Bytes.toBytes("v1");
067    ROW_BYTES = Bytes.toBytes("r1");
068    initialize(cps);
069  }
070
071  @TestTemplate
072  public void testHTableBatchWithEmptyPut() throws IOException, InterruptedException {
073    TEST_UTIL.createTable(tableName, new byte[][] { FAMILY });
074    TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS);
075    try (Connection conn = getConnection(); Table table = conn.getTable(tableName)) {
076      List<Put> actions = new ArrayList<>();
077      Object[] results = new Object[2];
078      // create an empty Put
079      Put put1 = new Put(ROW);
080      actions.add(put1);
081
082      Put put2 = new Put(ANOTHERROW);
083      put2.addColumn(FAMILY, QUALIFIER, VALUE);
084      actions.add(put2);
085
086      table.batch(actions, results);
087      fail("Empty Put should have failed the batch call");
088    } catch (IllegalArgumentException iae) {
089    }
090  }
091
092  // Test Table.batch with large amount of mutations against the same key.
093  // It used to trigger read lock's "Maximum lock count exceeded" Error.
094  @TestTemplate
095  public void testHTableWithLargeBatch() throws IOException, InterruptedException {
096    int sixtyFourK = 64 * 1024;
097    List<Put> actions = new ArrayList<>();
098    Object[] results = new Object[(sixtyFourK + 1) * 2];
099
100    TEST_UTIL.createTable(tableName, new byte[][] { FAMILY });
101    TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS);
102    try (Connection conn = getConnection(); Table table = conn.getTable(tableName)) {
103      for (int i = 0; i < sixtyFourK + 1; i++) {
104        Put put1 = new Put(ROW);
105        put1.addColumn(FAMILY, QUALIFIER, VALUE);
106        actions.add(put1);
107
108        Put put2 = new Put(ANOTHERROW);
109        put2.addColumn(FAMILY, QUALIFIER, VALUE);
110        actions.add(put2);
111      }
112
113      table.batch(actions, results);
114    }
115  }
116
117  @TestTemplate
118  public void testBatchWithRowMutation() throws Exception {
119    LOG.info("Starting testBatchWithRowMutation");
120    byte[][] QUALIFIERS = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b") };
121    TEST_UTIL.createTable(tableName, new byte[][] { FAMILY });
122    TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS);
123    try (Connection conn = getConnection(); Table table = conn.getTable(tableName)) {
124
125      RowMutations arm = RowMutations
126        .of(Collections.singletonList(new Put(ROW).addColumn(FAMILY, QUALIFIERS[0], VALUE)));
127      Object[] batchResult = new Object[1];
128      table.batch(Arrays.asList(arm), batchResult);
129
130      Get g = new Get(ROW);
131      Result r = table.get(g);
132      assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[0])));
133
134      arm = RowMutations.of(Arrays.asList(new Put(ROW).addColumn(FAMILY, QUALIFIERS[1], VALUE),
135        new Delete(ROW).addColumns(FAMILY, QUALIFIERS[0])));
136      table.batch(Arrays.asList(arm), batchResult);
137      r = table.get(g);
138      assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[1])));
139      assertNull(r.getValue(FAMILY, QUALIFIERS[0]));
140
141      // Test that we get the correct remote exception for RowMutations from batch()
142      RetriesExhaustedException e = assertThrows(RetriesExhaustedException.class, () -> {
143        RowMutations m = RowMutations.of(Collections.singletonList(
144          new Put(ROW).addColumn(new byte[] { 'b', 'o', 'g', 'u', 's' }, QUALIFIERS[0], VALUE)));
145        table.batch(Arrays.asList(m), batchResult);
146      }, "Expected RetriesExhaustedWithDetailsException with NoSuchColumnFamilyException");
147      assertThat(e.getMessage(), containsString("NoSuchColumnFamilyException"));
148    }
149  }
150
151  @TestTemplate
152  public void testBatchWithCheckAndMutate() throws Exception {
153    TEST_UTIL.createTable(tableName, new byte[][] { FAMILY });
154    TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS);
155    try (Connection conn = getConnection(); Table table = conn.getTable(tableName)) {
156      byte[] row1 = Bytes.toBytes("row1");
157      byte[] row2 = Bytes.toBytes("row2");
158      byte[] row3 = Bytes.toBytes("row3");
159      byte[] row4 = Bytes.toBytes("row4");
160      byte[] row5 = Bytes.toBytes("row5");
161      byte[] row6 = Bytes.toBytes("row6");
162      byte[] row7 = Bytes.toBytes("row7");
163
164      table
165        .put(Arrays.asList(new Put(row1).addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")),
166          new Put(row2).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")),
167          new Put(row3).addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")),
168          new Put(row4).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d")),
169          new Put(row5).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("e")),
170          new Put(row6).addColumn(FAMILY, Bytes.toBytes("F"), Bytes.toBytes(10L)),
171          new Put(row7).addColumn(FAMILY, Bytes.toBytes("G"), Bytes.toBytes("g"))));
172
173      CheckAndMutate checkAndMutate1 =
174        CheckAndMutate.newBuilder(row1).ifEquals(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a"))
175          .build(new RowMutations(row1)
176            .add(new Put(row1).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("g")))
177            .add(new Delete(row1).addColumns(FAMILY, Bytes.toBytes("A")))
178            .add(new Increment(row1).addColumn(FAMILY, Bytes.toBytes("C"), 3L))
179            .add(new Append(row1).addColumn(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("d"))));
180      Get get = new Get(row2).addColumn(FAMILY, Bytes.toBytes("B"));
181      RowMutations mutations =
182        new RowMutations(row3).add(new Delete(row3).addColumns(FAMILY, Bytes.toBytes("C")))
183          .add(new Put(row3).addColumn(FAMILY, Bytes.toBytes("F"), Bytes.toBytes("f")))
184          .add(new Increment(row3).addColumn(FAMILY, Bytes.toBytes("A"), 5L))
185          .add(new Append(row3).addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")));
186      CheckAndMutate checkAndMutate2 =
187        CheckAndMutate.newBuilder(row4).ifEquals(FAMILY, Bytes.toBytes("D"), Bytes.toBytes("a"))
188          .build(new Put(row4).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("h")));
189      Put put = new Put(row5).addColumn(FAMILY, Bytes.toBytes("E"), Bytes.toBytes("f"));
190      CheckAndMutate checkAndMutate3 =
191        CheckAndMutate.newBuilder(row6).ifEquals(FAMILY, Bytes.toBytes("F"), Bytes.toBytes(10L))
192          .build(new Increment(row6).addColumn(FAMILY, Bytes.toBytes("F"), 1));
193      CheckAndMutate checkAndMutate4 =
194        CheckAndMutate.newBuilder(row7).ifEquals(FAMILY, Bytes.toBytes("G"), Bytes.toBytes("g"))
195          .build(new Append(row7).addColumn(FAMILY, Bytes.toBytes("G"), Bytes.toBytes("g")));
196
197      List<Row> actions = Arrays.asList(checkAndMutate1, get, mutations, checkAndMutate2, put,
198        checkAndMutate3, checkAndMutate4);
199      Object[] results = new Object[actions.size()];
200      table.batch(actions, results);
201
202      CheckAndMutateResult checkAndMutateResult = (CheckAndMutateResult) results[0];
203      assertTrue(checkAndMutateResult.isSuccess());
204      assertEquals(3L,
205        Bytes.toLong(checkAndMutateResult.getResult().getValue(FAMILY, Bytes.toBytes("C"))));
206      assertEquals("d",
207        Bytes.toString(checkAndMutateResult.getResult().getValue(FAMILY, Bytes.toBytes("D"))));
208
209      assertEquals("b", Bytes.toString(((Result) results[1]).getValue(FAMILY, Bytes.toBytes("B"))));
210
211      Result result = (Result) results[2];
212      assertTrue(result.getExists());
213      assertEquals(5L, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("A"))));
214      assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B"))));
215
216      checkAndMutateResult = (CheckAndMutateResult) results[3];
217      assertFalse(checkAndMutateResult.isSuccess());
218      assertNull(checkAndMutateResult.getResult());
219
220      assertTrue(((Result) results[4]).isEmpty());
221
222      checkAndMutateResult = (CheckAndMutateResult) results[5];
223      assertTrue(checkAndMutateResult.isSuccess());
224      assertEquals(11,
225        Bytes.toLong(checkAndMutateResult.getResult().getValue(FAMILY, Bytes.toBytes("F"))));
226
227      checkAndMutateResult = (CheckAndMutateResult) results[6];
228      assertTrue(checkAndMutateResult.isSuccess());
229      assertEquals("gg",
230        Bytes.toString(checkAndMutateResult.getResult().getValue(FAMILY, Bytes.toBytes("G"))));
231
232      result = table.get(new Get(row1));
233      assertEquals("g", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B"))));
234      assertNull(result.getValue(FAMILY, Bytes.toBytes("A")));
235      assertEquals(3L, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("C"))));
236      assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D"))));
237
238      result = table.get(new Get(row3));
239      assertNull(result.getValue(FAMILY, Bytes.toBytes("C")));
240      assertEquals("f", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("F"))));
241      assertNull(Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("C"))));
242      assertEquals(5L, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("A"))));
243      assertEquals("b", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B"))));
244
245      result = table.get(new Get(row4));
246      assertEquals("d", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("D"))));
247
248      result = table.get(new Get(row5));
249      assertEquals("f", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("E"))));
250
251      result = table.get(new Get(row6));
252      assertEquals(11, Bytes.toLong(result.getValue(FAMILY, Bytes.toBytes("F"))));
253
254      result = table.get(new Get(row7));
255      assertEquals("gg", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("G"))));
256    }
257  }
258
259  @TestTemplate
260  public void testHTableExistsMethodSingleRegionSingleGet()
261    throws IOException, InterruptedException {
262    TEST_UTIL.createTable(tableName, new byte[][] { FAMILY });
263    TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS);
264    try (Connection conn = getConnection(); Table table = conn.getTable(tableName)) {
265      // Test with a single region table.
266      Put put = new Put(ROW);
267      put.addColumn(FAMILY, QUALIFIER, VALUE);
268
269      Get get = new Get(ROW);
270
271      boolean exist = table.exists(get);
272      assertFalse(exist);
273
274      table.put(put);
275
276      exist = table.exists(get);
277      assertTrue(exist);
278    }
279  }
280
281  @TestTemplate
282  public void testHTableExistsMethodSingleRegionMultipleGets()
283    throws IOException, InterruptedException {
284    TEST_UTIL.createTable(tableName, new byte[][] { FAMILY });
285    TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS);
286    try (Connection conn = getConnection(); Table table = conn.getTable(tableName)) {
287      Put put = new Put(ROW);
288      put.addColumn(FAMILY, QUALIFIER, VALUE);
289      table.put(put);
290
291      List<Get> gets = new ArrayList<>();
292      gets.add(new Get(ROW));
293      gets.add(new Get(ANOTHERROW));
294
295      boolean[] results = table.exists(gets);
296      assertTrue(results[0]);
297      assertFalse(results[1]);
298    }
299  }
300
301  @TestTemplate
302  public void testHTableExistsBeforeGet() throws IOException, InterruptedException {
303    TEST_UTIL.createTable(tableName, new byte[][] { FAMILY });
304    TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS);
305    try (Connection conn = getConnection(); Table table = conn.getTable(tableName)) {
306      Put put = new Put(ROW);
307      put.addColumn(FAMILY, QUALIFIER, VALUE);
308      table.put(put);
309
310      Get get = new Get(ROW);
311
312      boolean exist = table.exists(get);
313      assertEquals(true, exist);
314
315      Result result = table.get(get);
316      assertEquals(false, result.isEmpty());
317      assertTrue(Bytes.equals(VALUE, result.getValue(FAMILY, QUALIFIER)));
318    }
319  }
320
321  @TestTemplate
322  public void testHTableExistsAllBeforeGet() throws IOException, InterruptedException {
323    TEST_UTIL.createTable(tableName, new byte[][] { FAMILY });
324    TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS);
325    try (Connection conn = getConnection(); Table table = conn.getTable(tableName)) {
326      final byte[] ROW2 = Bytes.add(ROW, Bytes.toBytes("2"));
327      Put put = new Put(ROW);
328      put.addColumn(FAMILY, QUALIFIER, VALUE);
329      table.put(put);
330      put = new Put(ROW2);
331      put.addColumn(FAMILY, QUALIFIER, VALUE);
332      table.put(put);
333
334      Get get = new Get(ROW);
335      Get get2 = new Get(ROW2);
336      ArrayList<Get> getList = new ArrayList<>(2);
337      getList.add(get);
338      getList.add(get2);
339
340      boolean[] exists = table.exists(getList);
341      assertEquals(true, exists[0]);
342      assertEquals(true, exists[1]);
343
344      Result[] result = table.get(getList);
345      assertEquals(false, result[0].isEmpty());
346      assertTrue(Bytes.equals(VALUE, result[0].getValue(FAMILY, QUALIFIER)));
347      assertEquals(false, result[1].isEmpty());
348      assertTrue(Bytes.equals(VALUE, result[1].getValue(FAMILY, QUALIFIER)));
349    }
350  }
351
352  @TestTemplate
353  public void testGetEmptyRow() throws Exception {
354    // Create a table and put in 1 row
355    TEST_UTIL.createTable(tableName, new byte[][] { FAMILY });
356    TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS);
357    try (Connection conn = getConnection(); Table table = conn.getTable(tableName)) {
358      Put put = new Put(ROW_BYTES);
359      put.addColumn(FAMILY, COL_QUAL, VAL_BYTES);
360      table.put(put);
361
362      // Try getting the row with an empty row key
363      Result res = null;
364      try {
365        res = table.get(new Get(new byte[0]));
366        fail();
367      } catch (IllegalArgumentException e) {
368        // Expected.
369      }
370      assertTrue(res == null);
371      res = table.get(new Get(Bytes.toBytes("r1-not-exist")));
372      assertTrue(res.isEmpty() == true);
373      res = table.get(new Get(ROW_BYTES));
374      assertTrue(Arrays.equals(res.getValue(FAMILY, COL_QUAL), VAL_BYTES));
375    }
376  }
377
378  @TestTemplate
379  public void testConnectionDefaultUsesCodec() throws Exception {
380    try (
381      RpcClient client = RpcClientFactory.createClient(TEST_UTIL.getConfiguration(), "cluster")) {
382      assertTrue(client.hasCellBlockSupport());
383    }
384  }
385
386  private void randomCFPuts(Table table, byte[] row, byte[] family, int nPuts) throws Exception {
387    Put put = new Put(row);
388    Random rand = ThreadLocalRandom.current();
389    for (int i = 0; i < nPuts; i++) {
390      byte[] qualifier = Bytes.toBytes(rand.nextInt());
391      byte[] value = Bytes.toBytes(rand.nextInt());
392      put.addColumn(family, qualifier, value);
393    }
394    table.put(put);
395  }
396
397  private void performMultiplePutAndFlush(Admin admin, Table table, byte[] row, byte[] family,
398    int nFlushes, int nPuts) throws Exception {
399    for (int i = 0; i < nFlushes; i++) {
400      randomCFPuts(table, row, family, nPuts);
401      admin.flush(table.getName());
402    }
403  }
404
405  private int getStoreFileCount(Admin admin, ServerName serverName, RegionInfo region)
406    throws IOException {
407    for (RegionMetrics metrics : admin.getRegionMetrics(serverName, region.getTable())) {
408      if (Bytes.equals(region.getRegionName(), metrics.getRegionName())) {
409        return metrics.getStoreFileCount();
410      }
411    }
412    return 0;
413  }
414
415  // override the config settings at the CF level and ensure priority
416  @TestTemplate
417  public void testAdvancedConfigOverride() throws Exception {
418    /*
419     * Overall idea: (1) create 3 store files and issue a compaction. config's compaction.min == 3,
420     * so should work. (2) Increase the compaction.min toggle in the HTD to 5 and modify table. If
421     * we use the HTD value instead of the default config value, adding 3 files and issuing a
422     * compaction SHOULD NOT work (3) Decrease the compaction.min toggle in the HCD to 2 and modify
423     * table. The CF schema should override the Table schema and now cause a minor compaction.
424     */
425    TEST_UTIL.getConfiguration().setInt("hbase.hstore.compaction.min", 3);
426    TEST_UTIL.createTable(tableName, FAMILY, 10);
427    TEST_UTIL.waitTableAvailable(tableName, WAITTABLE_MILLIS);
428    try (Connection conn = getConnection(); Table table = conn.getTable(tableName);
429      Admin admin = conn.getAdmin()) {
430      // Create 3 store files.
431      byte[] row = Bytes.toBytes(ThreadLocalRandom.current().nextInt());
432      performMultiplePutAndFlush(admin, table, row, FAMILY, 3, 100);
433
434      try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
435        // Verify we have multiple store files.
436        HRegionLocation loc = locator.getRegionLocation(row, true);
437        assertTrue(getStoreFileCount(admin, loc.getServerName(), loc.getRegion()) > 1);
438
439        // Issue a compaction request
440        admin.compact(tableName);
441
442        // poll wait for the compactions to happen
443        for (int i = 0; i < 10 * 1000 / 40; ++i) {
444          // The number of store files after compaction should be lesser.
445          loc = locator.getRegionLocation(row, true);
446          if (!loc.getRegion().isOffline()) {
447            if (getStoreFileCount(admin, loc.getServerName(), loc.getRegion()) <= 1) {
448              break;
449            }
450          }
451          Thread.sleep(40);
452        }
453        // verify the compactions took place and that we didn't just time out
454        assertTrue(getStoreFileCount(admin, loc.getServerName(), loc.getRegion()) <= 1);
455
456        // change the compaction.min config option for this table to 5
457        LOG.info("hbase.hstore.compaction.min should now be 5");
458        TableDescriptor htd = TableDescriptorBuilder.newBuilder(table.getDescriptor())
459          .setValue("hbase.hstore.compaction.min", String.valueOf(5)).build();
460        admin.modifyTable(htd);
461        LOG.info("alter status finished");
462
463        // Create 3 more store files.
464        performMultiplePutAndFlush(admin, table, row, FAMILY, 3, 10);
465
466        // Issue a compaction request
467        admin.compact(tableName);
468
469        // This time, the compaction request should not happen
470        Thread.sleep(10 * 1000);
471        loc = locator.getRegionLocation(row, true);
472        int sfCount = getStoreFileCount(admin, loc.getServerName(), loc.getRegion());
473        assertTrue(sfCount > 1);
474
475        // change an individual CF's config option to 2 & online schema update
476        LOG.info("hbase.hstore.compaction.min should now be 2");
477        htd = TableDescriptorBuilder.newBuilder(htd)
478          .modifyColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(htd.getColumnFamily(FAMILY))
479            .setValue("hbase.hstore.compaction.min", String.valueOf(2)).build())
480          .build();
481        admin.modifyTable(htd);
482        LOG.info("alter status finished");
483
484        // Issue a compaction request
485        admin.compact(tableName);
486
487        // poll wait for the compactions to happen
488        for (int i = 0; i < 10 * 1000 / 40; ++i) {
489          loc = locator.getRegionLocation(row, true);
490          try {
491            if (getStoreFileCount(admin, loc.getServerName(), loc.getRegion()) < sfCount) {
492              break;
493            }
494          } catch (Exception e) {
495            LOG.debug("Waiting for region to come online: "
496              + Bytes.toStringBinary(loc.getRegion().getRegionName()));
497          }
498          Thread.sleep(40);
499        }
500
501        // verify the compaction took place and that we didn't just time out
502        assertTrue(getStoreFileCount(admin, loc.getServerName(), loc.getRegion()) < sfCount);
503
504        // Finally, ensure that we can remove a custom config value after we made it
505        LOG.info("Removing CF config value");
506        LOG.info("hbase.hstore.compaction.min should now be 5");
507        htd = TableDescriptorBuilder.newBuilder(htd)
508          .modifyColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(htd.getColumnFamily(FAMILY))
509            .setValue("hbase.hstore.compaction.min", null).build())
510          .build();
511        admin.modifyTable(htd);
512        LOG.info("alter status finished");
513        assertNull(table.getDescriptor().getColumnFamily(FAMILY)
514          .getValue(Bytes.toBytes("hbase.hstore.compaction.min")));
515      }
516    }
517  }
518}