001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertNull;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.nio.charset.StandardCharsets;
025import java.util.ArrayList;
026import java.util.List;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.fs.Path;
029import org.apache.hadoop.hbase.Cell;
030import org.apache.hadoop.hbase.CellUtil;
031import org.apache.hadoop.hbase.HBaseClassTestRule;
032import org.apache.hadoop.hbase.HBaseTestingUtil;
033import org.apache.hadoop.hbase.HConstants;
034import org.apache.hadoop.hbase.MetaTableAccessor;
035import org.apache.hadoop.hbase.TableDescriptors;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.client.Delete;
038import org.apache.hadoop.hbase.client.Durability;
039import org.apache.hadoop.hbase.client.Put;
040import org.apache.hadoop.hbase.client.RegionInfo;
041import org.apache.hadoop.hbase.client.RegionInfoBuilder;
042import org.apache.hadoop.hbase.client.Result;
043import org.apache.hadoop.hbase.client.Scan;
044import org.apache.hadoop.hbase.client.TableDescriptor;
045import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
046import org.apache.hadoop.hbase.testclassification.RegionServerTests;
047import org.apache.hadoop.hbase.testclassification.SmallTests;
048import org.apache.hadoop.hbase.util.Bytes;
049import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
050import org.apache.hadoop.hbase.util.FSTableDescriptors;
051import org.apache.hadoop.hbase.wal.WAL;
052import org.junit.ClassRule;
053import org.junit.Rule;
054import org.junit.Test;
055import org.junit.experimental.categories.Category;
056import org.junit.rules.TestName;
057import org.slf4j.Logger;
058import org.slf4j.LoggerFactory;
059
060/**
061 * TestGet is a medley of tests of get all done up as a single test. It was originally written to
062 * test a method since removed, getClosestAtOrBefore but the test is retained because it runs some
063 * interesting exercises.
064 */
065@Category({ RegionServerTests.class, SmallTests.class })
066public class TestGetClosestAtOrBefore {
067
068  @ClassRule
069  public static final HBaseClassTestRule CLASS_RULE =
070    HBaseClassTestRule.forClass(TestGetClosestAtOrBefore.class);
071
072  @Rule
073  public TestName testName = new TestName();
074  private static final Logger LOG = LoggerFactory.getLogger(TestGetClosestAtOrBefore.class);
075
076  private static final byte[] T00 = Bytes.toBytes("000");
077  private static final byte[] T10 = Bytes.toBytes("010");
078  private static final byte[] T11 = Bytes.toBytes("011");
079  private static final byte[] T12 = Bytes.toBytes("012");
080  private static final byte[] T20 = Bytes.toBytes("020");
081  private static final byte[] T30 = Bytes.toBytes("030");
082  private static final byte[] T31 = Bytes.toBytes("031");
083  private static final byte[] T35 = Bytes.toBytes("035");
084  private static final byte[] T40 = Bytes.toBytes("040");
085
086  private static HBaseTestingUtil UTIL = new HBaseTestingUtil();
087  private static Configuration conf = UTIL.getConfiguration();
088
089  @Test
090  public void testUsingMetaAndBinary() throws IOException {
091    Path rootdir = UTIL.getDataTestDirOnTestFS();
092    // Up flush size else we bind up when we use default catalog flush of 16k.
093    TableDescriptors tds = new FSTableDescriptors(UTIL.getConfiguration());
094    FSTableDescriptors.tryUpdateMetaTableDescriptor(UTIL.getConfiguration());
095    TableDescriptor td = tds.get(TableName.META_TABLE_NAME);
096    td = TableDescriptorBuilder.newBuilder(td).setMemStoreFlushSize(64 * 1024 * 1024).build();
097    HRegion mr = HBaseTestingUtil.createRegionAndWAL(RegionInfoBuilder.FIRST_META_REGIONINFO,
098      rootdir, conf, td);
099    try {
100      // Write rows for three tables 'A', 'B', and 'C'.
101      for (char c = 'A'; c < 'D'; c++) {
102        TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf("" + c)).build();
103        final int last = 128;
104        final int interval = 2;
105        for (int i = 0; i <= last; i += interval) {
106          RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName())
107            .setStartKey(i == 0 ? HConstants.EMPTY_BYTE_ARRAY : Bytes.toBytes((byte) i))
108            .setEndKey(i == last ? HConstants.EMPTY_BYTE_ARRAY : Bytes.toBytes((byte) i + interval))
109            .build();
110          Put put =
111            MetaTableAccessor.makePutFromRegionInfo(hri, EnvironmentEdgeManager.currentTime());
112          put.setDurability(Durability.SKIP_WAL);
113          LOG.info("Put {}", put);
114          mr.put(put);
115        }
116      }
117      InternalScanner s = mr.getScanner(new Scan());
118      try {
119        List<Cell> keys = new ArrayList<>();
120        while (s.next(keys)) {
121          LOG.info("Scan {}", keys);
122          keys.clear();
123        }
124      } finally {
125        s.close();
126      }
127      findRow(mr, 'C', 44, 44);
128      findRow(mr, 'C', 45, 44);
129      findRow(mr, 'C', 46, 46);
130      findRow(mr, 'C', 43, 42);
131      mr.flush(true);
132      findRow(mr, 'C', 44, 44);
133      findRow(mr, 'C', 45, 44);
134      findRow(mr, 'C', 46, 46);
135      findRow(mr, 'C', 43, 42);
136      // Now delete 'C' and make sure I don't get entries from 'B'.
137      byte[] firstRowInC = RegionInfo.createRegionName(TableName.valueOf("" + 'C'),
138        HConstants.EMPTY_BYTE_ARRAY, HConstants.ZEROES, false);
139      Scan scan = new Scan().withStartRow(firstRowInC);
140      s = mr.getScanner(scan);
141      try {
142        List<Cell> keys = new ArrayList<>();
143        while (s.next(keys)) {
144          LOG.info("Delete {}", keys);
145          mr.delete(new Delete(CellUtil.cloneRow(keys.get(0))));
146          keys.clear();
147        }
148      } finally {
149        s.close();
150      }
151      // Assert we get null back (pass -1).
152      findRow(mr, 'C', 44, -1);
153      findRow(mr, 'C', 45, -1);
154      findRow(mr, 'C', 46, -1);
155      findRow(mr, 'C', 43, -1);
156      mr.flush(true);
157      findRow(mr, 'C', 44, -1);
158      findRow(mr, 'C', 45, -1);
159      findRow(mr, 'C', 46, -1);
160      findRow(mr, 'C', 43, -1);
161    } finally {
162      HBaseTestingUtil.closeRegionAndWAL(mr);
163    }
164  }
165
166  /**
167   * @param answer Pass -1 if we're not to find anything.
168   * @return Row found.
169   */
170  private byte[] findRow(final Region mr, final char table, final int rowToFind, final int answer)
171    throws IOException {
172    TableName tableb = TableName.valueOf("" + table);
173    // Find the row.
174    byte[] tofindBytes = Bytes.toBytes((short) rowToFind);
175    byte[] metaKey = RegionInfo.createRegionName(tableb, tofindBytes, HConstants.NINES, false);
176    LOG.info("find=" + new String(metaKey, StandardCharsets.UTF_8));
177    Result r = UTIL.getClosestRowBefore(mr, metaKey, HConstants.CATALOG_FAMILY);
178    if (answer == -1) {
179      assertNull(r);
180      return null;
181    }
182    assertTrue(
183      Bytes.compareTo(Bytes.toBytes((short) answer), extractRowFromMetaRow(r.getRow())) == 0);
184    return r.getRow();
185  }
186
187  private byte[] extractRowFromMetaRow(final byte[] b) {
188    int firstDelimiter = Bytes.searchDelimiterIndex(b, 0, b.length, HConstants.DELIMITER);
189    int lastDelimiter = Bytes.searchDelimiterIndexInReverse(b, 0, b.length, HConstants.DELIMITER);
190    int length = lastDelimiter - firstDelimiter - 1;
191    byte[] row = new byte[length];
192    System.arraycopy(b, firstDelimiter + 1, row, 0, length);
193    return row;
194  }
195
196  /**
197   * Test file of multiple deletes and with deletes as final key.
198   * @see <a href="https://issues.apache.org/jira/browse/HBASE-751">HBASE-751</a>
199   */
200  @Test
201  public void testGetClosestRowBefore3() throws IOException {
202    HRegion region = null;
203    byte[] c0 = HBaseTestingUtil.COLUMNS[0];
204    byte[] c1 = HBaseTestingUtil.COLUMNS[1];
205    try {
206      TableName tn = TableName.valueOf(testName.getMethodName());
207      TableDescriptor htd = UTIL.createTableDescriptor(tn);
208      region = UTIL.createLocalHRegion(htd, null, null);
209
210      Put p = new Put(T00);
211      p.addColumn(c0, c0, T00);
212      region.put(p);
213
214      p = new Put(T10);
215      p.addColumn(c0, c0, T10);
216      region.put(p);
217
218      p = new Put(T20);
219      p.addColumn(c0, c0, T20);
220      region.put(p);
221
222      Result r = UTIL.getClosestRowBefore(region, T20, c0);
223      assertTrue(Bytes.equals(T20, r.getRow()));
224
225      Delete d = new Delete(T20);
226      d.addColumn(c0, c0);
227      region.delete(d);
228
229      r = UTIL.getClosestRowBefore(region, T20, c0);
230      assertTrue(Bytes.equals(T10, r.getRow()));
231
232      p = new Put(T30);
233      p.addColumn(c0, c0, T30);
234      region.put(p);
235
236      r = UTIL.getClosestRowBefore(region, T30, c0);
237      assertTrue(Bytes.equals(T30, r.getRow()));
238
239      d = new Delete(T30);
240      d.addColumn(c0, c0);
241      region.delete(d);
242
243      r = UTIL.getClosestRowBefore(region, T30, c0);
244      assertTrue(Bytes.equals(T10, r.getRow()));
245      r = UTIL.getClosestRowBefore(region, T31, c0);
246      assertTrue(Bytes.equals(T10, r.getRow()));
247
248      region.flush(true);
249
250      // try finding "010" after flush
251      r = UTIL.getClosestRowBefore(region, T30, c0);
252      assertTrue(Bytes.equals(T10, r.getRow()));
253      r = UTIL.getClosestRowBefore(region, T31, c0);
254      assertTrue(Bytes.equals(T10, r.getRow()));
255
256      // Put into a different column family. Should make it so I still get t10
257      p = new Put(T20);
258      p.addColumn(c1, c1, T20);
259      region.put(p);
260
261      r = UTIL.getClosestRowBefore(region, T30, c0);
262      assertTrue(Bytes.equals(T10, r.getRow()));
263      r = UTIL.getClosestRowBefore(region, T31, c0);
264      assertTrue(Bytes.equals(T10, r.getRow()));
265
266      region.flush(true);
267
268      r = UTIL.getClosestRowBefore(region, T30, c0);
269      assertTrue(Bytes.equals(T10, r.getRow()));
270      r = UTIL.getClosestRowBefore(region, T31, c0);
271      assertTrue(Bytes.equals(T10, r.getRow()));
272
273      // Now try combo of memcache and mapfiles. Delete the t20 COLUMS[1]
274      // in memory; make sure we get back t10 again.
275      d = new Delete(T20);
276      d.addColumn(c1, c1);
277      region.delete(d);
278      r = UTIL.getClosestRowBefore(region, T30, c0);
279      assertTrue(Bytes.equals(T10, r.getRow()));
280
281      // Ask for a value off the end of the file. Should return t10.
282      r = UTIL.getClosestRowBefore(region, T31, c0);
283      assertTrue(Bytes.equals(T10, r.getRow()));
284      region.flush(true);
285      r = UTIL.getClosestRowBefore(region, T31, c0);
286      assertTrue(Bytes.equals(T10, r.getRow()));
287
288      // Ok. Let the candidate come out of hfile but have delete of
289      // the candidate be in memory.
290      p = new Put(T11);
291      p.addColumn(c0, c0, T11);
292      region.put(p);
293      d = new Delete(T10);
294      d.addColumn(c1, c1);
295      r = UTIL.getClosestRowBefore(region, T12, c0);
296      assertTrue(Bytes.equals(T11, r.getRow()));
297    } finally {
298      if (region != null) {
299        try {
300          WAL wal = region.getWAL();
301          region.close();
302          wal.close();
303        } catch (Exception e) {
304          e.printStackTrace();
305        }
306      }
307    }
308  }
309
310  /** For HBASE-694 */
311  @Test
312  public void testGetClosestRowBefore2() throws IOException {
313    HRegion region = null;
314    byte[] c0 = HBaseTestingUtil.COLUMNS[0];
315    try {
316      TableName tn = TableName.valueOf(testName.getMethodName());
317      TableDescriptor htd = UTIL.createTableDescriptor(tn);
318      region = UTIL.createLocalHRegion(htd, null, null);
319
320      Put p = new Put(T10);
321      p.addColumn(c0, c0, T10);
322      region.put(p);
323
324      p = new Put(T30);
325      p.addColumn(c0, c0, T30);
326      region.put(p);
327
328      p = new Put(T40);
329      p.addColumn(c0, c0, T40);
330      region.put(p);
331
332      // try finding "035"
333      Result r = UTIL.getClosestRowBefore(region, T35, c0);
334      assertTrue(Bytes.equals(T30, r.getRow()));
335
336      region.flush(true);
337
338      // try finding "035"
339      r = UTIL.getClosestRowBefore(region, T35, c0);
340      assertTrue(Bytes.equals(T30, r.getRow()));
341
342      p = new Put(T20);
343      p.addColumn(c0, c0, T20);
344      region.put(p);
345
346      // try finding "035"
347      r = UTIL.getClosestRowBefore(region, T35, c0);
348      assertTrue(Bytes.equals(T30, r.getRow()));
349
350      region.flush(true);
351
352      // try finding "035"
353      r = UTIL.getClosestRowBefore(region, T35, c0);
354      assertTrue(Bytes.equals(T30, r.getRow()));
355    } finally {
356      if (region != null) {
357        try {
358          WAL wal = region.getWAL();
359          region.close();
360          wal.close();
361        } catch (Exception e) {
362          e.printStackTrace();
363        }
364      }
365    }
366  }
367
368}