001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertNull;
021import static org.junit.Assert.assertTrue;
022import java.io.IOException;
023import java.nio.charset.StandardCharsets;
024import java.util.ArrayList;
025import java.util.List;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.Cell;
029import org.apache.hadoop.hbase.CellUtil;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.HBaseTestingUtility;
032import org.apache.hadoop.hbase.HConstants;
033import org.apache.hadoop.hbase.HRegionInfo;
034import org.apache.hadoop.hbase.HTableDescriptor;
035import org.apache.hadoop.hbase.MetaTableAccessor;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.client.Delete;
038import org.apache.hadoop.hbase.client.Durability;
039import org.apache.hadoop.hbase.client.Put;
040import org.apache.hadoop.hbase.client.RegionInfo;
041import org.apache.hadoop.hbase.client.RegionInfoBuilder;
042import org.apache.hadoop.hbase.client.Result;
043import org.apache.hadoop.hbase.client.Scan;
044import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
045import org.apache.hadoop.hbase.testclassification.RegionServerTests;
046import org.apache.hadoop.hbase.testclassification.SmallTests;
047import org.apache.hadoop.hbase.util.Bytes;
048import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
049import org.apache.hadoop.hbase.wal.WAL;
050import org.junit.ClassRule;
051import org.junit.Rule;
052import org.junit.Test;
053import org.junit.experimental.categories.Category;
054import org.junit.rules.TestName;
055import org.slf4j.Logger;
056import org.slf4j.LoggerFactory;
057
058/**
059 * TestGet is a medley of tests of get all done up as a single test.
060 * It was originally written to test a method since removed, getClosestAtOrBefore
061 * but the test is retained because it runs some interesting exercises.
062 */
063@Category({RegionServerTests.class, SmallTests.class})
064public class TestGetClosestAtOrBefore  {
065
066  @ClassRule
067  public static final HBaseClassTestRule CLASS_RULE =
068      HBaseClassTestRule.forClass(TestGetClosestAtOrBefore.class);
069
070  @Rule public TestName testName = new TestName();
071  private static final Logger LOG = LoggerFactory.getLogger(TestGetClosestAtOrBefore.class);
072
073  private static final byte[] T00 = Bytes.toBytes("000");
074  private static final byte[] T10 = Bytes.toBytes("010");
075  private static final byte[] T11 = Bytes.toBytes("011");
076  private static final byte[] T12 = Bytes.toBytes("012");
077  private static final byte[] T20 = Bytes.toBytes("020");
078  private static final byte[] T30 = Bytes.toBytes("030");
079  private static final byte[] T31 = Bytes.toBytes("031");
080  private static final byte[] T35 = Bytes.toBytes("035");
081  private static final byte[] T40 = Bytes.toBytes("040");
082
083  private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
084  private static Configuration conf = UTIL.getConfiguration();
085
086  @Test
087  public void testUsingMetaAndBinary() throws IOException {
088    Path rootdir = UTIL.getDataTestDirOnTestFS();
089    // Up flush size else we bind up when we use default catalog flush of 16k.
090    TableDescriptorBuilder metaBuilder = UTIL.getMetaTableDescriptorBuilder()
091            .setMemStoreFlushSize(64 * 1024 * 1024);
092    HRegion mr = HBaseTestingUtility.createRegionAndWAL(HRegionInfo.FIRST_META_REGIONINFO,
093        rootdir, this.conf, metaBuilder.build());
094    try {
095      // Write rows for three tables 'A', 'B', and 'C'.
096      for (char c = 'A'; c < 'D'; c++) {
097        HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("" + c));
098        final int last = 128;
099        final int interval = 2;
100        for (int i = 0; i <= last; i += interval) {
101          RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName())
102            .setStartKey(i == 0 ? HConstants.EMPTY_BYTE_ARRAY : Bytes.toBytes((byte)i))
103            .setEndKey(i == last ? HConstants.EMPTY_BYTE_ARRAY :
104              Bytes.toBytes((byte)i + interval)).build();
105          Put put =
106            MetaTableAccessor.makePutFromRegionInfo(hri, EnvironmentEdgeManager.currentTime());
107          put.setDurability(Durability.SKIP_WAL);
108          LOG.info("Put {}", put);
109          mr.put(put);
110        }
111      }
112      InternalScanner s = mr.getScanner(new Scan());
113      try {
114        List<Cell> keys = new ArrayList<>();
115        while (s.next(keys)) {
116          LOG.info("Scan {}", keys);
117          keys.clear();
118        }
119      } finally {
120        s.close();
121      }
122      findRow(mr, 'C', 44, 44);
123      findRow(mr, 'C', 45, 44);
124      findRow(mr, 'C', 46, 46);
125      findRow(mr, 'C', 43, 42);
126      mr.flush(true);
127      findRow(mr, 'C', 44, 44);
128      findRow(mr, 'C', 45, 44);
129      findRow(mr, 'C', 46, 46);
130      findRow(mr, 'C', 43, 42);
131      // Now delete 'C' and make sure I don't get entries from 'B'.
132      byte[] firstRowInC = RegionInfo.createRegionName(TableName.valueOf("" + 'C'),
133        HConstants.EMPTY_BYTE_ARRAY, HConstants.ZEROES, false);
134      Scan scan = new Scan().withStartRow(firstRowInC);
135      s = mr.getScanner(scan);
136      try {
137        List<Cell> keys = new ArrayList<>();
138        while (s.next(keys)) {
139          LOG.info("Delete {}", keys);
140          mr.delete(new Delete(CellUtil.cloneRow(keys.get(0))));
141          keys.clear();
142        }
143      } finally {
144        s.close();
145      }
146      // Assert we get null back (pass -1).
147      findRow(mr, 'C', 44, -1);
148      findRow(mr, 'C', 45, -1);
149      findRow(mr, 'C', 46, -1);
150      findRow(mr, 'C', 43, -1);
151      mr.flush(true);
152      findRow(mr, 'C', 44, -1);
153      findRow(mr, 'C', 45, -1);
154      findRow(mr, 'C', 46, -1);
155      findRow(mr, 'C', 43, -1);
156    } finally {
157      HBaseTestingUtility.closeRegionAndWAL(mr);
158    }
159  }
160
161  /*
162   * @param mr
163   * @param table
164   * @param rowToFind
165   * @param answer Pass -1 if we're not to find anything.
166   * @return Row found.
167   * @throws IOException
168   */
169  private byte [] findRow(final Region mr, final char table,
170    final int rowToFind, final int answer)
171  throws IOException {
172    TableName tableb = TableName.valueOf("" + table);
173    // Find the row.
174    byte [] tofindBytes = Bytes.toBytes((short)rowToFind);
175    byte [] metaKey = HRegionInfo.createRegionName(
176        tableb, tofindBytes,
177      HConstants.NINES, false);
178    LOG.info("find=" + new String(metaKey, StandardCharsets.UTF_8));
179    Result r = UTIL.getClosestRowBefore(mr, metaKey, HConstants.CATALOG_FAMILY);
180    if (answer == -1) {
181      assertNull(r);
182      return null;
183    }
184    assertTrue(Bytes.compareTo(Bytes.toBytes((short)answer),
185      extractRowFromMetaRow(r.getRow())) == 0);
186    return r.getRow();
187  }
188
189  private byte [] extractRowFromMetaRow(final byte [] b) {
190    int firstDelimiter = Bytes.searchDelimiterIndex(b, 0, b.length,
191      HConstants.DELIMITER);
192    int lastDelimiter = Bytes.searchDelimiterIndexInReverse(b, 0, b.length,
193      HConstants.DELIMITER);
194    int length = lastDelimiter - firstDelimiter - 1;
195    byte [] row = new byte[length];
196    System.arraycopy(b, firstDelimiter + 1, row, 0, length);
197    return row;
198  }
199
200  /**
201   * Test file of multiple deletes and with deletes as final key.
202   * @see <a href="https://issues.apache.org/jira/browse/HBASE-751">HBASE-751</a>
203   */
204  @Test
205  public void testGetClosestRowBefore3() throws IOException{
206    HRegion region = null;
207    byte [] c0 = UTIL.COLUMNS[0];
208    byte [] c1 = UTIL.COLUMNS[1];
209    try {
210      TableName tn = TableName.valueOf(testName.getMethodName());
211      HTableDescriptor htd = UTIL.createTableDescriptor(tn);
212      region = UTIL.createLocalHRegion(htd, null, null);
213
214      Put p = new Put(T00);
215      p.addColumn(c0, c0, T00);
216      region.put(p);
217
218      p = new Put(T10);
219      p.addColumn(c0, c0, T10);
220      region.put(p);
221
222      p = new Put(T20);
223      p.addColumn(c0, c0, T20);
224      region.put(p);
225
226      Result r = UTIL.getClosestRowBefore(region, T20, c0);
227      assertTrue(Bytes.equals(T20, r.getRow()));
228
229      Delete d = new Delete(T20);
230      d.addColumn(c0, c0);
231      region.delete(d);
232
233      r = UTIL.getClosestRowBefore(region, T20, c0);
234      assertTrue(Bytes.equals(T10, r.getRow()));
235
236      p = new Put(T30);
237      p.addColumn(c0, c0, T30);
238      region.put(p);
239
240      r = UTIL.getClosestRowBefore(region, T30, c0);
241      assertTrue(Bytes.equals(T30, r.getRow()));
242
243      d = new Delete(T30);
244      d.addColumn(c0, c0);
245      region.delete(d);
246
247      r = UTIL.getClosestRowBefore(region, T30, c0);
248      assertTrue(Bytes.equals(T10, r.getRow()));
249      r = UTIL.getClosestRowBefore(region, T31, c0);
250      assertTrue(Bytes.equals(T10, r.getRow()));
251
252      region.flush(true);
253
254      // try finding "010" after flush
255      r = UTIL.getClosestRowBefore(region, T30, c0);
256      assertTrue(Bytes.equals(T10, r.getRow()));
257      r = UTIL.getClosestRowBefore(region, T31, c0);
258      assertTrue(Bytes.equals(T10, r.getRow()));
259
260      // Put into a different column family.  Should make it so I still get t10
261      p = new Put(T20);
262      p.addColumn(c1, c1, T20);
263      region.put(p);
264
265      r = UTIL.getClosestRowBefore(region, T30, c0);
266      assertTrue(Bytes.equals(T10, r.getRow()));
267      r = UTIL.getClosestRowBefore(region, T31, c0);
268      assertTrue(Bytes.equals(T10, r.getRow()));
269
270      region.flush(true);
271
272      r = UTIL.getClosestRowBefore(region, T30, c0);
273      assertTrue(Bytes.equals(T10, r.getRow()));
274      r = UTIL.getClosestRowBefore(region, T31, c0);
275      assertTrue(Bytes.equals(T10, r.getRow()));
276
277      // Now try combo of memcache and mapfiles.  Delete the t20 COLUMS[1]
278      // in memory; make sure we get back t10 again.
279      d = new Delete(T20);
280      d.addColumn(c1, c1);
281      region.delete(d);
282      r = UTIL.getClosestRowBefore(region, T30, c0);
283      assertTrue(Bytes.equals(T10, r.getRow()));
284
285      // Ask for a value off the end of the file.  Should return t10.
286      r = UTIL.getClosestRowBefore(region, T31, c0);
287      assertTrue(Bytes.equals(T10, r.getRow()));
288      region.flush(true);
289      r = UTIL.getClosestRowBefore(region, T31, c0);
290      assertTrue(Bytes.equals(T10, r.getRow()));
291
292      // Ok.  Let the candidate come out of hfile but have delete of
293      // the candidate be in memory.
294      p = new Put(T11);
295      p.addColumn(c0, c0, T11);
296      region.put(p);
297      d = new Delete(T10);
298      d.addColumn(c1, c1);
299      r = UTIL.getClosestRowBefore(region, T12, c0);
300      assertTrue(Bytes.equals(T11, r.getRow()));
301    } finally {
302      if (region != null) {
303        try {
304          WAL wal = region.getWAL();
305          region.close();
306          wal.close();
307        } catch (Exception e) {
308          e.printStackTrace();
309        }
310      }
311    }
312  }
313
314  /** For HBASE-694 */
315  @Test
316  public void testGetClosestRowBefore2() throws IOException{
317    HRegion region = null;
318    byte [] c0 = UTIL.COLUMNS[0];
319    try {
320      TableName tn = TableName.valueOf(testName.getMethodName());
321      HTableDescriptor htd = UTIL.createTableDescriptor(tn);
322      region = UTIL.createLocalHRegion(htd, null, null);
323
324      Put p = new Put(T10);
325      p.addColumn(c0, c0, T10);
326      region.put(p);
327
328      p = new Put(T30);
329      p.addColumn(c0, c0, T30);
330      region.put(p);
331
332      p = new Put(T40);
333      p.addColumn(c0, c0, T40);
334      region.put(p);
335
336      // try finding "035"
337      Result r = UTIL.getClosestRowBefore(region, T35, c0);
338      assertTrue(Bytes.equals(T30, r.getRow()));
339
340      region.flush(true);
341
342      // try finding "035"
343      r = UTIL.getClosestRowBefore(region, T35, c0);
344      assertTrue(Bytes.equals(T30, r.getRow()));
345
346      p = new Put(T20);
347      p.addColumn(c0, c0, T20);
348      region.put(p);
349
350      // try finding "035"
351      r = UTIL.getClosestRowBefore(region, T35, c0);
352      assertTrue(Bytes.equals(T30, r.getRow()));
353
354      region.flush(true);
355
356      // try finding "035"
357      r = UTIL.getClosestRowBefore(region, T35, c0);
358      assertTrue(Bytes.equals(T30, r.getRow()));
359    } finally {
360      if (region != null) {
361        try {
362          WAL wal = region.getWAL();
363          region.close();
364          wal.close();
365        } catch (Exception e) {
366          e.printStackTrace();
367        }
368      }
369    }
370  }
371
372}
373