001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.filter;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.Arrays;
023import java.util.List;
024import org.apache.commons.codec.binary.Hex;
025import org.apache.hadoop.hbase.HBaseClassTestRule;
026import org.apache.hadoop.hbase.TableName;
027import org.apache.hadoop.hbase.client.Put;
028import org.apache.hadoop.hbase.client.Result;
029import org.apache.hadoop.hbase.client.ResultScanner;
030import org.apache.hadoop.hbase.client.Scan;
031import org.apache.hadoop.hbase.client.Table;
032import org.apache.hadoop.hbase.testclassification.FilterTests;
033import org.apache.hadoop.hbase.testclassification.MediumTests;
034import org.apache.hadoop.hbase.util.Bytes;
035import org.junit.Assert;
036import org.junit.ClassRule;
037import org.junit.Rule;
038import org.junit.Test;
039import org.junit.experimental.categories.Category;
040import org.junit.rules.TestName;
041import org.slf4j.Logger;
042import org.slf4j.LoggerFactory;
043
044/**
045 * Test if Scan.setRowPrefixFilter works as intended.
046 */
047@Category({FilterTests.class, MediumTests.class})
048public class TestScanRowPrefix extends FilterTestingCluster {
049
050  @ClassRule
051  public static final HBaseClassTestRule CLASS_RULE =
052      HBaseClassTestRule.forClass(TestScanRowPrefix.class);
053
054  private static final Logger LOG = LoggerFactory
055      .getLogger(TestScanRowPrefix.class);
056
057  @Rule
058  public TestName name = new TestName();
059
060  @Test
061  public void testPrefixScanning() throws IOException {
062    final TableName tableName = TableName.valueOf(name.getMethodName());
063    createTable(tableName,"F");
064    Table table = openTable(tableName);
065
066    /**
067     * Note that about half of these tests were relevant for an different implementation approach
068     * of setRowPrefixFilter. These test cases have been retained to ensure that also the
069     * edge cases found there are still covered.
070     */
071
072    final byte[][] rowIds = {
073        {(byte) 0x11},                                                      //  0
074        {(byte) 0x12},                                                      //  1
075        {(byte) 0x12, (byte) 0x23, (byte) 0xFF, (byte) 0xFE},               //  2
076        {(byte) 0x12, (byte) 0x23, (byte) 0xFF, (byte) 0xFF},               //  3
077        {(byte) 0x12, (byte) 0x23, (byte) 0xFF, (byte) 0xFF, (byte) 0x00},  //  4
078        {(byte) 0x12, (byte) 0x23, (byte) 0xFF, (byte) 0xFF, (byte) 0x01},  //  5
079        {(byte) 0x12, (byte) 0x24},                                         //  6
080        {(byte) 0x12, (byte) 0x24, (byte) 0x00},                            //  7
081        {(byte) 0x12, (byte) 0x24, (byte) 0x00, (byte) 0x00},               //  8
082        {(byte) 0x12, (byte) 0x25},                                         //  9
083        {(byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF},  // 10
084    };
085    for (byte[] rowId: rowIds) {
086      Put p = new Put(rowId);
087      // Use the rowId as the column qualifier
088      p.addColumn(Bytes.toBytes("F"), rowId, Bytes.toBytes("Dummy value"));
089      table.put(p);
090    }
091
092    byte[] prefix0 = {};
093    List<byte[]> expected0 = new ArrayList<>(16);
094    expected0.addAll(Arrays.asList(rowIds)); // Expect all rows
095
096    byte[] prefix1 = {(byte) 0x12, (byte) 0x23};
097    List<byte[]> expected1 = new ArrayList<>(16);
098    expected1.add(rowIds[2]);
099    expected1.add(rowIds[3]);
100    expected1.add(rowIds[4]);
101    expected1.add(rowIds[5]);
102
103    byte[] prefix2 = {(byte) 0x12, (byte) 0x23, (byte) 0xFF, (byte) 0xFF};
104    List<byte[]> expected2 = new ArrayList<>();
105    expected2.add(rowIds[3]);
106    expected2.add(rowIds[4]);
107    expected2.add(rowIds[5]);
108
109    byte[] prefix3 = {(byte) 0x12, (byte) 0x24};
110    List<byte[]> expected3 = new ArrayList<>();
111    expected3.add(rowIds[6]);
112    expected3.add(rowIds[7]);
113    expected3.add(rowIds[8]);
114
115    byte[] prefix4 = {(byte) 0xFF, (byte) 0xFF};
116    List<byte[]> expected4 = new ArrayList<>();
117    expected4.add(rowIds[10]);
118
119    // ========
120    // PREFIX 0
121    Scan scan = new Scan();
122    scan.setRowPrefixFilter(prefix0);
123    verifyScanResult(table, scan, expected0, "Scan empty prefix failed");
124
125    // ========
126    // PREFIX 1
127    scan = new Scan();
128    scan.setRowPrefixFilter(prefix1);
129    verifyScanResult(table, scan, expected1, "Scan normal prefix failed");
130
131    scan.setRowPrefixFilter(null);
132    verifyScanResult(table, scan, expected0, "Scan after prefix reset failed");
133
134    scan = new Scan();
135    scan.setFilter(new ColumnPrefixFilter(prefix1));
136    verifyScanResult(table, scan, expected1, "Double check on column prefix failed");
137
138    // ========
139    // PREFIX 2
140    scan = new Scan();
141    scan.setRowPrefixFilter(prefix2);
142    verifyScanResult(table, scan, expected2, "Scan edge 0xFF prefix failed");
143
144    scan.setRowPrefixFilter(null);
145    verifyScanResult(table, scan, expected0, "Scan after prefix reset failed");
146
147    scan = new Scan();
148    scan.setFilter(new ColumnPrefixFilter(prefix2));
149    verifyScanResult(table, scan, expected2, "Double check on column prefix failed");
150
151    // ========
152    // PREFIX 3
153    scan = new Scan();
154    scan.setRowPrefixFilter(prefix3);
155    verifyScanResult(table, scan, expected3, "Scan normal with 0x00 ends failed");
156
157    scan.setRowPrefixFilter(null);
158    verifyScanResult(table, scan, expected0, "Scan after prefix reset failed");
159
160    scan = new Scan();
161    scan.setFilter(new ColumnPrefixFilter(prefix3));
162    verifyScanResult(table, scan, expected3, "Double check on column prefix failed");
163
164    // ========
165    // PREFIX 4
166    scan = new Scan();
167    scan.setRowPrefixFilter(prefix4);
168    verifyScanResult(table, scan, expected4, "Scan end prefix failed");
169
170    scan.setRowPrefixFilter(null);
171    verifyScanResult(table, scan, expected0, "Scan after prefix reset failed");
172
173    scan = new Scan();
174    scan.setFilter(new ColumnPrefixFilter(prefix4));
175    verifyScanResult(table, scan, expected4, "Double check on column prefix failed");
176
177    // ========
178    // COMBINED
179    // Prefix + Filter
180    scan = new Scan();
181    scan.setRowPrefixFilter(prefix1);
182    verifyScanResult(table, scan, expected1, "Prefix filter failed");
183
184    scan.setFilter(new ColumnPrefixFilter(prefix2));
185    verifyScanResult(table, scan, expected2, "Combined Prefix + Filter failed");
186
187    scan.setRowPrefixFilter(null);
188    verifyScanResult(table, scan, expected2, "Combined Prefix + Filter; removing Prefix failed");
189
190    scan.setFilter(null);
191    verifyScanResult(table, scan, expected0, "Scan after Filter reset failed");
192
193    // ========
194    // Reversed: Filter + Prefix
195    scan = new Scan();
196    scan.setFilter(new ColumnPrefixFilter(prefix2));
197    verifyScanResult(table, scan, expected2, "Test filter failed");
198
199    scan.setRowPrefixFilter(prefix1);
200    verifyScanResult(table, scan, expected2, "Combined Filter + Prefix failed");
201
202    scan.setFilter(null);
203    verifyScanResult(table, scan, expected1, "Combined Filter + Prefix ; removing Filter failed");
204
205    scan.setRowPrefixFilter(null);
206    verifyScanResult(table, scan, expected0, "Scan after prefix reset failed");
207  }
208
209  @Test
210  public void testRowPrefixFilterAndStartRow() throws IOException {
211    final TableName tableName = TableName.valueOf(name.getMethodName());
212    createTable(tableName,"F");
213    Table table = openTable(tableName);
214
215    final byte[][] rowkeys = {Bytes.toBytes("111"), Bytes.toBytes("112")};
216    final byte[] prefixFilter = Bytes.toBytes("11");
217    for (byte[] rowkey: rowkeys) {
218      Put p = new Put(rowkey);
219      p.addColumn(Bytes.toBytes("F"), Bytes.toBytes("f"), Bytes.toBytes("test value"));
220      table.put(p);
221    }
222
223    List<byte[]> expected0 = new ArrayList<>();
224    expected0.add(rowkeys[0]);
225    expected0.add(rowkeys[1]);
226
227    List<byte[]> expected1 = new ArrayList<>();
228    expected1.add(rowkeys[1]);
229
230    // ========
231    // First scan
232    // Set startRow before setRowPrefixFilter
233    Scan scan = new Scan();
234    scan.withStartRow(rowkeys[1]);
235    scan.setRowPrefixFilter(prefixFilter);
236    verifyScanResult(table, scan, expected0, "Set startRow before setRowPrefixFilter unexpected");
237
238    // ========
239    // Second scan
240    // Set startRow after setRowPrefixFilter
241    // The result is different from first scan
242    scan = new Scan();
243    scan.setRowPrefixFilter(prefixFilter);
244    scan.withStartRow(rowkeys[1]);
245    verifyScanResult(table, scan, expected1, "Set startRow after setRowPrefixFilter unexpected");
246  }
247
248  private void verifyScanResult(Table table, Scan scan, List<byte[]> expectedKeys, String message) {
249    List<byte[]> actualKeys = new ArrayList<>();
250    try {
251      ResultScanner scanner = table.getScanner(scan);
252      for (Result result : scanner) {
253        actualKeys.add(result.getRow());
254      }
255
256      String fullMessage = message;
257      if (LOG.isDebugEnabled()) {
258        fullMessage = message + "\n" + tableOfTwoListsOfByteArrays(
259                "Expected", expectedKeys,
260                "Actual  ", actualKeys);
261      }
262
263      Assert.assertArrayEquals(
264              fullMessage,
265              expectedKeys.toArray(),
266              actualKeys.toArray());
267    } catch (IOException e) {
268      e.printStackTrace();
269      Assert.fail();
270    }
271  }
272
273  private String printMultiple(char letter, int count) {
274    StringBuilder sb = new StringBuilder(count);
275    for (int i = 0; i < count; i++) {
276      sb.append(letter);
277    }
278    return sb.toString();
279  }
280
281  private String tableOfTwoListsOfByteArrays(
282          String label1, List<byte[]> listOfBytes1,
283          String label2, List<byte[]> listOfBytes2) {
284    int margin1 = calculateWidth(label1, listOfBytes1);
285    int margin2 = calculateWidth(label2, listOfBytes2);
286
287    StringBuilder sb = new StringBuilder(512);
288    String separator = '+' + printMultiple('-', margin1 + margin2 + 5) + '+' + '\n';
289    sb.append(separator);
290    sb.append(printLine(label1, margin1, label2, margin2)).append('\n');
291    sb.append(separator);
292    int maxLength = Math.max(listOfBytes1.size(), listOfBytes2.size());
293    for (int offset = 0; offset < maxLength; offset++) {
294      String value1 = getStringFromList(listOfBytes1, offset);
295      String value2 = getStringFromList(listOfBytes2, offset);
296      sb.append(printLine(value1, margin1, value2, margin2)).append('\n');
297    }
298    sb.append(separator).append('\n');
299    return sb.toString();
300  }
301
302  private String printLine(String leftValue, int leftWidth1, String rightValue, int rightWidth) {
303    return "| " +
304           leftValue  + printMultiple(' ', leftWidth1 - leftValue.length() ) +
305           " | " +
306           rightValue + printMultiple(' ', rightWidth - rightValue.length()) +
307           " |";
308  }
309
310  private int calculateWidth(String label1, List<byte[]> listOfBytes1) {
311    int longestList1 = label1.length();
312    for (byte[] value : listOfBytes1) {
313      longestList1 = Math.max(value.length * 2, longestList1);
314    }
315    return longestList1 + 5;
316  }
317
318  private String getStringFromList(List<byte[]> listOfBytes, int offset) {
319    String value1;
320    if (listOfBytes.size() > offset) {
321      value1 = Hex.encodeHexString(listOfBytes.get(offset));
322    } else {
323      value1 = "<missing>";
324    }
325    return value1;
326  }
327
328}