001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.filter;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.Arrays;
023import java.util.List;
024import org.apache.commons.codec.binary.Hex;
025import org.apache.hadoop.hbase.HBaseClassTestRule;
026import org.apache.hadoop.hbase.TableName;
027import org.apache.hadoop.hbase.client.Put;
028import org.apache.hadoop.hbase.client.Result;
029import org.apache.hadoop.hbase.client.ResultScanner;
030import org.apache.hadoop.hbase.client.Scan;
031import org.apache.hadoop.hbase.client.Table;
032import org.apache.hadoop.hbase.testclassification.FilterTests;
033import org.apache.hadoop.hbase.testclassification.MediumTests;
034import org.junit.Assert;
035import org.junit.ClassRule;
036import org.junit.Rule;
037import org.junit.Test;
038import org.junit.experimental.categories.Category;
039import org.junit.rules.TestName;
040import org.slf4j.Logger;
041import org.slf4j.LoggerFactory;
042
043/**
044 * Test if Scan.setRowPrefixFilter works as intended.
045 */
046@Category({FilterTests.class, MediumTests.class})
047public class TestScanRowPrefix extends FilterTestingCluster {
048
049  @ClassRule
050  public static final HBaseClassTestRule CLASS_RULE =
051      HBaseClassTestRule.forClass(TestScanRowPrefix.class);
052
053  private static final Logger LOG = LoggerFactory
054      .getLogger(TestScanRowPrefix.class);
055
056  @Rule
057  public TestName name = new TestName();
058
059  @Test
060  public void testPrefixScanning() throws IOException {
061    final TableName tableName = TableName.valueOf(name.getMethodName());
062    createTable(tableName,"F");
063    Table table = openTable(tableName);
064
065    /**
066     * Note that about half of these tests were relevant for an different implementation approach
067     * of setRowPrefixFilter. These test cases have been retained to ensure that also the
068     * edge cases found there are still covered.
069     */
070
071    final byte[][] rowIds = {
072        {(byte) 0x11},                                                      //  0
073        {(byte) 0x12},                                                      //  1
074        {(byte) 0x12, (byte) 0x23, (byte) 0xFF, (byte) 0xFE},               //  2
075        {(byte) 0x12, (byte) 0x23, (byte) 0xFF, (byte) 0xFF},               //  3
076        {(byte) 0x12, (byte) 0x23, (byte) 0xFF, (byte) 0xFF, (byte) 0x00},  //  4
077        {(byte) 0x12, (byte) 0x23, (byte) 0xFF, (byte) 0xFF, (byte) 0x01},  //  5
078        {(byte) 0x12, (byte) 0x24},                                         //  6
079        {(byte) 0x12, (byte) 0x24, (byte) 0x00},                            //  7
080        {(byte) 0x12, (byte) 0x24, (byte) 0x00, (byte) 0x00},               //  8
081        {(byte) 0x12, (byte) 0x25},                                         //  9
082        {(byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF},  // 10
083    };
084    for (byte[] rowId: rowIds) {
085      Put p = new Put(rowId);
086      // Use the rowId as the column qualifier
087      p.addColumn("F".getBytes(), rowId, "Dummy value".getBytes());
088      table.put(p);
089    }
090
091    byte[] prefix0 = {};
092    List<byte[]> expected0 = new ArrayList<>(16);
093    expected0.addAll(Arrays.asList(rowIds)); // Expect all rows
094
095    byte[] prefix1 = {(byte) 0x12, (byte) 0x23};
096    List<byte[]> expected1 = new ArrayList<>(16);
097    expected1.add(rowIds[2]);
098    expected1.add(rowIds[3]);
099    expected1.add(rowIds[4]);
100    expected1.add(rowIds[5]);
101
102    byte[] prefix2 = {(byte) 0x12, (byte) 0x23, (byte) 0xFF, (byte) 0xFF};
103    List<byte[]> expected2 = new ArrayList<>();
104    expected2.add(rowIds[3]);
105    expected2.add(rowIds[4]);
106    expected2.add(rowIds[5]);
107
108    byte[] prefix3 = {(byte) 0x12, (byte) 0x24};
109    List<byte[]> expected3 = new ArrayList<>();
110    expected3.add(rowIds[6]);
111    expected3.add(rowIds[7]);
112    expected3.add(rowIds[8]);
113
114    byte[] prefix4 = {(byte) 0xFF, (byte) 0xFF};
115    List<byte[]> expected4 = new ArrayList<>();
116    expected4.add(rowIds[10]);
117
118    // ========
119    // PREFIX 0
120    Scan scan = new Scan();
121    scan.setRowPrefixFilter(prefix0);
122    verifyScanResult(table, scan, expected0, "Scan empty prefix failed");
123
124    // ========
125    // PREFIX 1
126    scan = new Scan();
127    scan.setRowPrefixFilter(prefix1);
128    verifyScanResult(table, scan, expected1, "Scan normal prefix failed");
129
130    scan.setRowPrefixFilter(null);
131    verifyScanResult(table, scan, expected0, "Scan after prefix reset failed");
132
133    scan = new Scan();
134    scan.setFilter(new ColumnPrefixFilter(prefix1));
135    verifyScanResult(table, scan, expected1, "Double check on column prefix failed");
136
137    // ========
138    // PREFIX 2
139    scan = new Scan();
140    scan.setRowPrefixFilter(prefix2);
141    verifyScanResult(table, scan, expected2, "Scan edge 0xFF prefix failed");
142
143    scan.setRowPrefixFilter(null);
144    verifyScanResult(table, scan, expected0, "Scan after prefix reset failed");
145
146    scan = new Scan();
147    scan.setFilter(new ColumnPrefixFilter(prefix2));
148    verifyScanResult(table, scan, expected2, "Double check on column prefix failed");
149
150    // ========
151    // PREFIX 3
152    scan = new Scan();
153    scan.setRowPrefixFilter(prefix3);
154    verifyScanResult(table, scan, expected3, "Scan normal with 0x00 ends failed");
155
156    scan.setRowPrefixFilter(null);
157    verifyScanResult(table, scan, expected0, "Scan after prefix reset failed");
158
159    scan = new Scan();
160    scan.setFilter(new ColumnPrefixFilter(prefix3));
161    verifyScanResult(table, scan, expected3, "Double check on column prefix failed");
162
163    // ========
164    // PREFIX 4
165    scan = new Scan();
166    scan.setRowPrefixFilter(prefix4);
167    verifyScanResult(table, scan, expected4, "Scan end prefix failed");
168
169    scan.setRowPrefixFilter(null);
170    verifyScanResult(table, scan, expected0, "Scan after prefix reset failed");
171
172    scan = new Scan();
173    scan.setFilter(new ColumnPrefixFilter(prefix4));
174    verifyScanResult(table, scan, expected4, "Double check on column prefix failed");
175
176    // ========
177    // COMBINED
178    // Prefix + Filter
179    scan = new Scan();
180    scan.setRowPrefixFilter(prefix1);
181    verifyScanResult(table, scan, expected1, "Prefix filter failed");
182
183    scan.setFilter(new ColumnPrefixFilter(prefix2));
184    verifyScanResult(table, scan, expected2, "Combined Prefix + Filter failed");
185
186    scan.setRowPrefixFilter(null);
187    verifyScanResult(table, scan, expected2, "Combined Prefix + Filter; removing Prefix failed");
188
189    scan.setFilter(null);
190    verifyScanResult(table, scan, expected0, "Scan after Filter reset failed");
191
192    // ========
193    // Reversed: Filter + Prefix
194    scan = new Scan();
195    scan.setFilter(new ColumnPrefixFilter(prefix2));
196    verifyScanResult(table, scan, expected2, "Test filter failed");
197
198    scan.setRowPrefixFilter(prefix1);
199    verifyScanResult(table, scan, expected2, "Combined Filter + Prefix failed");
200
201    scan.setFilter(null);
202    verifyScanResult(table, scan, expected1, "Combined Filter + Prefix ; removing Filter failed");
203
204    scan.setRowPrefixFilter(null);
205    verifyScanResult(table, scan, expected0, "Scan after prefix reset failed");
206  }
207
208  private void verifyScanResult(Table table, Scan scan, List<byte[]> expectedKeys, String message) {
209    List<byte[]> actualKeys = new ArrayList<>();
210    try {
211      ResultScanner scanner = table.getScanner(scan);
212      for (Result result : scanner) {
213        actualKeys.add(result.getRow());
214      }
215
216      String fullMessage = message;
217      if (LOG.isDebugEnabled()) {
218        fullMessage = message + "\n" + tableOfTwoListsOfByteArrays(
219                "Expected", expectedKeys,
220                "Actual  ", actualKeys);
221      }
222
223      Assert.assertArrayEquals(
224              fullMessage,
225              expectedKeys.toArray(),
226              actualKeys.toArray());
227    } catch (IOException e) {
228      e.printStackTrace();
229      Assert.fail();
230    }
231  }
232
233  private String printMultiple(char letter, int count) {
234    StringBuilder sb = new StringBuilder(count);
235    for (int i = 0; i < count; i++) {
236      sb.append(letter);
237    }
238    return sb.toString();
239  }
240
241  private String tableOfTwoListsOfByteArrays(
242          String label1, List<byte[]> listOfBytes1,
243          String label2, List<byte[]> listOfBytes2) {
244    int margin1 = calculateWidth(label1, listOfBytes1);
245    int margin2 = calculateWidth(label2, listOfBytes2);
246
247    StringBuilder sb = new StringBuilder(512);
248    String separator = '+' + printMultiple('-', margin1 + margin2 + 5) + '+' + '\n';
249    sb.append(separator);
250    sb.append(printLine(label1, margin1, label2, margin2)).append('\n');
251    sb.append(separator);
252    int maxLength = Math.max(listOfBytes1.size(), listOfBytes2.size());
253    for (int offset = 0; offset < maxLength; offset++) {
254      String value1 = getStringFromList(listOfBytes1, offset);
255      String value2 = getStringFromList(listOfBytes2, offset);
256      sb.append(printLine(value1, margin1, value2, margin2)).append('\n');
257    }
258    sb.append(separator).append('\n');
259    return sb.toString();
260  }
261
262  private String printLine(String leftValue, int leftWidth1, String rightValue, int rightWidth) {
263    return "| " +
264           leftValue  + printMultiple(' ', leftWidth1 - leftValue.length() ) +
265           " | " +
266           rightValue + printMultiple(' ', rightWidth - rightValue.length()) +
267           " |";
268  }
269
270  private int calculateWidth(String label1, List<byte[]> listOfBytes1) {
271    int longestList1 = label1.length();
272    for (byte[] value : listOfBytes1) {
273      longestList1 = Math.max(value.length * 2, longestList1);
274    }
275    return longestList1 + 5;
276  }
277
278  private String getStringFromList(List<byte[]> listOfBytes, int offset) {
279    String value1;
280    if (listOfBytes.size() > offset) {
281      value1 = Hex.encodeHexString(listOfBytes.get(offset));
282    } else {
283      value1 = "<missing>";
284    }
285    return value1;
286  }
287
288}