001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertFalse;
022import static org.junit.jupiter.api.Assertions.assertTrue;
023
024import java.io.IOException;
025import java.util.ArrayList;
026import java.util.List;
027import java.util.stream.Stream;
028import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate;
029import org.apache.hadoop.hbase.HBaseTestingUtil;
030import org.apache.hadoop.hbase.TableName;
031import org.apache.hadoop.hbase.testclassification.ClientTests;
032import org.apache.hadoop.hbase.testclassification.LargeTests;
033import org.apache.hadoop.hbase.util.Bytes;
034import org.junit.jupiter.api.AfterAll;
035import org.junit.jupiter.api.BeforeAll;
036import org.junit.jupiter.api.Tag;
037import org.junit.jupiter.api.TestTemplate;
038import org.junit.jupiter.params.provider.Arguments;
039
040/**
041 * Testcase for newly added feature in HBASE-17143, such as startRow and stopRow
042 * inclusive/exclusive, limit for rows, etc.
043 */
044@Tag(LargeTests.TAG)
045@Tag(ClientTests.TAG)
046@HBaseParameterizedTestTemplate(name = "{index}: batch={0}, smallResultSize={1}, allowPartial={2}")
047public class TestScannersFromClientSide2 {
048
049  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
050
051  private static TableName TABLE_NAME = TableName.valueOf("scan");
052
053  private static byte[] FAMILY = Bytes.toBytes("cf");
054
055  private static byte[] CQ1 = Bytes.toBytes("cq1");
056
057  private static byte[] CQ2 = Bytes.toBytes("cq2");
058
059  private final boolean batch;
060
061  private final boolean smallResultSize;
062
063  private final boolean allowPartial;
064
065  public static Stream<Arguments> parameters() {
066    List<Arguments> params = new ArrayList<>();
067    boolean[] values = new boolean[] { false, true };
068    for (int i = 0; i < 2; i++) {
069      for (int j = 0; j < 2; j++) {
070        for (int k = 0; k < 2; k++) {
071          params.add(Arguments.of(values[i], values[j], values[k]));
072        }
073      }
074    }
075    return params.stream();
076  }
077
078  public TestScannersFromClientSide2(boolean batch, boolean smallResultSize, boolean allowPartial) {
079    this.batch = batch;
080    this.smallResultSize = smallResultSize;
081    this.allowPartial = allowPartial;
082  }
083
084  @BeforeAll
085  public static void setUp() throws Exception {
086    TEST_UTIL.startMiniCluster(3);
087    byte[][] splitKeys = new byte[8][];
088    for (int i = 111; i < 999; i += 111) {
089      splitKeys[i / 111 - 1] = Bytes.toBytes(String.format("%03d", i));
090    }
091    Table table = TEST_UTIL.createTable(TABLE_NAME, FAMILY, splitKeys);
092    List<Put> puts = new ArrayList<>();
093    for (int i = 0; i < 1000; i++) {
094      puts.add(new Put(Bytes.toBytes(String.format("%03d", i)))
095        .addColumn(FAMILY, CQ1, Bytes.toBytes(i)).addColumn(FAMILY, CQ2, Bytes.toBytes(i * i)));
096    }
097    TEST_UTIL.waitTableAvailable(TABLE_NAME);
098    table.put(puts);
099  }
100
101  @AfterAll
102  public static void tearDown() throws Exception {
103    TEST_UTIL.shutdownMiniCluster();
104  }
105
106  private Scan createScan() {
107    Scan scan = new Scan();
108    if (batch) {
109      scan.setBatch(1);
110    }
111    if (smallResultSize) {
112      scan.setMaxResultSize(1);
113    }
114    if (allowPartial) {
115      scan.setAllowPartialResults(true);
116    }
117    return scan;
118  }
119
120  private void assertResultEquals(Result result, int i) {
121    assertEquals(String.format("%03d", i), Bytes.toString(result.getRow()));
122    assertEquals(i, Bytes.toInt(result.getValue(FAMILY, CQ1)));
123    assertEquals(i * i, Bytes.toInt(result.getValue(FAMILY, CQ2)));
124  }
125
126  private List<Result> doScan(Scan scan) throws IOException {
127    List<Result> results = new ArrayList<>();
128    try (Table table = TEST_UTIL.getConnection().getTable(TABLE_NAME);
129      ResultScanner scanner = table.getScanner(scan)) {
130      for (Result r; (r = scanner.next()) != null;) {
131        results.add(r);
132      }
133    }
134    return assertAndCreateCompleteResults(results);
135  }
136
137  private List<Result> assertAndCreateCompleteResults(List<Result> results) throws IOException {
138    if ((!batch && !allowPartial) || (allowPartial && !batch && !smallResultSize)) {
139      for (Result result : results) {
140        assertFalse(result.mayHaveMoreCellsInRow(), "Should not have partial result");
141      }
142      return results;
143    }
144    List<Result> completeResults = new ArrayList<>();
145    List<Result> partialResults = new ArrayList<>();
146    for (Result result : results) {
147      if (!result.mayHaveMoreCellsInRow()) {
148        assertFalse(partialResults.isEmpty(), "Should have partial result");
149        partialResults.add(result);
150        completeResults.add(Result.createCompleteResult(partialResults));
151        partialResults.clear();
152      } else {
153        partialResults.add(result);
154      }
155    }
156    assertTrue(partialResults.isEmpty(), "Should not have orphan partial result");
157    return completeResults;
158  }
159
160  private void testScan(int start, boolean startInclusive, int stop, boolean stopInclusive,
161    int limit) throws Exception {
162    Scan scan =
163      createScan().withStartRow(Bytes.toBytes(String.format("%03d", start)), startInclusive)
164        .withStopRow(Bytes.toBytes(String.format("%03d", stop)), stopInclusive);
165    if (limit > 0) {
166      scan.setLimit(limit);
167    }
168    List<Result> results = doScan(scan);
169    int actualStart = startInclusive ? start : start + 1;
170    int actualStop = stopInclusive ? stop + 1 : stop;
171    int count = actualStop - actualStart;
172    if (limit > 0) {
173      count = Math.min(count, limit);
174    }
175    assertEquals(count, results.size());
176    for (int i = 0; i < count; i++) {
177      assertResultEquals(results.get(i), actualStart + i);
178    }
179  }
180
181  private void testReversedScan(int start, boolean startInclusive, int stop, boolean stopInclusive,
182    int limit) throws Exception {
183    Scan scan =
184      createScan().withStartRow(Bytes.toBytes(String.format("%03d", start)), startInclusive)
185        .withStopRow(Bytes.toBytes(String.format("%03d", stop)), stopInclusive).setReversed(true);
186    if (limit > 0) {
187      scan.setLimit(limit);
188    }
189    List<Result> results = doScan(scan);
190    int actualStart = startInclusive ? start : start - 1;
191    int actualStop = stopInclusive ? stop - 1 : stop;
192    int count = actualStart - actualStop;
193    if (limit > 0) {
194      count = Math.min(count, limit);
195    }
196    assertEquals(count, results.size());
197    for (int i = 0; i < count; i++) {
198      assertResultEquals(results.get(i), actualStart - i);
199    }
200  }
201
202  @TestTemplate
203  public void testScanWithLimit() throws Exception {
204    testScan(1, true, 998, false, 900); // from first region to last region
205    testScan(123, true, 345, true, 100);
206    testScan(234, true, 456, false, 100);
207    testScan(345, false, 567, true, 100);
208    testScan(456, false, 678, false, 100);
209
210  }
211
212  @TestTemplate
213  public void testScanWithLimitGreaterThanActualCount() throws Exception {
214    testScan(1, true, 998, false, 1000); // from first region to last region
215    testScan(123, true, 345, true, 200);
216    testScan(234, true, 456, false, 200);
217    testScan(345, false, 567, true, 200);
218    testScan(456, false, 678, false, 200);
219  }
220
221  @TestTemplate
222  public void testReversedScanWithLimit() throws Exception {
223    testReversedScan(998, true, 1, false, 900); // from last region to first region
224    testReversedScan(543, true, 321, true, 100);
225    testReversedScan(654, true, 432, false, 100);
226    testReversedScan(765, false, 543, true, 100);
227    testReversedScan(876, false, 654, false, 100);
228  }
229
230  @TestTemplate
231  public void testReversedScanWithLimitGreaterThanActualCount() throws Exception {
232    testReversedScan(998, true, 1, false, 1000); // from last region to first region
233    testReversedScan(543, true, 321, true, 200);
234    testReversedScan(654, true, 432, false, 200);
235    testReversedScan(765, false, 543, true, 200);
236    testReversedScan(876, false, 654, false, 200);
237  }
238
239  @TestTemplate
240  public void testStartRowStopRowInclusive() throws Exception {
241    testScan(1, true, 998, false, -1); // from first region to last region
242    testScan(123, true, 345, true, -1);
243    testScan(234, true, 456, false, -1);
244    testScan(345, false, 567, true, -1);
245    testScan(456, false, 678, false, -1);
246  }
247
248  @TestTemplate
249  public void testReversedStartRowStopRowInclusive() throws Exception {
250    testReversedScan(998, true, 1, false, -1); // from last region to first region
251    testReversedScan(543, true, 321, true, -1);
252    testReversedScan(654, true, 432, false, -1);
253    testReversedScan(765, false, 543, true, -1);
254    testReversedScan(876, false, 654, false, -1);
255  }
256}