001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.rest; 019 020import static org.junit.Assert.assertEquals; 021import static org.junit.Assert.assertNotNull; 022import static org.junit.Assert.assertTrue; 023 024import java.io.ByteArrayInputStream; 025import java.io.StringWriter; 026import java.util.ArrayList; 027import java.util.Arrays; 028import java.util.Iterator; 029import java.util.List; 030import javax.xml.bind.JAXBContext; 031import javax.xml.bind.Marshaller; 032import javax.xml.bind.Unmarshaller; 033import org.apache.hadoop.hbase.CellUtil; 034import org.apache.hadoop.hbase.CompareOperator; 035import org.apache.hadoop.hbase.HBaseClassTestRule; 036import org.apache.hadoop.hbase.HBaseTestingUtil; 037import org.apache.hadoop.hbase.HConstants; 038import org.apache.hadoop.hbase.KeyValue; 039import org.apache.hadoop.hbase.TableName; 040import org.apache.hadoop.hbase.client.Admin; 041import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; 042import org.apache.hadoop.hbase.client.Delete; 043import org.apache.hadoop.hbase.client.Durability; 044import org.apache.hadoop.hbase.client.Put; 045import org.apache.hadoop.hbase.client.Scan; 046import org.apache.hadoop.hbase.client.Table; 047import org.apache.hadoop.hbase.client.TableDescriptor; 048import org.apache.hadoop.hbase.client.TableDescriptorBuilder; 049import org.apache.hadoop.hbase.filter.BinaryComparator; 050import org.apache.hadoop.hbase.filter.Filter; 051import org.apache.hadoop.hbase.filter.FilterList; 052import org.apache.hadoop.hbase.filter.FilterList.Operator; 053import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; 054import org.apache.hadoop.hbase.filter.FuzzyRowFilter; 055import org.apache.hadoop.hbase.filter.InclusiveStopFilter; 056import org.apache.hadoop.hbase.filter.MultiRowRangeFilter; 057import org.apache.hadoop.hbase.filter.PageFilter; 058import org.apache.hadoop.hbase.filter.PrefixFilter; 059import org.apache.hadoop.hbase.filter.QualifierFilter; 060import org.apache.hadoop.hbase.filter.RegexStringComparator; 061import org.apache.hadoop.hbase.filter.RowFilter; 062import org.apache.hadoop.hbase.filter.SkipFilter; 063import org.apache.hadoop.hbase.filter.SubstringComparator; 064import org.apache.hadoop.hbase.filter.ValueFilter; 065import org.apache.hadoop.hbase.rest.client.Client; 066import org.apache.hadoop.hbase.rest.client.Cluster; 067import org.apache.hadoop.hbase.rest.client.Response; 068import org.apache.hadoop.hbase.rest.model.CellModel; 069import org.apache.hadoop.hbase.rest.model.CellSetModel; 070import org.apache.hadoop.hbase.rest.model.RowModel; 071import org.apache.hadoop.hbase.rest.model.ScannerModel; 072import org.apache.hadoop.hbase.testclassification.MediumTests; 073import org.apache.hadoop.hbase.testclassification.RestTests; 074import org.apache.hadoop.hbase.util.Bytes; 075import org.apache.hadoop.hbase.util.Pair; 076import org.junit.AfterClass; 077import org.junit.BeforeClass; 078import org.junit.ClassRule; 079import org.junit.Test; 080import org.junit.experimental.categories.Category; 081import org.slf4j.Logger; 082import org.slf4j.LoggerFactory; 083 084@Category({ RestTests.class, MediumTests.class }) 085public class TestScannersWithFilters { 086 @ClassRule 087 public static final HBaseClassTestRule CLASS_RULE = 088 HBaseClassTestRule.forClass(TestScannersWithFilters.class); 089 090 private static final Logger LOG = LoggerFactory.getLogger(TestScannersWithFilters.class); 091 092 private static final TableName TABLE = TableName.valueOf("TestScannersWithFilters"); 093 094 private static final byte[][] ROWS_ONE = { Bytes.toBytes("testRowOne-0"), 095 Bytes.toBytes("testRowOne-1"), Bytes.toBytes("testRowOne-2"), Bytes.toBytes("testRowOne-3") }; 096 097 private static final byte[][] ROWS_TWO = { Bytes.toBytes("testRowTwo-0"), 098 Bytes.toBytes("testRowTwo-1"), Bytes.toBytes("testRowTwo-2"), Bytes.toBytes("testRowTwo-3") }; 099 100 private static final byte[][] FAMILIES = 101 { Bytes.toBytes("testFamilyOne"), Bytes.toBytes("testFamilyTwo") }; 102 103 private static final byte[][] QUALIFIERS_ONE = 104 { Bytes.toBytes("testQualifierOne-0"), Bytes.toBytes("testQualifierOne-1"), 105 Bytes.toBytes("testQualifierOne-2"), Bytes.toBytes("testQualifierOne-3") }; 106 107 private static final byte[][] QUALIFIERS_TWO = 108 { Bytes.toBytes("testQualifierTwo-0"), Bytes.toBytes("testQualifierTwo-1"), 109 Bytes.toBytes("testQualifierTwo-2"), Bytes.toBytes("testQualifierTwo-3") }; 110 111 private static final byte[][] VALUES = 112 { Bytes.toBytes("testValueOne"), Bytes.toBytes("testValueTwo") }; 113 114 private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); 115 private static final HBaseRESTTestingUtility REST_TEST_UTIL = new HBaseRESTTestingUtility(); 116 private static Client client; 117 private static JAXBContext context; 118 private static Marshaller marshaller; 119 private static Unmarshaller unmarshaller; 120 private static long numRows = (long) ROWS_ONE.length + ROWS_TWO.length; 121 private static long colsPerRow = (long) FAMILIES.length * QUALIFIERS_ONE.length; 122 123 @BeforeClass 124 public static void setUpBeforeClass() throws Exception { 125 TEST_UTIL.startMiniCluster(3); 126 REST_TEST_UTIL.startServletContainer(TEST_UTIL.getConfiguration()); 127 context = JAXBContext.newInstance(CellModel.class, CellSetModel.class, RowModel.class, 128 ScannerModel.class); 129 marshaller = context.createMarshaller(); 130 unmarshaller = context.createUnmarshaller(); 131 client = new Client(new Cluster().add("localhost", REST_TEST_UTIL.getServletPort())); 132 Admin admin = TEST_UTIL.getAdmin(); 133 if (!admin.tableExists(TABLE)) { 134 TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(TABLE) 135 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILIES[0])) 136 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILIES[1])).build(); 137 admin.createTable(tableDescriptor); 138 Table table = TEST_UTIL.getConnection().getTable(TABLE); 139 // Insert first half 140 for (byte[] ROW : ROWS_ONE) { 141 Put p = new Put(ROW); 142 p.setDurability(Durability.SKIP_WAL); 143 for (byte[] QUALIFIER : QUALIFIERS_ONE) { 144 p.addColumn(FAMILIES[0], QUALIFIER, VALUES[0]); 145 } 146 table.put(p); 147 } 148 for (byte[] ROW : ROWS_TWO) { 149 Put p = new Put(ROW); 150 p.setDurability(Durability.SKIP_WAL); 151 for (byte[] QUALIFIER : QUALIFIERS_TWO) { 152 p.addColumn(FAMILIES[1], QUALIFIER, VALUES[1]); 153 } 154 table.put(p); 155 } 156 157 // Insert second half (reverse families) 158 for (byte[] ROW : ROWS_ONE) { 159 Put p = new Put(ROW); 160 p.setDurability(Durability.SKIP_WAL); 161 for (byte[] QUALIFIER : QUALIFIERS_ONE) { 162 p.addColumn(FAMILIES[1], QUALIFIER, VALUES[0]); 163 } 164 table.put(p); 165 } 166 for (byte[] ROW : ROWS_TWO) { 167 Put p = new Put(ROW); 168 p.setDurability(Durability.SKIP_WAL); 169 for (byte[] QUALIFIER : QUALIFIERS_TWO) { 170 p.addColumn(FAMILIES[0], QUALIFIER, VALUES[1]); 171 } 172 table.put(p); 173 } 174 175 // Delete the second qualifier from all rows and families 176 for (byte[] ROW : ROWS_ONE) { 177 Delete d = new Delete(ROW); 178 d.addColumns(FAMILIES[0], QUALIFIERS_ONE[1]); 179 d.addColumns(FAMILIES[1], QUALIFIERS_ONE[1]); 180 table.delete(d); 181 } 182 for (byte[] ROW : ROWS_TWO) { 183 Delete d = new Delete(ROW); 184 d.addColumns(FAMILIES[0], QUALIFIERS_TWO[1]); 185 d.addColumns(FAMILIES[1], QUALIFIERS_TWO[1]); 186 table.delete(d); 187 } 188 colsPerRow -= 2; 189 190 // Delete the second rows from both groups, one column at a time 191 for (byte[] QUALIFIER : QUALIFIERS_ONE) { 192 Delete d = new Delete(ROWS_ONE[1]); 193 d.addColumns(FAMILIES[0], QUALIFIER); 194 d.addColumns(FAMILIES[1], QUALIFIER); 195 table.delete(d); 196 } 197 for (byte[] QUALIFIER : QUALIFIERS_TWO) { 198 Delete d = new Delete(ROWS_TWO[1]); 199 d.addColumns(FAMILIES[0], QUALIFIER); 200 d.addColumns(FAMILIES[1], QUALIFIER); 201 table.delete(d); 202 } 203 numRows -= 2; 204 table.close(); 205 } 206 } 207 208 @AfterClass 209 public static void tearDownAfterClass() throws Exception { 210 REST_TEST_UTIL.shutdownServletContainer(); 211 TEST_UTIL.shutdownMiniCluster(); 212 } 213 214 private static void verifyScan(Scan s, long expectedRows, long expectedKeys) throws Exception { 215 ScannerModel model = ScannerModel.fromScan(s); 216 model.setBatch(Integer.MAX_VALUE); // fetch it all at once 217 StringWriter writer = new StringWriter(); 218 marshaller.marshal(model, writer); 219 LOG.debug(writer.toString()); 220 byte[] body = Bytes.toBytes(writer.toString()); 221 Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body); 222 assertEquals(201, response.getCode()); 223 String scannerURI = response.getLocation(); 224 assertNotNull(scannerURI); 225 226 // get a cell set 227 response = client.get(scannerURI, Constants.MIMETYPE_XML); 228 assertEquals(200, response.getCode()); 229 assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); 230 CellSetModel cells = 231 (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody())); 232 233 int rows = cells.getRows().size(); 234 assertEquals( 235 "Unexpected number of rows! Expected " + expectedRows + " total but scanned " + rows, 236 expectedRows, rows); 237 for (RowModel row : cells.getRows()) { 238 int count = row.getCells().size(); 239 assertEquals("Expected " + expectedKeys + " keys per row but " + "returned " + count, 240 expectedKeys, count); 241 } 242 243 // delete the scanner 244 response = client.delete(scannerURI); 245 assertEquals(200, response.getCode()); 246 } 247 248 private static void verifyScanFull(Scan s, KeyValue[] kvs) throws Exception { 249 ScannerModel model = ScannerModel.fromScan(s); 250 model.setBatch(Integer.MAX_VALUE); // fetch it all at once 251 StringWriter writer = new StringWriter(); 252 marshaller.marshal(model, writer); 253 LOG.debug(writer.toString()); 254 byte[] body = Bytes.toBytes(writer.toString()); 255 Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body); 256 assertEquals(201, response.getCode()); 257 String scannerURI = response.getLocation(); 258 assertNotNull(scannerURI); 259 260 // get a cell set 261 response = client.get(scannerURI, Constants.MIMETYPE_XML); 262 assertEquals(200, response.getCode()); 263 assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); 264 CellSetModel cellSet = 265 (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody())); 266 267 // delete the scanner 268 response = client.delete(scannerURI); 269 assertEquals(200, response.getCode()); 270 271 int row = 0; 272 int idx = 0; 273 Iterator<RowModel> i = cellSet.getRows().iterator(); 274 for (boolean done = true; done; row++) { 275 done = i.hasNext(); 276 if (!done) { 277 break; 278 } 279 280 RowModel rowModel = i.next(); 281 List<CellModel> cells = rowModel.getCells(); 282 if (cells.isEmpty()) { 283 break; 284 } 285 286 assertTrue("Scanned too many keys! Only expected " + kvs.length 287 + " total but already scanned " + (cells.size() + idx), kvs.length >= idx + cells.size()); 288 for (CellModel cell : cells) { 289 assertTrue("Row mismatch", Bytes.equals(rowModel.getKey(), CellUtil.cloneRow(kvs[idx]))); 290 byte[][] split = CellUtil.parseColumn(cell.getColumn()); 291 assertTrue("Family mismatch", Bytes.equals(split[0], CellUtil.cloneFamily(kvs[idx]))); 292 assertTrue("Qualifier mismatch", Bytes.equals(split[1], CellUtil.cloneQualifier(kvs[idx]))); 293 assertTrue("Value mismatch", Bytes.equals(cell.getValue(), CellUtil.cloneValue(kvs[idx]))); 294 idx++; 295 } 296 } 297 assertEquals("Expected " + kvs.length + " total keys but scanned " + idx, kvs.length, idx); 298 } 299 300 private static void verifyScanNoEarlyOut(Scan s, long expectedRows, long expectedKeys) 301 throws Exception { 302 ScannerModel model = ScannerModel.fromScan(s); 303 model.setBatch(Integer.MAX_VALUE); // fetch it all at once 304 StringWriter writer = new StringWriter(); 305 marshaller.marshal(model, writer); 306 LOG.debug(writer.toString()); 307 byte[] body = Bytes.toBytes(writer.toString()); 308 Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body); 309 assertEquals(201, response.getCode()); 310 String scannerURI = response.getLocation(); 311 assertNotNull(scannerURI); 312 313 // get a cell set 314 response = client.get(scannerURI, Constants.MIMETYPE_XML); 315 assertEquals(200, response.getCode()); 316 assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); 317 CellSetModel cellSet = 318 (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody())); 319 320 // delete the scanner 321 response = client.delete(scannerURI); 322 assertEquals(200, response.getCode()); 323 324 Iterator<RowModel> i = cellSet.getRows().iterator(); 325 int j = 0; 326 for (boolean done = true; done; j++) { 327 done = i.hasNext(); 328 if (!done) { 329 break; 330 } 331 332 RowModel rowModel = i.next(); 333 List<CellModel> cells = rowModel.getCells(); 334 if (cells.isEmpty()) { 335 break; 336 } 337 338 assertTrue("Scanned too many rows! Only expected " + expectedRows 339 + " total but already scanned " + (j + 1), expectedRows > j); 340 assertEquals("Expected " + expectedKeys + " keys per row but " + "returned " + cells.size(), 341 expectedKeys, cells.size()); 342 } 343 assertEquals("Expected " + expectedRows + " rows but scanned " + j + " rows", expectedRows, j); 344 } 345 346 @Test 347 public void testNoFilter() throws Exception { 348 // No filter 349 long expectedRows = numRows; 350 long expectedKeys = colsPerRow; 351 352 // Both families 353 Scan s = new Scan(); 354 verifyScan(s, expectedRows, expectedKeys); 355 356 // One family 357 s = new Scan(); 358 s.addFamily(FAMILIES[0]); 359 verifyScan(s, expectedRows, expectedKeys / 2); 360 } 361 362 @Test 363 public void testPrefixFilter() throws Exception { 364 // Grab rows from group one (half of total) 365 long expectedRows = numRows / 2; 366 long expectedKeys = colsPerRow; 367 Scan s = new Scan(); 368 s.setFilter(new PrefixFilter(Bytes.toBytes("testRowOne"))); 369 verifyScan(s, expectedRows, expectedKeys); 370 } 371 372 @Test 373 public void testPageFilter() throws Exception { 374 // KVs in first 6 rows 375 KeyValue[] expectedKVs = { 376 // testRowOne-0 377 new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 378 new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), 379 new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), 380 new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), 381 new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), 382 new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), 383 // testRowOne-2 384 new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 385 new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), 386 new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), 387 new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), 388 new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), 389 new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), 390 // testRowOne-3 391 new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 392 new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), 393 new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), 394 new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), 395 new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), 396 new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), 397 // testRowTwo-0 398 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 399 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 400 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 401 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 402 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 403 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), 404 // testRowTwo-2 405 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 406 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 407 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 408 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 409 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 410 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), 411 // testRowTwo-3 412 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 413 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 414 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 415 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 416 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 417 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]) }; 418 419 // Grab all 6 rows 420 long expectedRows = 6; 421 long expectedKeys = colsPerRow; 422 Scan s = new Scan(); 423 s.setFilter(new PageFilter(expectedRows)); 424 verifyScan(s, expectedRows, expectedKeys); 425 s.setFilter(new PageFilter(expectedRows)); 426 verifyScanFull(s, expectedKVs); 427 428 // Grab first 4 rows (6 cols per row) 429 expectedRows = 4; 430 expectedKeys = colsPerRow; 431 s = new Scan(); 432 s.setFilter(new PageFilter(expectedRows)); 433 verifyScan(s, expectedRows, expectedKeys); 434 s.setFilter(new PageFilter(expectedRows)); 435 verifyScanFull(s, Arrays.copyOf(expectedKVs, 24)); 436 437 // Grab first 2 rows 438 expectedRows = 2; 439 expectedKeys = colsPerRow; 440 s = new Scan(); 441 s.setFilter(new PageFilter(expectedRows)); 442 verifyScan(s, expectedRows, expectedKeys); 443 s.setFilter(new PageFilter(expectedRows)); 444 verifyScanFull(s, Arrays.copyOf(expectedKVs, 12)); 445 446 // Grab first row 447 expectedRows = 1; 448 expectedKeys = colsPerRow; 449 s = new Scan(); 450 s.setFilter(new PageFilter(expectedRows)); 451 verifyScan(s, expectedRows, expectedKeys); 452 s.setFilter(new PageFilter(expectedRows)); 453 verifyScanFull(s, Arrays.copyOf(expectedKVs, 6)); 454 } 455 456 @Test 457 public void testInclusiveStopFilter() throws Exception { 458 // Grab rows from group one 459 460 // If we just use start/stop row, we get total/2 - 1 rows 461 long expectedRows = (numRows / 2) - 1; 462 long expectedKeys = colsPerRow; 463 Scan s = new Scan().withStartRow(Bytes.toBytes("testRowOne-0")) 464 .withStopRow(Bytes.toBytes("testRowOne-3")); 465 verifyScan(s, expectedRows, expectedKeys); 466 467 // Now use start row with inclusive stop filter 468 expectedRows = numRows / 2; 469 s = new Scan().withStartRow(Bytes.toBytes("testRowOne-0")); 470 s.setFilter(new InclusiveStopFilter(Bytes.toBytes("testRowOne-3"))); 471 verifyScan(s, expectedRows, expectedKeys); 472 473 // Grab rows from group two 474 475 // If we just use start/stop row, we get total/2 - 1 rows 476 expectedRows = (numRows / 2) - 1; 477 expectedKeys = colsPerRow; 478 s = new Scan().withStartRow(Bytes.toBytes("testRowTwo-0")) 479 .withStopRow(Bytes.toBytes("testRowTwo-3")); 480 verifyScan(s, expectedRows, expectedKeys); 481 482 // Now use start row with inclusive stop filter 483 expectedRows = numRows / 2; 484 s = new Scan().withStartRow(Bytes.toBytes("testRowTwo-0")); 485 s.setFilter(new InclusiveStopFilter(Bytes.toBytes("testRowTwo-3"))); 486 verifyScan(s, expectedRows, expectedKeys); 487 } 488 489 @Test 490 public void testQualifierFilter() throws Exception { 491 // Match two keys (one from each family) in half the rows 492 long expectedRows = numRows / 2; 493 long expectedKeys = 2; 494 Filter f = new QualifierFilter(CompareOperator.EQUAL, 495 new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); 496 Scan s = new Scan(); 497 s.setFilter(f); 498 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 499 500 // Match keys less than same qualifier 501 // Expect only two keys (one from each family) in half the rows 502 expectedRows = numRows / 2; 503 expectedKeys = 2; 504 f = new QualifierFilter(CompareOperator.LESS, 505 new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); 506 s = new Scan(); 507 s.setFilter(f); 508 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 509 510 // Match keys less than or equal. Expect four keys (two from each family) in half the rows 511 expectedRows = numRows / 2; 512 expectedKeys = 4; 513 f = new QualifierFilter(CompareOperator.LESS_OR_EQUAL, 514 new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); 515 s = new Scan(); 516 s.setFilter(f); 517 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 518 519 // Match keys not equal. Expect four keys (two from each family) 520 // Only look in first group of rows 521 expectedRows = numRows / 2; 522 expectedKeys = 4; 523 f = new QualifierFilter(CompareOperator.NOT_EQUAL, 524 new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); 525 s = 526 new Scan().withStartRow(HConstants.EMPTY_START_ROW).withStopRow(Bytes.toBytes("testRowTwo")); 527 s.setFilter(f); 528 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 529 530 // Match keys greater or equal. Expect four keys (two from each family) 531 // Only look in first group of rows 532 expectedRows = numRows / 2; 533 expectedKeys = 4; 534 f = new QualifierFilter(CompareOperator.GREATER_OR_EQUAL, 535 new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); 536 s = 537 new Scan().withStartRow(HConstants.EMPTY_START_ROW).withStopRow(Bytes.toBytes("testRowTwo")); 538 s.setFilter(f); 539 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 540 541 // Match keys greater. Expect two keys (one from each family) 542 // Only look in first group of rows 543 expectedRows = numRows / 2; 544 expectedKeys = 2; 545 f = new QualifierFilter(CompareOperator.GREATER, 546 new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))); 547 s = 548 new Scan().withStartRow(HConstants.EMPTY_START_ROW).withStopRow(Bytes.toBytes("testRowTwo")); 549 s.setFilter(f); 550 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 551 552 // Match keys not equal to. Look across rows and fully validate the keys and ordering 553 // Expect varied numbers of keys, 4 per row in group one, 6 per row in group two 554 f = new QualifierFilter(CompareOperator.NOT_EQUAL, new BinaryComparator(QUALIFIERS_ONE[2])); 555 s = new Scan(); 556 s.setFilter(f); 557 558 KeyValue[] kvs = { 559 // testRowOne-0 560 new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 561 new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), 562 new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), 563 new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), 564 // testRowOne-2 565 new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 566 new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), 567 new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), 568 new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), 569 // testRowOne-3 570 new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 571 new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), 572 new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), 573 new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), 574 // testRowTwo-0 575 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 576 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 577 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 578 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 579 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 580 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), 581 // testRowTwo-2 582 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 583 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 584 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 585 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 586 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 587 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), 588 // testRowTwo-3 589 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 590 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 591 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 592 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 593 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 594 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), }; 595 verifyScanFull(s, kvs); 596 597 // Test across rows and groups with a regex. Filter out "test*-2" 598 // Expect 4 keys per row across both groups 599 f = new QualifierFilter(CompareOperator.NOT_EQUAL, new RegexStringComparator("test.+-2")); 600 s = new Scan(); 601 s.setFilter(f); 602 603 kvs = new KeyValue[] { 604 // testRowOne-0 605 new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 606 new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), 607 new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), 608 new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), 609 // testRowOne-2 610 new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 611 new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), 612 new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), 613 new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), 614 // testRowOne-3 615 new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 616 new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), 617 new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), 618 new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), 619 // testRowTwo-0 620 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 621 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 622 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 623 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), 624 // testRowTwo-2 625 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 626 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 627 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 628 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), 629 // testRowTwo-3 630 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 631 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 632 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 633 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), }; 634 verifyScanFull(s, kvs); 635 } 636 637 @Test 638 public void testRowFilter() throws Exception { 639 // Match a single row, all keys 640 long expectedRows = 1; 641 long expectedKeys = colsPerRow; 642 Filter f = 643 new RowFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("testRowOne-2"))); 644 Scan s = new Scan(); 645 s.setFilter(f); 646 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 647 648 // Match a two rows, one from each group, using regex 649 expectedRows = 2; 650 expectedKeys = colsPerRow; 651 f = new RowFilter(CompareOperator.EQUAL, new RegexStringComparator("testRow.+-2")); 652 s = new Scan(); 653 s.setFilter(f); 654 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 655 656 // Match rows less than 657 // Expect all keys in one row 658 expectedRows = 1; 659 expectedKeys = colsPerRow; 660 f = new RowFilter(CompareOperator.LESS, new BinaryComparator(Bytes.toBytes("testRowOne-2"))); 661 s = new Scan(); 662 s.setFilter(f); 663 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 664 665 // Match rows less than or equal 666 // Expect all keys in two rows 667 expectedRows = 2; 668 expectedKeys = colsPerRow; 669 f = new RowFilter(CompareOperator.LESS_OR_EQUAL, 670 new BinaryComparator(Bytes.toBytes("testRowOne-2"))); 671 s = new Scan(); 672 s.setFilter(f); 673 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 674 675 // Match rows not equal 676 // Expect all keys in all but one row 677 expectedRows = numRows - 1; 678 expectedKeys = colsPerRow; 679 f = 680 new RowFilter(CompareOperator.NOT_EQUAL, new BinaryComparator(Bytes.toBytes("testRowOne-2"))); 681 s = new Scan(); 682 s.setFilter(f); 683 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 684 685 // Match keys greater or equal 686 // Expect all keys in all but one row 687 expectedRows = numRows - 1; 688 expectedKeys = colsPerRow; 689 f = new RowFilter(CompareOperator.GREATER_OR_EQUAL, 690 new BinaryComparator(Bytes.toBytes("testRowOne-2"))); 691 s = new Scan(); 692 s.setFilter(f); 693 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 694 695 // Match keys greater 696 // Expect all keys in all but two rows 697 expectedRows = numRows - 2; 698 expectedKeys = colsPerRow; 699 f = new RowFilter(CompareOperator.GREATER, new BinaryComparator(Bytes.toBytes("testRowOne-2"))); 700 s = new Scan(); 701 s.setFilter(f); 702 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 703 704 // Match rows not equal to testRowTwo-2 705 // Look across rows and fully validate the keys and ordering 706 // Should see all keys in all rows but testRowTwo-2 707 f = 708 new RowFilter(CompareOperator.NOT_EQUAL, new BinaryComparator(Bytes.toBytes("testRowOne-2"))); 709 s = new Scan(); 710 s.setFilter(f); 711 712 KeyValue[] kvs = { 713 // testRowOne-0 714 new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 715 new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), 716 new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), 717 new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), 718 new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), 719 new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), 720 // testRowOne-3 721 new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 722 new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), 723 new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), 724 new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), 725 new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), 726 new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), 727 // testRowTwo-0 728 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 729 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 730 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 731 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 732 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 733 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), 734 // testRowTwo-2 735 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 736 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 737 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 738 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 739 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 740 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), 741 // testRowTwo-3 742 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 743 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 744 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 745 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 746 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 747 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), }; 748 verifyScanFull(s, kvs); 749 750 // Test across rows and groups with a regex 751 // Filter out everything that doesn't match "*-2" 752 // Expect all keys in two rows 753 f = new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(".+-2")); 754 s = new Scan(); 755 s.setFilter(f); 756 757 kvs = new KeyValue[] { 758 // testRowOne-2 759 new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 760 new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), 761 new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), 762 new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), 763 new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]), 764 new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]), 765 // testRowTwo-2 766 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 767 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 768 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 769 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 770 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 771 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]) }; 772 verifyScanFull(s, kvs); 773 } 774 775 @Test 776 public void testValueFilter() throws Exception { 777 // Match group one rows 778 long expectedRows = numRows / 2; 779 long expectedKeys = colsPerRow; 780 Filter f = 781 new ValueFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("testValueOne"))); 782 Scan s = new Scan(); 783 s.setFilter(f); 784 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 785 786 // Match group two rows 787 expectedRows = numRows / 2; 788 expectedKeys = colsPerRow; 789 f = new ValueFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("testValueTwo"))); 790 s = new Scan(); 791 s.setFilter(f); 792 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 793 794 // Match all values using regex 795 expectedRows = numRows; 796 expectedKeys = colsPerRow; 797 f = new ValueFilter(CompareOperator.EQUAL, new RegexStringComparator("testValue((One)|(Two))")); 798 s = new Scan(); 799 s.setFilter(f); 800 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 801 802 // Match values less than 803 // Expect group one rows 804 expectedRows = numRows / 2; 805 expectedKeys = colsPerRow; 806 f = new ValueFilter(CompareOperator.LESS, new BinaryComparator(Bytes.toBytes("testValueTwo"))); 807 s = new Scan(); 808 s.setFilter(f); 809 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 810 811 // Match values less than or equal 812 // Expect all rows 813 expectedRows = numRows; 814 expectedKeys = colsPerRow; 815 f = new ValueFilter(CompareOperator.LESS_OR_EQUAL, 816 new BinaryComparator(Bytes.toBytes("testValueTwo"))); 817 s = new Scan(); 818 s.setFilter(f); 819 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 820 821 // Match values less than or equal 822 // Expect group one rows 823 expectedRows = numRows / 2; 824 expectedKeys = colsPerRow; 825 f = new ValueFilter(CompareOperator.LESS_OR_EQUAL, 826 new BinaryComparator(Bytes.toBytes("testValueOne"))); 827 s = new Scan(); 828 s.setFilter(f); 829 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 830 831 // Match values not equal 832 // Expect half the rows 833 expectedRows = numRows / 2; 834 expectedKeys = colsPerRow; 835 f = new ValueFilter(CompareOperator.NOT_EQUAL, 836 new BinaryComparator(Bytes.toBytes("testValueOne"))); 837 s = new Scan(); 838 s.setFilter(f); 839 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 840 841 // Match values greater or equal 842 // Expect all rows 843 expectedRows = numRows; 844 expectedKeys = colsPerRow; 845 f = new ValueFilter(CompareOperator.GREATER_OR_EQUAL, 846 new BinaryComparator(Bytes.toBytes("testValueOne"))); 847 s = new Scan(); 848 s.setFilter(f); 849 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 850 851 // Match values greater 852 // Expect half rows 853 expectedRows = numRows / 2; 854 expectedKeys = colsPerRow; 855 f = 856 new ValueFilter(CompareOperator.GREATER, new BinaryComparator(Bytes.toBytes("testValueOne"))); 857 s = new Scan(); 858 s.setFilter(f); 859 verifyScanNoEarlyOut(s, expectedRows, expectedKeys); 860 861 // Match values not equal to testValueOne 862 // Look across rows and fully validate the keys and ordering 863 // Should see all keys in all group two rows 864 f = new ValueFilter(CompareOperator.NOT_EQUAL, 865 new BinaryComparator(Bytes.toBytes("testValueOne"))); 866 s = new Scan(); 867 s.setFilter(f); 868 869 KeyValue[] kvs = { 870 // testRowTwo-0 871 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 872 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 873 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 874 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 875 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 876 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), 877 // testRowTwo-2 878 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 879 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 880 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 881 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 882 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 883 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), 884 // testRowTwo-3 885 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 886 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 887 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 888 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 889 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 890 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), }; 891 verifyScanFull(s, kvs); 892 } 893 894 @Test 895 public void testSkipFilter() throws Exception { 896 // Test for qualifier regex: "testQualifierOne-2" 897 // Should only get rows from second group, and all keys 898 Filter f = new SkipFilter(new QualifierFilter(CompareOperator.NOT_EQUAL, 899 new BinaryComparator(Bytes.toBytes("testQualifierOne-2")))); 900 Scan s = new Scan(); 901 s.setFilter(f); 902 903 KeyValue[] kvs = { 904 // testRowTwo-0 905 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 906 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 907 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 908 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 909 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 910 new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), 911 // testRowTwo-2 912 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 913 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 914 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 915 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 916 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 917 new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), 918 // testRowTwo-3 919 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 920 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), 921 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), 922 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), 923 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), 924 new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), }; 925 verifyScanFull(s, kvs); 926 } 927 928 @Test 929 public void testFilterList() throws Exception { 930 // Test getting a single row, single key using Row, Qualifier, and Value 931 // regular expression and substring filters 932 // Use must pass all 933 List<Filter> filters = new ArrayList<>(3); 934 filters.add(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(".+-2"))); 935 filters.add(new QualifierFilter(CompareOperator.EQUAL, new RegexStringComparator(".+-2"))); 936 filters.add(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("One"))); 937 Filter f = new FilterList(Operator.MUST_PASS_ALL, filters); 938 Scan s = new Scan(); 939 s.addFamily(FAMILIES[0]); 940 s.setFilter(f); 941 KeyValue[] kvs = { new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]) }; 942 verifyScanFull(s, kvs); 943 944 // Test getting everything with a MUST_PASS_ONE filter including row, qf, 945 // val, regular expression and substring filters 946 filters.clear(); 947 filters.add(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(".+Two.+"))); 948 filters.add(new QualifierFilter(CompareOperator.EQUAL, new RegexStringComparator(".+-2"))); 949 filters.add(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("One"))); 950 f = new FilterList(Operator.MUST_PASS_ONE, filters); 951 s = new Scan(); 952 s.setFilter(f); 953 verifyScanNoEarlyOut(s, numRows, colsPerRow); 954 } 955 956 @Test 957 public void testFirstKeyOnlyFilter() throws Exception { 958 Scan s = new Scan(); 959 s.setFilter(new FirstKeyOnlyFilter()); 960 // Expected KVs, the first KV from each of the remaining 6 rows 961 KeyValue[] kvs = { new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 962 new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 963 new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), 964 new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 965 new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), 966 new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]) }; 967 verifyScanFull(s, kvs); 968 } 969 970 @Test 971 public void testMultiRowRangeFilter() throws Exception { 972 long expectedRows = 2; 973 long expectedKeys = colsPerRow; 974 List<MultiRowRangeFilter.RowRange> ranges = new ArrayList<>(); 975 // Both return only the third element, as the second one is deleted during initialization. 976 ranges.add(new MultiRowRangeFilter.RowRange(ROWS_ONE[1], true, ROWS_ONE[2], true)); 977 ranges.add(new MultiRowRangeFilter.RowRange(ROWS_TWO[0], false, ROWS_TWO[3], false)); 978 979 Scan s = new Scan(); 980 s.setFilter(new MultiRowRangeFilter(ranges)); 981 verifyScan(s, expectedRows, expectedKeys); 982 } 983 984 @Test 985 public void testFuzzyRowFilter() throws Exception { 986 long expectedRows = 4; 987 long expectedKeys = colsPerRow; 988 List<Pair<byte[], byte[]>> fuzzyKeys = new ArrayList<>(); 989 990 // Exact match for ROWS_ONE[0] (one row) 991 byte[] rowOneMask = new byte[ROWS_ONE[0].length]; 992 Arrays.fill(rowOneMask, (byte) 0); 993 fuzzyKeys.add(new Pair<>(ROWS_ONE[0], rowOneMask)); 994 // All ROW_TWO keys (three rows) 995 byte[] rowTwoMask = new byte[ROWS_TWO[0].length]; 996 Arrays.fill(rowTwoMask, (byte) 0); 997 rowTwoMask[rowTwoMask.length - 1] = (byte) 1; 998 fuzzyKeys.add(new Pair<>(ROWS_TWO[2], rowTwoMask)); 999 1000 Scan s = new Scan(); 1001 s.setFilter(new FuzzyRowFilter(fuzzyKeys)); 1002 verifyScan(s, expectedRows, expectedKeys); 1003 } 1004}