Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.rest;
019
020import static org.junit.jupiter.api.Assertions.assertEquals;
021import static org.junit.jupiter.api.Assertions.assertNotNull;
022import static org.junit.jupiter.api.Assertions.assertNull;
023import static org.junit.jupiter.api.Assertions.assertTrue;
024
025import java.io.ByteArrayInputStream;
026import java.io.IOException;
027import java.io.StringWriter;
028import java.util.ArrayList;
029import java.util.Iterator;
030import java.util.List;
031import java.util.Random;
032import java.util.concurrent.ThreadLocalRandom;
033import javax.xml.bind.JAXBContext;
034import javax.xml.bind.JAXBException;
035import javax.xml.bind.Marshaller;
036import javax.xml.bind.Unmarshaller;
037import org.apache.hadoop.conf.Configuration;
038import org.apache.hadoop.hbase.CellUtil;
039import org.apache.hadoop.hbase.HBaseTestingUtil;
040import org.apache.hadoop.hbase.TableName;
041import org.apache.hadoop.hbase.client.Admin;
042import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
043import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
044import org.apache.hadoop.hbase.client.Connection;
045import org.apache.hadoop.hbase.client.ConnectionFactory;
046import org.apache.hadoop.hbase.client.Durability;
047import org.apache.hadoop.hbase.client.Put;
048import org.apache.hadoop.hbase.client.Table;
049import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
050import org.apache.hadoop.hbase.rest.client.Client;
051import org.apache.hadoop.hbase.rest.client.Cluster;
052import org.apache.hadoop.hbase.rest.client.Response;
053import org.apache.hadoop.hbase.rest.model.CellModel;
054import org.apache.hadoop.hbase.rest.model.CellSetModel;
055import org.apache.hadoop.hbase.rest.model.RowModel;
056import org.apache.hadoop.hbase.rest.model.ScannerModel;
057import org.apache.hadoop.hbase.testclassification.MediumTests;
058import org.apache.hadoop.hbase.testclassification.RestTests;
059import org.apache.hadoop.hbase.util.Bytes;
060import org.apache.http.Header;
061import org.junit.jupiter.api.AfterAll;
062import org.junit.jupiter.api.BeforeAll;
063import org.junit.jupiter.api.Tag;
064import org.junit.jupiter.api.Test;
065
066@Tag(RestTests.TAG)
067@Tag(MediumTests.TAG)
068public class TestScannerResource {
069
070  private static final TableName TABLE = TableName.valueOf("TestScannerResource");
071  private static final TableName TABLE_TO_BE_DISABLED = TableName.valueOf("ScannerResourceDisable");
072  private static final String NONEXISTENT_TABLE = "ThisTableDoesNotExist";
073  private static final String CFA = "a";
074  private static final String CFB = "b";
075  private static final String COLUMN_1 = CFA + ":1";
076  private static final String COLUMN_2 = CFB + ":2";
077
078  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
079  private static final HBaseRESTTestingUtility REST_TEST_UTIL = new HBaseRESTTestingUtility();
080  private static Client client;
081  private static JAXBContext context;
082  private static Marshaller marshaller;
083  private static Unmarshaller unmarshaller;
084  private static int expectedRows1;
085  private static int expectedRows2;
086  private static Configuration conf;
087
088  static int insertData(Configuration conf, TableName tableName, String column, double prob)
089    throws IOException {
090    Random rng = ThreadLocalRandom.current();
091    byte[] k = new byte[3];
092    byte[][] famAndQf = CellUtil.parseColumn(Bytes.toBytes(column));
093    List<Put> puts = new ArrayList<>();
094    for (byte b1 = 'a'; b1 < 'z'; b1++) {
095      for (byte b2 = 'a'; b2 < 'z'; b2++) {
096        for (byte b3 = 'a'; b3 < 'z'; b3++) {
097          if (rng.nextDouble() < prob) {
098            k[0] = b1;
099            k[1] = b2;
100            k[2] = b3;
101            Put put = new Put(k);
102            put.setDurability(Durability.SKIP_WAL);
103            put.addColumn(famAndQf[0], famAndQf[1], k);
104            puts.add(put);
105          }
106        }
107      }
108    }
109    try (Connection conn = ConnectionFactory.createConnection(conf);
110      Table table = conn.getTable(tableName)) {
111      table.put(puts);
112    }
113    return puts.size();
114  }
115
116  static int countCellSet(CellSetModel model) {
117    int count = 0;
118    Iterator<RowModel> rows = model.getRows().iterator();
119    while (rows.hasNext()) {
120      RowModel row = rows.next();
121      Iterator<CellModel> cells = row.getCells().iterator();
122      while (cells.hasNext()) {
123        cells.next();
124        count++;
125      }
126    }
127    return count;
128  }
129
130  private static int fullTableScan(ScannerModel model) throws IOException {
131    model.setBatch(100);
132    Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
133      model.createProtobufOutput());
134    assertEquals(201, response.getCode());
135    String scannerURI = response.getLocation();
136    assertNotNull(scannerURI);
137    int count = 0;
138    while (true) {
139      response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
140      assertTrue(response.getCode() == 200 || response.getCode() == 204);
141      if (response.getCode() == 200) {
142        assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
143        CellSetModel cellSet = new CellSetModel();
144        cellSet.getObjectFromMessage(response.getBody());
145        Iterator<RowModel> rows = cellSet.getRows().iterator();
146        while (rows.hasNext()) {
147          RowModel row = rows.next();
148          Iterator<CellModel> cells = row.getCells().iterator();
149          while (cells.hasNext()) {
150            cells.next();
151            count++;
152          }
153        }
154      } else {
155        break;
156      }
157    }
158    // delete the scanner
159    response = client.delete(scannerURI);
160    assertEquals(200, response.getCode());
161    return count;
162  }
163
164  @BeforeAll
165  public static void setUpBeforeClass() throws Exception {
166    conf = TEST_UTIL.getConfiguration();
167    TEST_UTIL.startMiniCluster();
168    REST_TEST_UTIL.startServletContainer(conf);
169    client = new Client(new Cluster().add("localhost", REST_TEST_UTIL.getServletPort()));
170    context = JAXBContext.newInstance(CellModel.class, CellSetModel.class, RowModel.class,
171      ScannerModel.class);
172    marshaller = context.createMarshaller();
173    unmarshaller = context.createUnmarshaller();
174    Admin admin = TEST_UTIL.getAdmin();
175    if (admin.tableExists(TABLE)) {
176      return;
177    }
178    TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(TABLE);
179    ColumnFamilyDescriptor columnFamilyDescriptor =
180      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(CFA)).build();
181    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
182    columnFamilyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(CFB)).build();
183    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
184
185    admin.createTable(tableDescriptorBuilder.build());
186    expectedRows1 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_1, 1.0);
187    expectedRows2 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_2, 0.5);
188
189    tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(TABLE_TO_BE_DISABLED);
190    columnFamilyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(CFA)).build();
191    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
192    columnFamilyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(CFB)).build();
193    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
194
195    admin.createTable(tableDescriptorBuilder.build());
196  }
197
198  @AfterAll
199  public static void tearDownAfterClass() throws Exception {
200    REST_TEST_UTIL.shutdownServletContainer();
201    TEST_UTIL.shutdownMiniCluster();
202  }
203
204  @Test
205  public void testSimpleScannerXML() throws IOException, JAXBException {
206    final int BATCH_SIZE = 5;
207    // new scanner
208    ScannerModel model = new ScannerModel();
209    model.setBatch(BATCH_SIZE);
210    model.addColumn(Bytes.toBytes(COLUMN_1));
211    StringWriter writer = new StringWriter();
212    marshaller.marshal(model, writer);
213    byte[] body = Bytes.toBytes(writer.toString());
214
215    // test put operation is forbidden in read-only mode
216    conf.set("hbase.rest.readonly", "true");
217    Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
218    assertEquals(403, response.getCode());
219    String scannerURI = response.getLocation();
220    assertNull(scannerURI);
221
222    // recall previous put operation with read-only off
223    conf.set("hbase.rest.readonly", "false");
224    response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
225    assertEquals(201, response.getCode());
226    scannerURI = response.getLocation();
227    assertNotNull(scannerURI);
228
229    // get a cell set
230    response = client.get(scannerURI, Constants.MIMETYPE_XML);
231    assertEquals(200, response.getCode());
232    assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
233    CellSetModel cellSet =
234      (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
235    // confirm batch size conformance
236    assertEquals(BATCH_SIZE, countCellSet(cellSet));
237
238    // test delete scanner operation is forbidden in read-only mode
239    conf.set("hbase.rest.readonly", "true");
240    response = client.delete(scannerURI);
241    assertEquals(403, response.getCode());
242
243    // recall previous delete scanner operation with read-only off
244    conf.set("hbase.rest.readonly", "false");
245    response = client.delete(scannerURI);
246    assertEquals(200, response.getCode());
247  }
248
249  @Test
250  public void testSimpleScannerPB() throws IOException {
251    final int BATCH_SIZE = 10;
252    // new scanner
253    ScannerModel model = new ScannerModel();
254    model.setBatch(BATCH_SIZE);
255    model.addColumn(Bytes.toBytes(COLUMN_1));
256
257    // test put operation is forbidden in read-only mode
258    conf.set("hbase.rest.readonly", "true");
259    Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
260      model.createProtobufOutput());
261    assertEquals(403, response.getCode());
262    String scannerURI = response.getLocation();
263    assertNull(scannerURI);
264
265    // recall previous put operation with read-only off
266    conf.set("hbase.rest.readonly", "false");
267    response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
268      model.createProtobufOutput());
269    assertEquals(201, response.getCode());
270    scannerURI = response.getLocation();
271    assertNotNull(scannerURI);
272
273    // get a cell set
274    response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
275    assertEquals(200, response.getCode());
276    assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
277    CellSetModel cellSet = new CellSetModel();
278    cellSet.getObjectFromMessage(response.getBody());
279    // confirm batch size conformance
280    assertEquals(BATCH_SIZE, countCellSet(cellSet));
281
282    // test delete scanner operation is forbidden in read-only mode
283    conf.set("hbase.rest.readonly", "true");
284    response = client.delete(scannerURI);
285    assertEquals(403, response.getCode());
286
287    // recall previous delete scanner operation with read-only off
288    conf.set("hbase.rest.readonly", "false");
289    response = client.delete(scannerURI);
290    assertEquals(200, response.getCode());
291  }
292
293  @Test
294  public void testSimpleScannerBinary() throws IOException {
295    // new scanner
296    ScannerModel model = new ScannerModel();
297    model.setBatch(1);
298    model.addColumn(Bytes.toBytes(COLUMN_1));
299
300    // test put operation is forbidden in read-only mode
301    conf.set("hbase.rest.readonly", "true");
302    Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
303      model.createProtobufOutput());
304    assertEquals(403, response.getCode());
305    String scannerURI = response.getLocation();
306    assertNull(scannerURI);
307
308    // recall previous put operation with read-only off
309    conf.set("hbase.rest.readonly", "false");
310    response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
311      model.createProtobufOutput());
312    assertEquals(201, response.getCode());
313    scannerURI = response.getLocation();
314    assertNotNull(scannerURI);
315
316    // get a cell
317    response = client.get(scannerURI, Constants.MIMETYPE_BINARY);
318    assertEquals(200, response.getCode());
319    assertEquals(Constants.MIMETYPE_BINARY, response.getHeader("content-type"));
320    // verify that data was returned
321    assertTrue(response.getBody().length > 0);
322    // verify that the expected X-headers are present
323    boolean foundRowHeader = false, foundColumnHeader = false, foundTimestampHeader = false;
324    for (Header header : response.getHeaders()) {
325      if (header.getName().equals("X-Row")) {
326        foundRowHeader = true;
327      } else if (header.getName().equals("X-Column")) {
328        foundColumnHeader = true;
329      } else if (header.getName().equals("X-Timestamp")) {
330        foundTimestampHeader = true;
331      }
332    }
333    assertTrue(foundRowHeader);
334    assertTrue(foundColumnHeader);
335    assertTrue(foundTimestampHeader);
336
337    // test delete scanner operation is forbidden in read-only mode
338    conf.set("hbase.rest.readonly", "true");
339    response = client.delete(scannerURI);
340    assertEquals(403, response.getCode());
341
342    // recall previous delete scanner operation with read-only off
343    conf.set("hbase.rest.readonly", "false");
344    response = client.delete(scannerURI);
345    assertEquals(200, response.getCode());
346  }
347
348  @Test
349  public void testFullTableScan() throws IOException {
350    ScannerModel model = new ScannerModel();
351    model.addColumn(Bytes.toBytes(COLUMN_1));
352    assertEquals(expectedRows1, fullTableScan(model));
353
354    model = new ScannerModel();
355    model.addColumn(Bytes.toBytes(COLUMN_2));
356    assertEquals(expectedRows2, fullTableScan(model));
357  }
358
359  @Test
360  public void testTableDoesNotExist() throws IOException, JAXBException {
361    ScannerModel model = new ScannerModel();
362    StringWriter writer = new StringWriter();
363    marshaller.marshal(model, writer);
364    byte[] body = Bytes.toBytes(writer.toString());
365    Response response =
366      client.put("/" + NONEXISTENT_TABLE + "/scanner", Constants.MIMETYPE_XML, body);
367    String scannerURI = response.getLocation();
368    assertNotNull(scannerURI);
369    response = client.get(scannerURI, Constants.MIMETYPE_XML);
370    assertEquals(404, response.getCode());
371  }
372
373  @Test
374  public void testTableScanWithTableDisable() throws IOException {
375    TEST_UTIL.getAdmin().disableTable(TABLE_TO_BE_DISABLED);
376    ScannerModel model = new ScannerModel();
377    model.addColumn(Bytes.toBytes(COLUMN_1));
378    model.setCaching(1);
379    Response response = client.put("/" + TABLE_TO_BE_DISABLED + "/scanner",
380      Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
381    // we will see the exception when we actually want to get the result.
382    assertEquals(201, response.getCode());
383    String scannerURI = response.getLocation();
384    assertNotNull(scannerURI);
385    response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
386    assertEquals(410, response.getCode());
387  }
388
389  @Test
390  public void deleteNonExistent() throws IOException {
391    Response response = client.delete("/" + TABLE + "/scanner/NONEXISTENT_SCAN");
392    assertEquals(404, response.getCode());
393  }
394
395  @Test
396  public void testScannerWithIncludeStartStopRowXML() throws IOException, JAXBException {
397    final int BATCH_SIZE = 5;
398    // new scanner
399    ScannerModel model = new ScannerModel();
400    // model.setBatch(BATCH_SIZE);
401    model.addColumn(Bytes.toBytes(COLUMN_1));
402    model.setStartRow(Bytes.toBytes("aaa"));
403    model.setEndRow(Bytes.toBytes("aae"));
404    StringWriter writer = new StringWriter();
405    marshaller.marshal(model, writer);
406    byte[] body = Bytes.toBytes(writer.toString());
407
408    conf.set("hbase.rest.readonly", "false");
409    Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
410    assertEquals(201, response.getCode());
411    String scannerURI = response.getLocation();
412    assertNotNull(scannerURI);
413
414    // get a cell set
415    response = client.get(scannerURI, Constants.MIMETYPE_XML);
416    assertEquals(200, response.getCode());
417    assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
418    CellSetModel cellSet =
419      (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
420
421    assertEquals(4, countCellSet(cellSet));
422
423    // test with include the start row false
424    model.setIncludeStartRow(false);
425    writer = new StringWriter();
426    marshaller.marshal(model, writer);
427    body = Bytes.toBytes(writer.toString());
428
429    conf.set("hbase.rest.readonly", "false");
430    response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
431    assertEquals(201, response.getCode());
432    scannerURI = response.getLocation();
433    assertNotNull(scannerURI);
434
435    // get a cell set
436    response = client.get(scannerURI, Constants.MIMETYPE_XML);
437    assertEquals(200, response.getCode());
438    assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
439    cellSet = (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
440
441    assertEquals(3, countCellSet(cellSet));
442
443    // test with include stop row true and start row false
444    model.setIncludeStartRow(false);
445    model.setIncludeStopRow(true);
446    writer = new StringWriter();
447    marshaller.marshal(model, writer);
448    body = Bytes.toBytes(writer.toString());
449
450    conf.set("hbase.rest.readonly", "false");
451    response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
452    assertEquals(201, response.getCode());
453    scannerURI = response.getLocation();
454    assertNotNull(scannerURI);
455
456    // get a cell set
457    response = client.get(scannerURI, Constants.MIMETYPE_XML);
458    assertEquals(200, response.getCode());
459    assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
460    cellSet = (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
461
462    assertEquals(4, countCellSet(cellSet));
463
464    // test with including the start row true and stop row true
465    model.setIncludeStartRow(true);
466    model.setIncludeStopRow(true);
467    writer = new StringWriter();
468    marshaller.marshal(model, writer);
469    body = Bytes.toBytes(writer.toString());
470
471    conf.set("hbase.rest.readonly", "false");
472    response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
473    assertEquals(201, response.getCode());
474    scannerURI = response.getLocation();
475    assertNotNull(scannerURI);
476
477    // get a cell set
478    response = client.get(scannerURI, Constants.MIMETYPE_XML);
479    assertEquals(200, response.getCode());
480    assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
481    cellSet = (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
482
483    assertEquals(5, countCellSet(cellSet));
484  }
485
486  @Test
487  public void testScannerWithIncludeStartStopRowPB() throws IOException {
488    final int BATCH_SIZE = 10;
489    // new scanner
490    ScannerModel model = new ScannerModel();
491    // model.setBatch(BATCH_SIZE);
492    model.addColumn(Bytes.toBytes(COLUMN_1));
493    model.setStartRow(Bytes.toBytes("aaa"));
494    model.setEndRow(Bytes.toBytes("aae"));
495
496    // test put operation is forbidden in read-only mode
497    conf.set("hbase.rest.readonly", "false");
498    Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
499      model.createProtobufOutput());
500    assertEquals(201, response.getCode());
501    String scannerURI = response.getLocation();
502    assertNotNull(scannerURI);
503
504    // get a cell set
505    response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
506    assertEquals(200, response.getCode());
507    assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
508    CellSetModel cellSet = new CellSetModel();
509    cellSet.getObjectFromMessage(response.getBody());
510    assertEquals(4, countCellSet(cellSet));
511
512    // test with include start row false
513    model.setIncludeStartRow(false);
514    response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
515      model.createProtobufOutput());
516    assertEquals(201, response.getCode());
517    scannerURI = response.getLocation();
518    assertNotNull(scannerURI);
519
520    // get a cell set
521    response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
522    assertEquals(200, response.getCode());
523    assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
524    cellSet = new CellSetModel();
525    cellSet.getObjectFromMessage(response.getBody());
526    assertEquals(3, countCellSet(cellSet));
527
528    // test with include stop row true
529    model.setIncludeStartRow(true);
530    model.setIncludeStopRow(true);
531    response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
532      model.createProtobufOutput());
533    assertEquals(201, response.getCode());
534    scannerURI = response.getLocation();
535    assertNotNull(scannerURI);
536
537    // get a cell set
538    response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
539    assertEquals(200, response.getCode());
540    assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
541    cellSet = new CellSetModel();
542    cellSet.getObjectFromMessage(response.getBody());
543    assertEquals(5, countCellSet(cellSet));
544  }
545}