001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.rest;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertNull;
023import static org.junit.Assert.assertTrue;
024
025import java.io.ByteArrayInputStream;
026import java.io.IOException;
027import java.io.StringWriter;
028import java.util.ArrayList;
029import java.util.Iterator;
030import java.util.List;
031import java.util.Random;
032import java.util.concurrent.ThreadLocalRandom;
033import javax.xml.bind.JAXBContext;
034import javax.xml.bind.JAXBException;
035import javax.xml.bind.Marshaller;
036import javax.xml.bind.Unmarshaller;
037import org.apache.hadoop.conf.Configuration;
038import org.apache.hadoop.hbase.CellUtil;
039import org.apache.hadoop.hbase.HBaseClassTestRule;
040import org.apache.hadoop.hbase.HBaseTestingUtility;
041import org.apache.hadoop.hbase.HColumnDescriptor;
042import org.apache.hadoop.hbase.HTableDescriptor;
043import org.apache.hadoop.hbase.TableName;
044import org.apache.hadoop.hbase.client.Admin;
045import org.apache.hadoop.hbase.client.Connection;
046import org.apache.hadoop.hbase.client.ConnectionFactory;
047import org.apache.hadoop.hbase.client.Durability;
048import org.apache.hadoop.hbase.client.Put;
049import org.apache.hadoop.hbase.client.Table;
050import org.apache.hadoop.hbase.rest.client.Client;
051import org.apache.hadoop.hbase.rest.client.Cluster;
052import org.apache.hadoop.hbase.rest.client.Response;
053import org.apache.hadoop.hbase.rest.model.CellModel;
054import org.apache.hadoop.hbase.rest.model.CellSetModel;
055import org.apache.hadoop.hbase.rest.model.RowModel;
056import org.apache.hadoop.hbase.rest.model.ScannerModel;
057import org.apache.hadoop.hbase.testclassification.MediumTests;
058import org.apache.hadoop.hbase.testclassification.RestTests;
059import org.apache.hadoop.hbase.util.Bytes;
060import org.apache.http.Header;
061import org.junit.AfterClass;
062import org.junit.BeforeClass;
063import org.junit.ClassRule;
064import org.junit.Test;
065import org.junit.experimental.categories.Category;
066import org.slf4j.Logger;
067import org.slf4j.LoggerFactory;
068
069@Category({ RestTests.class, MediumTests.class })
070public class TestScannerResource {
071
072  @ClassRule
073  public static final HBaseClassTestRule CLASS_RULE =
074    HBaseClassTestRule.forClass(TestScannerResource.class);
075
076  private static final Logger LOG = LoggerFactory.getLogger(TestScannerResource.class);
077  private static final TableName TABLE = TableName.valueOf("TestScannerResource");
078  private static final TableName TABLE_TO_BE_DISABLED = TableName.valueOf("ScannerResourceDisable");
079  private static final String NONEXISTENT_TABLE = "ThisTableDoesNotExist";
080  private static final String CFA = "a";
081  private static final String CFB = "b";
082  private static final String COLUMN_1 = CFA + ":1";
083  private static final String COLUMN_2 = CFB + ":2";
084
085  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
086  private static final HBaseRESTTestingUtility REST_TEST_UTIL = new HBaseRESTTestingUtility();
087  private static Client client;
088  private static JAXBContext context;
089  private static Marshaller marshaller;
090  private static Unmarshaller unmarshaller;
091  private static int expectedRows1;
092  private static int expectedRows2;
093  private static Configuration conf;
094
095  static int insertData(Configuration conf, TableName tableName, String column, double prob)
096    throws IOException {
097    Random rng = ThreadLocalRandom.current();
098    byte[] k = new byte[3];
099    byte[][] famAndQf = CellUtil.parseColumn(Bytes.toBytes(column));
100    List<Put> puts = new ArrayList<>();
101    for (byte b1 = 'a'; b1 < 'z'; b1++) {
102      for (byte b2 = 'a'; b2 < 'z'; b2++) {
103        for (byte b3 = 'a'; b3 < 'z'; b3++) {
104          if (rng.nextDouble() < prob) {
105            k[0] = b1;
106            k[1] = b2;
107            k[2] = b3;
108            Put put = new Put(k);
109            put.setDurability(Durability.SKIP_WAL);
110            put.addColumn(famAndQf[0], famAndQf[1], k);
111            puts.add(put);
112          }
113        }
114      }
115    }
116    try (Connection conn = ConnectionFactory.createConnection(conf);
117      Table table = conn.getTable(tableName)) {
118      table.put(puts);
119    }
120    return puts.size();
121  }
122
123  static int countCellSet(CellSetModel model) {
124    int count = 0;
125    Iterator<RowModel> rows = model.getRows().iterator();
126    while (rows.hasNext()) {
127      RowModel row = rows.next();
128      Iterator<CellModel> cells = row.getCells().iterator();
129      while (cells.hasNext()) {
130        cells.next();
131        count++;
132      }
133    }
134    return count;
135  }
136
137  private static int fullTableScan(ScannerModel model) throws IOException {
138    model.setBatch(100);
139    Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
140      model.createProtobufOutput());
141    assertEquals(201, response.getCode());
142    String scannerURI = response.getLocation();
143    assertNotNull(scannerURI);
144    int count = 0;
145    while (true) {
146      response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
147      assertTrue(response.getCode() == 200 || response.getCode() == 204);
148      if (response.getCode() == 200) {
149        assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
150        CellSetModel cellSet = new CellSetModel();
151        cellSet.getObjectFromMessage(response.getBody());
152        Iterator<RowModel> rows = cellSet.getRows().iterator();
153        while (rows.hasNext()) {
154          RowModel row = rows.next();
155          Iterator<CellModel> cells = row.getCells().iterator();
156          while (cells.hasNext()) {
157            cells.next();
158            count++;
159          }
160        }
161      } else {
162        break;
163      }
164    }
165    // delete the scanner
166    response = client.delete(scannerURI);
167    assertEquals(200, response.getCode());
168    return count;
169  }
170
171  @BeforeClass
172  public static void setUpBeforeClass() throws Exception {
173    conf = TEST_UTIL.getConfiguration();
174    TEST_UTIL.startMiniCluster();
175    REST_TEST_UTIL.startServletContainer(conf);
176    client = new Client(new Cluster().add("localhost", REST_TEST_UTIL.getServletPort()));
177    context = JAXBContext.newInstance(CellModel.class, CellSetModel.class, RowModel.class,
178      ScannerModel.class);
179    marshaller = context.createMarshaller();
180    unmarshaller = context.createUnmarshaller();
181    Admin admin = TEST_UTIL.getAdmin();
182    if (admin.tableExists(TABLE)) {
183      return;
184    }
185    HTableDescriptor htd = new HTableDescriptor(TABLE);
186    htd.addFamily(new HColumnDescriptor(CFA));
187    htd.addFamily(new HColumnDescriptor(CFB));
188    admin.createTable(htd);
189    expectedRows1 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_1, 1.0);
190    expectedRows2 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_2, 0.5);
191
192    htd = new HTableDescriptor(TABLE_TO_BE_DISABLED);
193    htd.addFamily(new HColumnDescriptor(CFA));
194    htd.addFamily(new HColumnDescriptor(CFB));
195    admin.createTable(htd);
196  }
197
198  @AfterClass
199  public static void tearDownAfterClass() throws Exception {
200    REST_TEST_UTIL.shutdownServletContainer();
201    TEST_UTIL.shutdownMiniCluster();
202  }
203
204  @Test
205  public void testSimpleScannerXML() throws IOException, JAXBException {
206    final int BATCH_SIZE = 5;
207    // new scanner
208    ScannerModel model = new ScannerModel();
209    model.setBatch(BATCH_SIZE);
210    model.addColumn(Bytes.toBytes(COLUMN_1));
211    StringWriter writer = new StringWriter();
212    marshaller.marshal(model, writer);
213    byte[] body = Bytes.toBytes(writer.toString());
214
215    // test put operation is forbidden in read-only mode
216    conf.set("hbase.rest.readonly", "true");
217    Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
218    assertEquals(403, response.getCode());
219    String scannerURI = response.getLocation();
220    assertNull(scannerURI);
221
222    // recall previous put operation with read-only off
223    conf.set("hbase.rest.readonly", "false");
224    response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
225    assertEquals(201, response.getCode());
226    scannerURI = response.getLocation();
227    assertNotNull(scannerURI);
228
229    // get a cell set
230    response = client.get(scannerURI, Constants.MIMETYPE_XML);
231    assertEquals(200, response.getCode());
232    assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
233    CellSetModel cellSet =
234      (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
235    // confirm batch size conformance
236    assertEquals(BATCH_SIZE, countCellSet(cellSet));
237
238    // test delete scanner operation is forbidden in read-only mode
239    conf.set("hbase.rest.readonly", "true");
240    response = client.delete(scannerURI);
241    assertEquals(403, response.getCode());
242
243    // recall previous delete scanner operation with read-only off
244    conf.set("hbase.rest.readonly", "false");
245    response = client.delete(scannerURI);
246    assertEquals(200, response.getCode());
247  }
248
249  @Test
250  public void testSimpleScannerPB() throws IOException {
251    final int BATCH_SIZE = 10;
252    // new scanner
253    ScannerModel model = new ScannerModel();
254    model.setBatch(BATCH_SIZE);
255    model.addColumn(Bytes.toBytes(COLUMN_1));
256
257    // test put operation is forbidden in read-only mode
258    conf.set("hbase.rest.readonly", "true");
259    Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
260      model.createProtobufOutput());
261    assertEquals(403, response.getCode());
262    String scannerURI = response.getLocation();
263    assertNull(scannerURI);
264
265    // recall previous put operation with read-only off
266    conf.set("hbase.rest.readonly", "false");
267    response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
268      model.createProtobufOutput());
269    assertEquals(201, response.getCode());
270    scannerURI = response.getLocation();
271    assertNotNull(scannerURI);
272
273    // get a cell set
274    response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
275    assertEquals(200, response.getCode());
276    assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
277    CellSetModel cellSet = new CellSetModel();
278    cellSet.getObjectFromMessage(response.getBody());
279    // confirm batch size conformance
280    assertEquals(BATCH_SIZE, countCellSet(cellSet));
281
282    // test delete scanner operation is forbidden in read-only mode
283    conf.set("hbase.rest.readonly", "true");
284    response = client.delete(scannerURI);
285    assertEquals(403, response.getCode());
286
287    // recall previous delete scanner operation with read-only off
288    conf.set("hbase.rest.readonly", "false");
289    response = client.delete(scannerURI);
290    assertEquals(200, response.getCode());
291  }
292
293  @Test
294  public void testSimpleScannerBinary() throws IOException {
295    // new scanner
296    ScannerModel model = new ScannerModel();
297    model.setBatch(1);
298    model.addColumn(Bytes.toBytes(COLUMN_1));
299
300    // test put operation is forbidden in read-only mode
301    conf.set("hbase.rest.readonly", "true");
302    Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
303      model.createProtobufOutput());
304    assertEquals(403, response.getCode());
305    String scannerURI = response.getLocation();
306    assertNull(scannerURI);
307
308    // recall previous put operation with read-only off
309    conf.set("hbase.rest.readonly", "false");
310    response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
311      model.createProtobufOutput());
312    assertEquals(201, response.getCode());
313    scannerURI = response.getLocation();
314    assertNotNull(scannerURI);
315
316    // get a cell
317    response = client.get(scannerURI, Constants.MIMETYPE_BINARY);
318    assertEquals(200, response.getCode());
319    assertEquals(Constants.MIMETYPE_BINARY, response.getHeader("content-type"));
320    // verify that data was returned
321    assertTrue(response.getBody().length > 0);
322    // verify that the expected X-headers are present
323    boolean foundRowHeader = false, foundColumnHeader = false, foundTimestampHeader = false;
324    for (Header header : response.getHeaders()) {
325      if (header.getName().equals("X-Row")) {
326        foundRowHeader = true;
327      } else if (header.getName().equals("X-Column")) {
328        foundColumnHeader = true;
329      } else if (header.getName().equals("X-Timestamp")) {
330        foundTimestampHeader = true;
331      }
332    }
333    assertTrue(foundRowHeader);
334    assertTrue(foundColumnHeader);
335    assertTrue(foundTimestampHeader);
336
337    // test delete scanner operation is forbidden in read-only mode
338    conf.set("hbase.rest.readonly", "true");
339    response = client.delete(scannerURI);
340    assertEquals(403, response.getCode());
341
342    // recall previous delete scanner operation with read-only off
343    conf.set("hbase.rest.readonly", "false");
344    response = client.delete(scannerURI);
345    assertEquals(200, response.getCode());
346  }
347
348  @Test
349  public void testFullTableScan() throws IOException {
350    ScannerModel model = new ScannerModel();
351    model.addColumn(Bytes.toBytes(COLUMN_1));
352    assertEquals(expectedRows1, fullTableScan(model));
353
354    model = new ScannerModel();
355    model.addColumn(Bytes.toBytes(COLUMN_2));
356    assertEquals(expectedRows2, fullTableScan(model));
357  }
358
359  @Test
360  public void testTableDoesNotExist() throws IOException, JAXBException {
361    ScannerModel model = new ScannerModel();
362    StringWriter writer = new StringWriter();
363    marshaller.marshal(model, writer);
364    byte[] body = Bytes.toBytes(writer.toString());
365    Response response =
366      client.put("/" + NONEXISTENT_TABLE + "/scanner", Constants.MIMETYPE_XML, body);
367    String scannerURI = response.getLocation();
368    assertNotNull(scannerURI);
369    response = client.get(scannerURI, Constants.MIMETYPE_XML);
370    assertEquals(404, response.getCode());
371  }
372
373  // performs table scan during which the underlying table is disabled
374  // assert that we get 410 (Gone)
375  @Test
376  public void testTableScanWithTableDisable() throws IOException {
377    ScannerModel model = new ScannerModel();
378    model.addColumn(Bytes.toBytes(COLUMN_1));
379    model.setCaching(1);
380    Response response = client.put("/" + TABLE_TO_BE_DISABLED + "/scanner",
381      Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
382    assertEquals(201, response.getCode());
383    String scannerURI = response.getLocation();
384    assertNotNull(scannerURI);
385    TEST_UTIL.getAdmin().disableTable(TABLE_TO_BE_DISABLED);
386    response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
387    assertTrue("got " + response.getCode(), response.getCode() == 410);
388  }
389
390}