001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.rest;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertNull;
023import static org.junit.Assert.assertTrue;
024
025import java.io.ByteArrayInputStream;
026import java.io.IOException;
027import java.io.StringWriter;
028import java.util.ArrayList;
029import java.util.Iterator;
030import java.util.List;
031import java.util.Random;
032import javax.xml.bind.JAXBContext;
033import javax.xml.bind.JAXBException;
034import javax.xml.bind.Marshaller;
035import javax.xml.bind.Unmarshaller;
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.hbase.CellUtil;
038import org.apache.hadoop.hbase.HBaseClassTestRule;
039import org.apache.hadoop.hbase.HBaseTestingUtility;
040import org.apache.hadoop.hbase.HColumnDescriptor;
041import org.apache.hadoop.hbase.HTableDescriptor;
042import org.apache.hadoop.hbase.TableName;
043import org.apache.hadoop.hbase.client.Admin;
044import org.apache.hadoop.hbase.client.Connection;
045import org.apache.hadoop.hbase.client.ConnectionFactory;
046import org.apache.hadoop.hbase.client.Durability;
047import org.apache.hadoop.hbase.client.Put;
048import org.apache.hadoop.hbase.client.Table;
049import org.apache.hadoop.hbase.rest.client.Client;
050import org.apache.hadoop.hbase.rest.client.Cluster;
051import org.apache.hadoop.hbase.rest.client.Response;
052import org.apache.hadoop.hbase.rest.model.CellModel;
053import org.apache.hadoop.hbase.rest.model.CellSetModel;
054import org.apache.hadoop.hbase.rest.model.RowModel;
055import org.apache.hadoop.hbase.rest.model.ScannerModel;
056import org.apache.hadoop.hbase.testclassification.MediumTests;
057import org.apache.hadoop.hbase.testclassification.RestTests;
058import org.apache.hadoop.hbase.util.Bytes;
059import org.apache.http.Header;
060import org.junit.AfterClass;
061import org.junit.BeforeClass;
062import org.junit.ClassRule;
063import org.junit.Test;
064import org.junit.experimental.categories.Category;
065import org.slf4j.Logger;
066import org.slf4j.LoggerFactory;
067
068@Category({RestTests.class, MediumTests.class})
069public class TestScannerResource {
070
071  @ClassRule
072  public static final HBaseClassTestRule CLASS_RULE =
073      HBaseClassTestRule.forClass(TestScannerResource.class);
074
075  private static final Logger LOG = LoggerFactory.getLogger(TestScannerResource.class);
076  private static final TableName TABLE = TableName.valueOf("TestScannerResource");
077  private static final TableName TABLE_TO_BE_DISABLED = TableName.valueOf("ScannerResourceDisable");
078  private static final String NONEXISTENT_TABLE = "ThisTableDoesNotExist";
079  private static final String CFA = "a";
080  private static final String CFB = "b";
081  private static final String COLUMN_1 = CFA + ":1";
082  private static final String COLUMN_2 = CFB + ":2";
083
084  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
085  private static final HBaseRESTTestingUtility REST_TEST_UTIL =
086    new HBaseRESTTestingUtility();
087  private static Client client;
088  private static JAXBContext context;
089  private static Marshaller marshaller;
090  private static Unmarshaller unmarshaller;
091  private static int expectedRows1;
092  private static int expectedRows2;
093  private static Configuration conf;
094
095  static int insertData(Configuration conf, TableName tableName, String column, double prob)
096      throws IOException {
097    Random rng = new Random();
098    byte[] k = new byte[3];
099    byte [][] famAndQf = CellUtil.parseColumn(Bytes.toBytes(column));
100    List<Put> puts = new ArrayList<>();
101    for (byte b1 = 'a'; b1 < 'z'; b1++) {
102      for (byte b2 = 'a'; b2 < 'z'; b2++) {
103        for (byte b3 = 'a'; b3 < 'z'; b3++) {
104          if (rng.nextDouble() < prob) {
105            k[0] = b1;
106            k[1] = b2;
107            k[2] = b3;
108            Put put = new Put(k);
109            put.setDurability(Durability.SKIP_WAL);
110            put.addColumn(famAndQf[0], famAndQf[1], k);
111            puts.add(put);
112          }
113        }
114      }
115    }
116    try (Connection conn = ConnectionFactory.createConnection(conf);
117        Table table = conn.getTable(tableName)) {
118      table.put(puts);
119    }
120    return puts.size();
121  }
122
123  static int countCellSet(CellSetModel model) {
124    int count = 0;
125    Iterator<RowModel> rows = model.getRows().iterator();
126    while (rows.hasNext()) {
127      RowModel row = rows.next();
128      Iterator<CellModel> cells = row.getCells().iterator();
129      while (cells.hasNext()) {
130        cells.next();
131        count++;
132      }
133    }
134    return count;
135  }
136
137  private static int fullTableScan(ScannerModel model) throws IOException {
138    model.setBatch(100);
139    Response response = client.put("/" + TABLE + "/scanner",
140      Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
141    assertEquals(201, response.getCode());
142    String scannerURI = response.getLocation();
143    assertNotNull(scannerURI);
144    int count = 0;
145    while (true) {
146      response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
147      assertTrue(response.getCode() == 200 || response.getCode() == 204);
148      if (response.getCode() == 200) {
149        assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
150        CellSetModel cellSet = new CellSetModel();
151        cellSet.getObjectFromMessage(response.getBody());
152        Iterator<RowModel> rows = cellSet.getRows().iterator();
153        while (rows.hasNext()) {
154          RowModel row = rows.next();
155          Iterator<CellModel> cells = row.getCells().iterator();
156          while (cells.hasNext()) {
157            cells.next();
158            count++;
159          }
160        }
161      } else {
162        break;
163      }
164    }
165    // delete the scanner
166    response = client.delete(scannerURI);
167    assertEquals(200, response.getCode());
168    return count;
169  }
170
171  @BeforeClass
172  public static void setUpBeforeClass() throws Exception {
173    conf = TEST_UTIL.getConfiguration();
174    TEST_UTIL.startMiniCluster();
175    REST_TEST_UTIL.startServletContainer(conf);
176    client = new Client(new Cluster().add("localhost",
177      REST_TEST_UTIL.getServletPort()));
178    context = JAXBContext.newInstance(
179      CellModel.class,
180      CellSetModel.class,
181      RowModel.class,
182      ScannerModel.class);
183    marshaller = context.createMarshaller();
184    unmarshaller = context.createUnmarshaller();
185    Admin admin = TEST_UTIL.getAdmin();
186    if (admin.tableExists(TABLE)) {
187      return;
188    }
189    HTableDescriptor htd = new HTableDescriptor(TABLE);
190    htd.addFamily(new HColumnDescriptor(CFA));
191    htd.addFamily(new HColumnDescriptor(CFB));
192    admin.createTable(htd);
193    expectedRows1 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_1, 1.0);
194    expectedRows2 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_2, 0.5);
195
196    htd = new HTableDescriptor(TABLE_TO_BE_DISABLED);
197    htd.addFamily(new HColumnDescriptor(CFA));
198    htd.addFamily(new HColumnDescriptor(CFB));
199    admin.createTable(htd);
200  }
201
202  @AfterClass
203  public static void tearDownAfterClass() throws Exception {
204    REST_TEST_UTIL.shutdownServletContainer();
205    TEST_UTIL.shutdownMiniCluster();
206  }
207
208  @Test
209  public void testSimpleScannerXML() throws IOException, JAXBException {
210    final int BATCH_SIZE = 5;
211    // new scanner
212    ScannerModel model = new ScannerModel();
213    model.setBatch(BATCH_SIZE);
214    model.addColumn(Bytes.toBytes(COLUMN_1));
215    StringWriter writer = new StringWriter();
216    marshaller.marshal(model, writer);
217    byte[] body = Bytes.toBytes(writer.toString());
218
219    // test put operation is forbidden in read-only mode
220    conf.set("hbase.rest.readonly", "true");
221    Response response = client.put("/" + TABLE + "/scanner",
222      Constants.MIMETYPE_XML, body);
223    assertEquals(403, response.getCode());
224    String scannerURI = response.getLocation();
225    assertNull(scannerURI);
226
227    // recall previous put operation with read-only off
228    conf.set("hbase.rest.readonly", "false");
229    response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML,
230      body);
231    assertEquals(201, response.getCode());
232    scannerURI = response.getLocation();
233    assertNotNull(scannerURI);
234
235    // get a cell set
236    response = client.get(scannerURI, Constants.MIMETYPE_XML);
237    assertEquals(200, response.getCode());
238    assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
239    CellSetModel cellSet = (CellSetModel)
240      unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
241    // confirm batch size conformance
242    assertEquals(BATCH_SIZE, countCellSet(cellSet));
243
244    // test delete scanner operation is forbidden in read-only mode
245    conf.set("hbase.rest.readonly", "true");
246    response = client.delete(scannerURI);
247    assertEquals(403, response.getCode());
248
249    // recall previous delete scanner operation with read-only off
250    conf.set("hbase.rest.readonly", "false");
251    response = client.delete(scannerURI);
252    assertEquals(200, response.getCode());
253  }
254
255  @Test
256  public void testSimpleScannerPB() throws IOException {
257    final int BATCH_SIZE = 10;
258    // new scanner
259    ScannerModel model = new ScannerModel();
260    model.setBatch(BATCH_SIZE);
261    model.addColumn(Bytes.toBytes(COLUMN_1));
262
263    // test put operation is forbidden in read-only mode
264    conf.set("hbase.rest.readonly", "true");
265    Response response = client.put("/" + TABLE + "/scanner",
266      Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
267    assertEquals(403, response.getCode());
268    String scannerURI = response.getLocation();
269    assertNull(scannerURI);
270
271    // recall previous put operation with read-only off
272    conf.set("hbase.rest.readonly", "false");
273    response = client.put("/" + TABLE + "/scanner",
274      Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
275    assertEquals(201, response.getCode());
276    scannerURI = response.getLocation();
277    assertNotNull(scannerURI);
278
279    // get a cell set
280    response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
281    assertEquals(200, response.getCode());
282    assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
283    CellSetModel cellSet = new CellSetModel();
284    cellSet.getObjectFromMessage(response.getBody());
285    // confirm batch size conformance
286    assertEquals(BATCH_SIZE, countCellSet(cellSet));
287
288    // test delete scanner operation is forbidden in read-only mode
289    conf.set("hbase.rest.readonly", "true");
290    response = client.delete(scannerURI);
291    assertEquals(403, response.getCode());
292
293    // recall previous delete scanner operation with read-only off
294    conf.set("hbase.rest.readonly", "false");
295    response = client.delete(scannerURI);
296    assertEquals(200, response.getCode());
297  }
298
299  @Test
300  public void testSimpleScannerBinary() throws IOException {
301    // new scanner
302    ScannerModel model = new ScannerModel();
303    model.setBatch(1);
304    model.addColumn(Bytes.toBytes(COLUMN_1));
305
306    // test put operation is forbidden in read-only mode
307    conf.set("hbase.rest.readonly", "true");
308    Response response = client.put("/" + TABLE + "/scanner",
309      Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
310    assertEquals(403, response.getCode());
311    String scannerURI = response.getLocation();
312    assertNull(scannerURI);
313
314    // recall previous put operation with read-only off
315    conf.set("hbase.rest.readonly", "false");
316    response = client.put("/" + TABLE + "/scanner",
317      Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
318    assertEquals(201, response.getCode());
319    scannerURI = response.getLocation();
320    assertNotNull(scannerURI);
321
322    // get a cell
323    response = client.get(scannerURI, Constants.MIMETYPE_BINARY);
324    assertEquals(200, response.getCode());
325    assertEquals(Constants.MIMETYPE_BINARY, response.getHeader("content-type"));
326    // verify that data was returned
327    assertTrue(response.getBody().length > 0);
328    // verify that the expected X-headers are present
329    boolean foundRowHeader = false, foundColumnHeader = false,
330      foundTimestampHeader = false;
331    for (Header header: response.getHeaders()) {
332      if (header.getName().equals("X-Row")) {
333        foundRowHeader = true;
334      } else if (header.getName().equals("X-Column")) {
335        foundColumnHeader = true;
336      } else if (header.getName().equals("X-Timestamp")) {
337        foundTimestampHeader = true;
338      }
339    }
340    assertTrue(foundRowHeader);
341    assertTrue(foundColumnHeader);
342    assertTrue(foundTimestampHeader);
343
344    // test delete scanner operation is forbidden in read-only mode
345    conf.set("hbase.rest.readonly", "true");
346    response = client.delete(scannerURI);
347    assertEquals(403, response.getCode());
348
349    // recall previous delete scanner operation with read-only off
350    conf.set("hbase.rest.readonly", "false");
351    response = client.delete(scannerURI);
352    assertEquals(200, response.getCode());
353  }
354
355  @Test
356  public void testFullTableScan() throws IOException {
357    ScannerModel model = new ScannerModel();
358    model.addColumn(Bytes.toBytes(COLUMN_1));
359    assertEquals(expectedRows1, fullTableScan(model));
360
361    model = new ScannerModel();
362    model.addColumn(Bytes.toBytes(COLUMN_2));
363    assertEquals(expectedRows2, fullTableScan(model));
364  }
365
366  @Test
367  public void testTableDoesNotExist() throws IOException, JAXBException {
368    ScannerModel model = new ScannerModel();
369    StringWriter writer = new StringWriter();
370    marshaller.marshal(model, writer);
371    byte[] body = Bytes.toBytes(writer.toString());
372    Response response = client.put("/" + NONEXISTENT_TABLE +
373      "/scanner", Constants.MIMETYPE_XML, body);
374    String scannerURI = response.getLocation();
375    assertNotNull(scannerURI);
376    response = client.get(scannerURI, Constants.MIMETYPE_XML);
377    assertEquals(404, response.getCode());
378  }
379
380  // performs table scan during which the underlying table is disabled
381  // assert that we get 410 (Gone)
382  @Test
383  public void testTableScanWithTableDisable() throws IOException {
384    ScannerModel model = new ScannerModel();
385    model.addColumn(Bytes.toBytes(COLUMN_1));
386    model.setCaching(1);
387    Response response = client.put("/" + TABLE_TO_BE_DISABLED + "/scanner",
388      Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
389    assertEquals(201, response.getCode());
390    String scannerURI = response.getLocation();
391    assertNotNull(scannerURI);
392    TEST_UTIL.getAdmin().disableTable(TABLE_TO_BE_DISABLED);
393      response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
394    assertTrue("got " + response.getCode(), response.getCode() == 410);
395  }
396
397}
398