View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.rest;
21  
22  import java.io.ByteArrayInputStream;
23  import java.io.IOException;
24  import java.io.StringWriter;
25  import java.util.Iterator;
26  import java.util.Random;
27  
28  import javax.xml.bind.JAXBContext;
29  import javax.xml.bind.JAXBException;
30  import javax.xml.bind.Marshaller;
31  import javax.xml.bind.Unmarshaller;
32  
33  import org.apache.commons.httpclient.Header;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.hbase.*;
36  import org.apache.hadoop.hbase.client.HBaseAdmin;
37  import org.apache.hadoop.hbase.client.HTable;
38  import org.apache.hadoop.hbase.client.Put;
39  import org.apache.hadoop.hbase.client.Durability;
40  import org.apache.hadoop.hbase.rest.client.Client;
41  import org.apache.hadoop.hbase.rest.client.Cluster;
42  import org.apache.hadoop.hbase.rest.client.Response;
43  import org.apache.hadoop.hbase.rest.model.CellModel;
44  import org.apache.hadoop.hbase.rest.model.CellSetModel;
45  import org.apache.hadoop.hbase.rest.model.RowModel;
46  import org.apache.hadoop.hbase.rest.model.ScannerModel;
47  import org.apache.hadoop.hbase.testclassification.MediumTests;
48  import org.apache.hadoop.hbase.util.Bytes;
49  
50  import static org.junit.Assert.*;
51  
52  import org.junit.AfterClass;
53  import org.junit.BeforeClass;
54  import org.junit.Test;
55  import org.junit.experimental.categories.Category;
56  
57  @Category(MediumTests.class)
58  public class TestScannerResource {
59    private static final String TABLE = "TestScannerResource";
60    private static final String NONEXISTENT_TABLE = "ThisTableDoesNotExist";
61    private static final String CFA = "a";
62    private static final String CFB = "b";
63    private static final String COLUMN_1 = CFA + ":1";
64    private static final String COLUMN_2 = CFB + ":2";
65  
66    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
67    private static final HBaseRESTTestingUtility REST_TEST_UTIL = 
68      new HBaseRESTTestingUtility();
69    private static Client client;
70    private static JAXBContext context;
71    private static Marshaller marshaller;
72    private static Unmarshaller unmarshaller;
73    private static int expectedRows1;
74    private static int expectedRows2;
75    private static Configuration conf;
76  
77    static int insertData(Configuration conf, String tableName, String column, double prob)
78        throws IOException {
79      Random rng = new Random();
80      int count = 0;
81      HTable table = new HTable(conf, tableName);
82      byte[] k = new byte[3];
83      byte [][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(column));
84      for (byte b1 = 'a'; b1 < 'z'; b1++) {
85        for (byte b2 = 'a'; b2 < 'z'; b2++) {
86          for (byte b3 = 'a'; b3 < 'z'; b3++) {
87            if (rng.nextDouble() < prob) {
88              k[0] = b1;
89              k[1] = b2;
90              k[2] = b3;
91              Put put = new Put(k);
92              put.setDurability(Durability.SKIP_WAL);
93              put.add(famAndQf[0], famAndQf[1], k);
94              table.put(put);
95              count++;
96            }
97          }
98        }
99      }
100     table.flushCommits();
101     table.close();
102     return count;
103   }
104 
105   static int countCellSet(CellSetModel model) {
106     int count = 0;
107     Iterator<RowModel> rows = model.getRows().iterator();
108     while (rows.hasNext()) {
109       RowModel row = rows.next();
110       Iterator<CellModel> cells = row.getCells().iterator();
111       while (cells.hasNext()) {
112         cells.next();
113         count++;
114       }
115     }
116     return count;
117   }
118 
119   private static int fullTableScan(ScannerModel model) throws IOException {
120     model.setBatch(100);
121     Response response = client.put("/" + TABLE + "/scanner",
122       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
123     assertEquals(response.getCode(), 201);
124     String scannerURI = response.getLocation();
125     assertNotNull(scannerURI);
126     int count = 0;
127     while (true) {
128       response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
129       assertTrue(response.getCode() == 200 || response.getCode() == 204);
130       if (response.getCode() == 200) {
131         assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
132         CellSetModel cellSet = new CellSetModel();
133         cellSet.getObjectFromMessage(response.getBody());
134         Iterator<RowModel> rows = cellSet.getRows().iterator();
135         while (rows.hasNext()) {
136           RowModel row = rows.next();
137           Iterator<CellModel> cells = row.getCells().iterator();
138           while (cells.hasNext()) {
139             cells.next();
140             count++;
141           }
142         }
143       } else {
144         break;
145       }
146     }
147     // delete the scanner
148     response = client.delete(scannerURI);
149     assertEquals(response.getCode(), 200);
150     return count;
151   }
152 
153   @BeforeClass
154   public static void setUpBeforeClass() throws Exception {
155     conf = TEST_UTIL.getConfiguration();
156     TEST_UTIL.startMiniCluster();
157     REST_TEST_UTIL.startServletContainer(conf);
158     client = new Client(new Cluster().add("localhost",
159       REST_TEST_UTIL.getServletPort()));
160     context = JAXBContext.newInstance(
161       CellModel.class,
162       CellSetModel.class,
163       RowModel.class,
164       ScannerModel.class);
165     marshaller = context.createMarshaller();
166     unmarshaller = context.createUnmarshaller();
167     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
168     if (admin.tableExists(TABLE)) {
169       return;
170     }
171     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
172     htd.addFamily(new HColumnDescriptor(CFA));
173     htd.addFamily(new HColumnDescriptor(CFB));
174     admin.createTable(htd);
175     expectedRows1 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_1, 1.0);
176     expectedRows2 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_2, 0.5);
177   }
178 
179   @AfterClass
180   public static void tearDownAfterClass() throws Exception {
181     REST_TEST_UTIL.shutdownServletContainer();
182     TEST_UTIL.shutdownMiniCluster();
183   }
184 
185   @Test
186   public void testSimpleScannerXML() throws IOException, JAXBException {
187     final int BATCH_SIZE = 5;
188     // new scanner
189     ScannerModel model = new ScannerModel();
190     model.setBatch(BATCH_SIZE);
191     model.addColumn(Bytes.toBytes(COLUMN_1));
192     StringWriter writer = new StringWriter();
193     marshaller.marshal(model, writer);
194     byte[] body = Bytes.toBytes(writer.toString());
195 
196     // test put operation is forbidden in read-only mode
197     conf.set("hbase.rest.readonly", "true");
198     Response response = client.put("/" + TABLE + "/scanner",
199       Constants.MIMETYPE_XML, body);
200     assertEquals(response.getCode(), 403);
201     String scannerURI = response.getLocation();
202     assertNull(scannerURI);
203 
204     // recall previous put operation with read-only off
205     conf.set("hbase.rest.readonly", "false");
206     response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML,
207       body);
208     assertEquals(response.getCode(), 201);
209     scannerURI = response.getLocation();
210     assertNotNull(scannerURI);
211 
212     // get a cell set
213     response = client.get(scannerURI, Constants.MIMETYPE_XML);
214     assertEquals(response.getCode(), 200);
215     assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
216     CellSetModel cellSet = (CellSetModel)
217       unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
218     // confirm batch size conformance
219     assertEquals(countCellSet(cellSet), BATCH_SIZE);
220 
221     // test delete scanner operation is forbidden in read-only mode
222     conf.set("hbase.rest.readonly", "true");
223     response = client.delete(scannerURI);
224     assertEquals(response.getCode(), 403);
225 
226     // recall previous delete scanner operation with read-only off
227     conf.set("hbase.rest.readonly", "false");
228     response = client.delete(scannerURI);
229     assertEquals(response.getCode(), 200);
230   }
231 
232   @Test
233   public void testSimpleScannerPB() throws IOException {
234     final int BATCH_SIZE = 10;
235     // new scanner
236     ScannerModel model = new ScannerModel();
237     model.setBatch(BATCH_SIZE);
238     model.addColumn(Bytes.toBytes(COLUMN_1));
239 
240     // test put operation is forbidden in read-only mode
241     conf.set("hbase.rest.readonly", "true");
242     Response response = client.put("/" + TABLE + "/scanner",
243       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
244     assertEquals(response.getCode(), 403);
245     String scannerURI = response.getLocation();
246     assertNull(scannerURI);
247 
248     // recall previous put operation with read-only off
249     conf.set("hbase.rest.readonly", "false");
250     response = client.put("/" + TABLE + "/scanner",
251       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
252     assertEquals(response.getCode(), 201);
253     scannerURI = response.getLocation();
254     assertNotNull(scannerURI);
255 
256     // get a cell set
257     response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
258     assertEquals(response.getCode(), 200);
259     assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
260     CellSetModel cellSet = new CellSetModel();
261     cellSet.getObjectFromMessage(response.getBody());
262     // confirm batch size conformance
263     assertEquals(countCellSet(cellSet), BATCH_SIZE);
264 
265     // test delete scanner operation is forbidden in read-only mode
266     conf.set("hbase.rest.readonly", "true");
267     response = client.delete(scannerURI);
268     assertEquals(response.getCode(), 403);
269 
270     // recall previous delete scanner operation with read-only off
271     conf.set("hbase.rest.readonly", "false");
272     response = client.delete(scannerURI);
273     assertEquals(response.getCode(), 200);
274   }
275 
276   @Test
277   public void testSimpleScannerBinary() throws IOException {
278     // new scanner
279     ScannerModel model = new ScannerModel();
280     model.setBatch(1);
281     model.addColumn(Bytes.toBytes(COLUMN_1));
282 
283     // test put operation is forbidden in read-only mode
284     conf.set("hbase.rest.readonly", "true");
285     Response response = client.put("/" + TABLE + "/scanner",
286       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
287     assertEquals(response.getCode(), 403);
288     String scannerURI = response.getLocation();
289     assertNull(scannerURI);
290 
291     // recall previous put operation with read-only off
292     conf.set("hbase.rest.readonly", "false");
293     response = client.put("/" + TABLE + "/scanner",
294       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
295     assertEquals(response.getCode(), 201);
296     scannerURI = response.getLocation();
297     assertNotNull(scannerURI);
298 
299     // get a cell
300     response = client.get(scannerURI, Constants.MIMETYPE_BINARY);
301     assertEquals(response.getCode(), 200);
302     assertEquals(Constants.MIMETYPE_BINARY, response.getHeader("content-type"));
303     // verify that data was returned
304     assertTrue(response.getBody().length > 0);
305     // verify that the expected X-headers are present
306     boolean foundRowHeader = false, foundColumnHeader = false,
307       foundTimestampHeader = false;
308     for (Header header: response.getHeaders()) {
309       if (header.getName().equals("X-Row")) {
310         foundRowHeader = true;
311       } else if (header.getName().equals("X-Column")) {
312         foundColumnHeader = true;
313       } else if (header.getName().equals("X-Timestamp")) {
314         foundTimestampHeader = true;
315       }
316     }
317     assertTrue(foundRowHeader);
318     assertTrue(foundColumnHeader);
319     assertTrue(foundTimestampHeader);
320 
321     // test delete scanner operation is forbidden in read-only mode
322     conf.set("hbase.rest.readonly", "true");
323     response = client.delete(scannerURI);
324     assertEquals(response.getCode(), 403);
325 
326     // recall previous delete scanner operation with read-only off
327     conf.set("hbase.rest.readonly", "false");
328     response = client.delete(scannerURI);
329     assertEquals(response.getCode(), 200);
330   }
331 
332   @Test
333   public void testFullTableScan() throws IOException {
334     ScannerModel model = new ScannerModel();
335     model.addColumn(Bytes.toBytes(COLUMN_1));
336     assertEquals(fullTableScan(model), expectedRows1);
337 
338     model = new ScannerModel();
339     model.addColumn(Bytes.toBytes(COLUMN_2));
340     assertEquals(fullTableScan(model), expectedRows2);
341   }
342 
343   @Test
344   public void testTableDoesNotExist() throws IOException, JAXBException {
345     ScannerModel model = new ScannerModel();
346     StringWriter writer = new StringWriter();
347     marshaller.marshal(model, writer);
348     byte[] body = Bytes.toBytes(writer.toString());
349     Response response = client.put("/" + NONEXISTENT_TABLE +
350       "/scanner", Constants.MIMETYPE_XML, body);
351     assertEquals(response.getCode(), 404);
352   }
353 
354 }
355