1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.rest;
21
22 import java.io.ByteArrayInputStream;
23 import java.io.IOException;
24 import java.io.StringWriter;
25 import java.util.Iterator;
26 import java.util.Random;
27
28 import javax.xml.bind.JAXBContext;
29 import javax.xml.bind.JAXBException;
30 import javax.xml.bind.Marshaller;
31 import javax.xml.bind.Unmarshaller;
32
33 import org.apache.commons.httpclient.Header;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.hbase.*;
36 import org.apache.hadoop.hbase.client.HBaseAdmin;
37 import org.apache.hadoop.hbase.client.HTable;
38 import org.apache.hadoop.hbase.client.Put;
39 import org.apache.hadoop.hbase.client.Durability;
40 import org.apache.hadoop.hbase.rest.client.Client;
41 import org.apache.hadoop.hbase.rest.client.Cluster;
42 import org.apache.hadoop.hbase.rest.client.Response;
43 import org.apache.hadoop.hbase.rest.model.CellModel;
44 import org.apache.hadoop.hbase.rest.model.CellSetModel;
45 import org.apache.hadoop.hbase.rest.model.RowModel;
46 import org.apache.hadoop.hbase.rest.model.ScannerModel;
47 import org.apache.hadoop.hbase.testclassification.MediumTests;
48 import org.apache.hadoop.hbase.util.Bytes;
49
50 import static org.junit.Assert.*;
51
52 import org.junit.AfterClass;
53 import org.junit.BeforeClass;
54 import org.junit.Test;
55 import org.junit.experimental.categories.Category;
56
57 @Category(MediumTests.class)
58 public class TestScannerResource {
59 private static final String TABLE = "TestScannerResource";
60 private static final String NONEXISTENT_TABLE = "ThisTableDoesNotExist";
61 private static final String CFA = "a";
62 private static final String CFB = "b";
63 private static final String COLUMN_1 = CFA + ":1";
64 private static final String COLUMN_2 = CFB + ":2";
65
66 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
67 private static final HBaseRESTTestingUtility REST_TEST_UTIL =
68 new HBaseRESTTestingUtility();
69 private static Client client;
70 private static JAXBContext context;
71 private static Marshaller marshaller;
72 private static Unmarshaller unmarshaller;
73 private static int expectedRows1;
74 private static int expectedRows2;
75 private static Configuration conf;
76
77 static int insertData(Configuration conf, String tableName, String column, double prob)
78 throws IOException {
79 Random rng = new Random();
80 int count = 0;
81 HTable table = new HTable(conf, tableName);
82 byte[] k = new byte[3];
83 byte [][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(column));
84 for (byte b1 = 'a'; b1 < 'z'; b1++) {
85 for (byte b2 = 'a'; b2 < 'z'; b2++) {
86 for (byte b3 = 'a'; b3 < 'z'; b3++) {
87 if (rng.nextDouble() < prob) {
88 k[0] = b1;
89 k[1] = b2;
90 k[2] = b3;
91 Put put = new Put(k);
92 put.setDurability(Durability.SKIP_WAL);
93 put.add(famAndQf[0], famAndQf[1], k);
94 table.put(put);
95 count++;
96 }
97 }
98 }
99 }
100 table.flushCommits();
101 table.close();
102 return count;
103 }
104
105 static int countCellSet(CellSetModel model) {
106 int count = 0;
107 Iterator<RowModel> rows = model.getRows().iterator();
108 while (rows.hasNext()) {
109 RowModel row = rows.next();
110 Iterator<CellModel> cells = row.getCells().iterator();
111 while (cells.hasNext()) {
112 cells.next();
113 count++;
114 }
115 }
116 return count;
117 }
118
119 private static int fullTableScan(ScannerModel model) throws IOException {
120 model.setBatch(100);
121 Response response = client.put("/" + TABLE + "/scanner",
122 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
123 assertEquals(response.getCode(), 201);
124 String scannerURI = response.getLocation();
125 assertNotNull(scannerURI);
126 int count = 0;
127 while (true) {
128 response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
129 assertTrue(response.getCode() == 200 || response.getCode() == 204);
130 if (response.getCode() == 200) {
131 assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
132 CellSetModel cellSet = new CellSetModel();
133 cellSet.getObjectFromMessage(response.getBody());
134 Iterator<RowModel> rows = cellSet.getRows().iterator();
135 while (rows.hasNext()) {
136 RowModel row = rows.next();
137 Iterator<CellModel> cells = row.getCells().iterator();
138 while (cells.hasNext()) {
139 cells.next();
140 count++;
141 }
142 }
143 } else {
144 break;
145 }
146 }
147
148 response = client.delete(scannerURI);
149 assertEquals(response.getCode(), 200);
150 return count;
151 }
152
153 @BeforeClass
154 public static void setUpBeforeClass() throws Exception {
155 conf = TEST_UTIL.getConfiguration();
156 TEST_UTIL.startMiniCluster();
157 REST_TEST_UTIL.startServletContainer(conf);
158 client = new Client(new Cluster().add("localhost",
159 REST_TEST_UTIL.getServletPort()));
160 context = JAXBContext.newInstance(
161 CellModel.class,
162 CellSetModel.class,
163 RowModel.class,
164 ScannerModel.class);
165 marshaller = context.createMarshaller();
166 unmarshaller = context.createUnmarshaller();
167 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
168 if (admin.tableExists(TABLE)) {
169 return;
170 }
171 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
172 htd.addFamily(new HColumnDescriptor(CFA));
173 htd.addFamily(new HColumnDescriptor(CFB));
174 admin.createTable(htd);
175 expectedRows1 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_1, 1.0);
176 expectedRows2 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_2, 0.5);
177 }
178
179 @AfterClass
180 public static void tearDownAfterClass() throws Exception {
181 REST_TEST_UTIL.shutdownServletContainer();
182 TEST_UTIL.shutdownMiniCluster();
183 }
184
185 @Test
186 public void testSimpleScannerXML() throws IOException, JAXBException {
187 final int BATCH_SIZE = 5;
188
189 ScannerModel model = new ScannerModel();
190 model.setBatch(BATCH_SIZE);
191 model.addColumn(Bytes.toBytes(COLUMN_1));
192 StringWriter writer = new StringWriter();
193 marshaller.marshal(model, writer);
194 byte[] body = Bytes.toBytes(writer.toString());
195
196
197 conf.set("hbase.rest.readonly", "true");
198 Response response = client.put("/" + TABLE + "/scanner",
199 Constants.MIMETYPE_XML, body);
200 assertEquals(response.getCode(), 403);
201 String scannerURI = response.getLocation();
202 assertNull(scannerURI);
203
204
205 conf.set("hbase.rest.readonly", "false");
206 response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML,
207 body);
208 assertEquals(response.getCode(), 201);
209 scannerURI = response.getLocation();
210 assertNotNull(scannerURI);
211
212
213 response = client.get(scannerURI, Constants.MIMETYPE_XML);
214 assertEquals(response.getCode(), 200);
215 assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
216 CellSetModel cellSet = (CellSetModel)
217 unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
218
219 assertEquals(countCellSet(cellSet), BATCH_SIZE);
220
221
222 conf.set("hbase.rest.readonly", "true");
223 response = client.delete(scannerURI);
224 assertEquals(response.getCode(), 403);
225
226
227 conf.set("hbase.rest.readonly", "false");
228 response = client.delete(scannerURI);
229 assertEquals(response.getCode(), 200);
230 }
231
232 @Test
233 public void testSimpleScannerPB() throws IOException {
234 final int BATCH_SIZE = 10;
235
236 ScannerModel model = new ScannerModel();
237 model.setBatch(BATCH_SIZE);
238 model.addColumn(Bytes.toBytes(COLUMN_1));
239
240
241 conf.set("hbase.rest.readonly", "true");
242 Response response = client.put("/" + TABLE + "/scanner",
243 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
244 assertEquals(response.getCode(), 403);
245 String scannerURI = response.getLocation();
246 assertNull(scannerURI);
247
248
249 conf.set("hbase.rest.readonly", "false");
250 response = client.put("/" + TABLE + "/scanner",
251 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
252 assertEquals(response.getCode(), 201);
253 scannerURI = response.getLocation();
254 assertNotNull(scannerURI);
255
256
257 response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
258 assertEquals(response.getCode(), 200);
259 assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
260 CellSetModel cellSet = new CellSetModel();
261 cellSet.getObjectFromMessage(response.getBody());
262
263 assertEquals(countCellSet(cellSet), BATCH_SIZE);
264
265
266 conf.set("hbase.rest.readonly", "true");
267 response = client.delete(scannerURI);
268 assertEquals(response.getCode(), 403);
269
270
271 conf.set("hbase.rest.readonly", "false");
272 response = client.delete(scannerURI);
273 assertEquals(response.getCode(), 200);
274 }
275
276 @Test
277 public void testSimpleScannerBinary() throws IOException {
278
279 ScannerModel model = new ScannerModel();
280 model.setBatch(1);
281 model.addColumn(Bytes.toBytes(COLUMN_1));
282
283
284 conf.set("hbase.rest.readonly", "true");
285 Response response = client.put("/" + TABLE + "/scanner",
286 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
287 assertEquals(response.getCode(), 403);
288 String scannerURI = response.getLocation();
289 assertNull(scannerURI);
290
291
292 conf.set("hbase.rest.readonly", "false");
293 response = client.put("/" + TABLE + "/scanner",
294 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
295 assertEquals(response.getCode(), 201);
296 scannerURI = response.getLocation();
297 assertNotNull(scannerURI);
298
299
300 response = client.get(scannerURI, Constants.MIMETYPE_BINARY);
301 assertEquals(response.getCode(), 200);
302 assertEquals(Constants.MIMETYPE_BINARY, response.getHeader("content-type"));
303
304 assertTrue(response.getBody().length > 0);
305
306 boolean foundRowHeader = false, foundColumnHeader = false,
307 foundTimestampHeader = false;
308 for (Header header: response.getHeaders()) {
309 if (header.getName().equals("X-Row")) {
310 foundRowHeader = true;
311 } else if (header.getName().equals("X-Column")) {
312 foundColumnHeader = true;
313 } else if (header.getName().equals("X-Timestamp")) {
314 foundTimestampHeader = true;
315 }
316 }
317 assertTrue(foundRowHeader);
318 assertTrue(foundColumnHeader);
319 assertTrue(foundTimestampHeader);
320
321
322 conf.set("hbase.rest.readonly", "true");
323 response = client.delete(scannerURI);
324 assertEquals(response.getCode(), 403);
325
326
327 conf.set("hbase.rest.readonly", "false");
328 response = client.delete(scannerURI);
329 assertEquals(response.getCode(), 200);
330 }
331
332 @Test
333 public void testFullTableScan() throws IOException {
334 ScannerModel model = new ScannerModel();
335 model.addColumn(Bytes.toBytes(COLUMN_1));
336 assertEquals(fullTableScan(model), expectedRows1);
337
338 model = new ScannerModel();
339 model.addColumn(Bytes.toBytes(COLUMN_2));
340 assertEquals(fullTableScan(model), expectedRows2);
341 }
342
343 @Test
344 public void testTableDoesNotExist() throws IOException, JAXBException {
345 ScannerModel model = new ScannerModel();
346 StringWriter writer = new StringWriter();
347 marshaller.marshal(model, writer);
348 byte[] body = Bytes.toBytes(writer.toString());
349 Response response = client.put("/" + NONEXISTENT_TABLE +
350 "/scanner", Constants.MIMETYPE_XML, body);
351 assertEquals(response.getCode(), 404);
352 }
353
354 }
355