1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.thrift2;
19
20 import static java.nio.ByteBuffer.wrap;
21 import static org.junit.Assert.assertArrayEquals;
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertNull;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.nio.ByteBuffer;
28 import java.security.PrivilegedExceptionAction;
29 import java.util.ArrayList;
30 import java.util.Collections;
31 import java.util.Comparator;
32 import java.util.List;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.hbase.HBaseTestingUtility;
38 import org.apache.hadoop.hbase.HColumnDescriptor;
39 import org.apache.hadoop.hbase.HTableDescriptor;
40 import org.apache.hadoop.hbase.testclassification.MediumTests;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.hbase.client.HBaseAdmin;
43 import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
44 import org.apache.hadoop.hbase.security.User;
45 import org.apache.hadoop.hbase.security.UserProvider;
46 import org.apache.hadoop.hbase.security.visibility.ScanLabelGenerator;
47 import org.apache.hadoop.hbase.security.visibility.SimpleScanLabelGenerator;
48 import org.apache.hadoop.hbase.security.visibility.VisibilityClient;
49 import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
50 import org.apache.hadoop.hbase.security.visibility.VisibilityController;
51 import org.apache.hadoop.hbase.security.visibility.VisibilityUtils;
52 import org.apache.hadoop.hbase.thrift2.generated.TAppend;
53 import org.apache.hadoop.hbase.thrift2.generated.TAuthorization;
54 import org.apache.hadoop.hbase.thrift2.generated.TCellVisibility;
55 import org.apache.hadoop.hbase.thrift2.generated.TColumn;
56 import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement;
57 import org.apache.hadoop.hbase.thrift2.generated.TColumnValue;
58 import org.apache.hadoop.hbase.thrift2.generated.TGet;
59 import org.apache.hadoop.hbase.thrift2.generated.TIllegalArgument;
60 import org.apache.hadoop.hbase.thrift2.generated.TIncrement;
61 import org.apache.hadoop.hbase.thrift2.generated.TPut;
62 import org.apache.hadoop.hbase.thrift2.generated.TResult;
63 import org.apache.hadoop.hbase.thrift2.generated.TScan;
64 import org.apache.hadoop.hbase.util.Bytes;
65 import org.junit.AfterClass;
66 import org.junit.Assert;
67 import org.junit.Before;
68 import org.junit.BeforeClass;
69 import org.junit.Test;
70 import org.junit.experimental.categories.Category;
71
72 @Category(MediumTests.class)
73 public class TestThriftHBaseServiceHandlerWithLabels {
74
75 public static final Log LOG = LogFactory
76 .getLog(TestThriftHBaseServiceHandlerWithLabels.class);
77 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
78
79
80 private static byte[] tableAname = Bytes.toBytes("tableA");
81 private static byte[] familyAname = Bytes.toBytes("familyA");
82 private static byte[] familyBname = Bytes.toBytes("familyB");
83 private static byte[] qualifierAname = Bytes.toBytes("qualifierA");
84 private static byte[] qualifierBname = Bytes.toBytes("qualifierB");
85 private static byte[] valueAname = Bytes.toBytes("valueA");
86 private static byte[] valueBname = Bytes.toBytes("valueB");
87 private static HColumnDescriptor[] families = new HColumnDescriptor[] {
88 new HColumnDescriptor(familyAname).setMaxVersions(3),
89 new HColumnDescriptor(familyBname).setMaxVersions(2) };
90
91 private final static String TOPSECRET = "topsecret";
92 private final static String PUBLIC = "public";
93 private final static String PRIVATE = "private";
94 private final static String CONFIDENTIAL = "confidential";
95 private final static String SECRET = "secret";
96 private static User SUPERUSER;
97
98 private static Configuration conf;
99
100 public void assertTColumnValuesEqual(List<TColumnValue> columnValuesA,
101 List<TColumnValue> columnValuesB) {
102 assertEquals(columnValuesA.size(), columnValuesB.size());
103 Comparator<TColumnValue> comparator = new Comparator<TColumnValue>() {
104 @Override
105 public int compare(TColumnValue o1, TColumnValue o2) {
106 return Bytes.compareTo(Bytes.add(o1.getFamily(), o1.getQualifier()),
107 Bytes.add(o2.getFamily(), o2.getQualifier()));
108 }
109 };
110 Collections.sort(columnValuesA, comparator);
111 Collections.sort(columnValuesB, comparator);
112
113 for (int i = 0; i < columnValuesA.size(); i++) {
114 TColumnValue a = columnValuesA.get(i);
115 TColumnValue b = columnValuesB.get(i);
116 assertArrayEquals(a.getFamily(), b.getFamily());
117 assertArrayEquals(a.getQualifier(), b.getQualifier());
118 assertArrayEquals(a.getValue(), b.getValue());
119 }
120 }
121
122 @BeforeClass
123 public static void beforeClass() throws Exception {
124 SUPERUSER = User.createUserForTesting(conf, "admin",
125 new String[] { "supergroup" });
126 conf = UTIL.getConfiguration();
127 conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS,
128 SimpleScanLabelGenerator.class, ScanLabelGenerator.class);
129 conf.set("hbase.superuser", SUPERUSER.getShortName());
130 conf.set("hbase.coprocessor.master.classes",
131 VisibilityController.class.getName());
132 conf.set("hbase.coprocessor.region.classes",
133 VisibilityController.class.getName());
134 conf.setInt("hfile.format.version", 3);
135 UTIL.startMiniCluster(1);
136
137 UTIL.waitTableEnabled(VisibilityConstants.LABELS_TABLE_NAME.getName(), 50000);
138 createLabels();
139 HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration());
140 HTableDescriptor tableDescriptor = new HTableDescriptor(
141 TableName.valueOf(tableAname));
142 for (HColumnDescriptor family : families) {
143 tableDescriptor.addFamily(family);
144 }
145 admin.createTable(tableDescriptor);
146 admin.close();
147 setAuths();
148 }
149
150 private static void createLabels() throws IOException, InterruptedException {
151 PrivilegedExceptionAction<VisibilityLabelsResponse> action = new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
152 public VisibilityLabelsResponse run() throws Exception {
153 String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
154 try {
155 VisibilityClient.addLabels(conf, labels);
156 } catch (Throwable t) {
157 throw new IOException(t);
158 }
159 return null;
160 }
161 };
162 SUPERUSER.runAs(action);
163 }
164
165 private static void setAuths() throws IOException {
166 String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
167 try {
168 VisibilityClient.setAuths(conf, labels, User.getCurrent().getShortName());
169 } catch (Throwable t) {
170 throw new IOException(t);
171 }
172 }
173
174 @AfterClass
175 public static void afterClass() throws Exception {
176 UTIL.shutdownMiniCluster();
177 }
178
179 @Before
180 public void setup() throws Exception {
181
182 }
183
184 private ThriftHBaseServiceHandler createHandler() throws IOException {
185 return new ThriftHBaseServiceHandler(conf, UserProvider.instantiate(conf));
186 }
187
188 @Test
189 public void testScanWithVisibilityLabels() throws Exception {
190 ThriftHBaseServiceHandler handler = createHandler();
191 ByteBuffer table = wrap(tableAname);
192
193
194 TColumnValue columnValue = new TColumnValue(wrap(familyAname),
195 wrap(qualifierAname), wrap(valueAname));
196 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
197 columnValues.add(columnValue);
198 for (int i = 0; i < 10; i++) {
199 TPut put = new TPut(wrap(("testScan" + i).getBytes()), columnValues);
200 if (i == 5) {
201 put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
202 } else {
203 put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET
204 + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
205 }
206 handler.put(table, put);
207 }
208
209
210 TScan scan = new TScan();
211 List<TColumn> columns = new ArrayList<TColumn>();
212 TColumn column = new TColumn();
213 column.setFamily(familyAname);
214 column.setQualifier(qualifierAname);
215 columns.add(column);
216 scan.setColumns(columns);
217 scan.setStartRow("testScan".getBytes());
218 scan.setStopRow("testScan\uffff".getBytes());
219
220 TAuthorization tauth = new TAuthorization();
221 List<String> labels = new ArrayList<String>();
222 labels.add(SECRET);
223 labels.add(PRIVATE);
224 tauth.setLabels(labels);
225 scan.setAuthorizations(tauth);
226
227 int scanId = handler.openScanner(table, scan);
228 List<TResult> results = handler.getScannerRows(scanId, 10);
229 assertEquals(9, results.size());
230 Assert.assertFalse(Bytes.equals(results.get(5).getRow(),
231 ("testScan" + 5).getBytes()));
232 for (int i = 0; i < 9; i++) {
233 if (i < 5) {
234 assertArrayEquals(("testScan" + i).getBytes(), results.get(i).getRow());
235 } else if (i == 5) {
236 continue;
237 } else {
238 assertArrayEquals(("testScan" + (i + 1)).getBytes(), results.get(i)
239 .getRow());
240 }
241 }
242
243
244 results = handler.getScannerRows(scanId, 9);
245 assertEquals(0, results.size());
246
247
248 handler.closeScanner(scanId);
249 try {
250 handler.getScannerRows(scanId, 9);
251 fail("Scanner id should be invalid");
252 } catch (TIllegalArgument e) {
253 }
254 }
255
256 @Test
257 public void testGetScannerResultsWithAuthorizations() throws Exception {
258 ThriftHBaseServiceHandler handler = createHandler();
259 ByteBuffer table = wrap(tableAname);
260
261
262 TColumnValue columnValue = new TColumnValue(wrap(familyAname),
263 wrap(qualifierAname), wrap(valueAname));
264 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
265 columnValues.add(columnValue);
266 for (int i = 0; i < 20; i++) {
267 TPut put = new TPut(
268 wrap(("testGetScannerResults" + pad(i, (byte) 2)).getBytes()),
269 columnValues);
270 if (i == 3) {
271 put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
272 } else {
273 put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET
274 + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
275 }
276 handler.put(table, put);
277 }
278
279
280 TScan scan = new TScan();
281 List<TColumn> columns = new ArrayList<TColumn>();
282 TColumn column = new TColumn();
283 column.setFamily(familyAname);
284 column.setQualifier(qualifierAname);
285 columns.add(column);
286 scan.setColumns(columns);
287 scan.setStartRow("testGetScannerResults".getBytes());
288
289
290 scan.setStopRow("testGetScannerResults05".getBytes());
291 TAuthorization tauth = new TAuthorization();
292 List<String> labels = new ArrayList<String>();
293 labels.add(SECRET);
294 labels.add(PRIVATE);
295 tauth.setLabels(labels);
296 scan.setAuthorizations(tauth);
297 List<TResult> results = handler.getScannerResults(table, scan, 5);
298 assertEquals(4, results.size());
299 for (int i = 0; i < 4; i++) {
300 if (i < 3) {
301 assertArrayEquals(
302 ("testGetScannerResults" + pad(i, (byte) 2)).getBytes(),
303 results.get(i).getRow());
304 } else if (i == 3) {
305 continue;
306 } else {
307 assertArrayEquals(
308 ("testGetScannerResults" + pad(i + 1, (byte) 2)).getBytes(), results
309 .get(i).getRow());
310 }
311 }
312 }
313
314 @Test
315 public void testGetsWithLabels() throws Exception {
316 ThriftHBaseServiceHandler handler = createHandler();
317 byte[] rowName = "testPutGet".getBytes();
318 ByteBuffer table = wrap(tableAname);
319
320 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
321 columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
322 wrap(valueAname)));
323 columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname),
324 wrap(valueBname)));
325 TPut put = new TPut(wrap(rowName), columnValues);
326
327 put.setColumnValues(columnValues);
328 put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET + "|"
329 + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
330 handler.put(table, put);
331 TGet get = new TGet(wrap(rowName));
332 TAuthorization tauth = new TAuthorization();
333 List<String> labels = new ArrayList<String>();
334 labels.add(SECRET);
335 labels.add(PRIVATE);
336 tauth.setLabels(labels);
337 get.setAuthorizations(tauth);
338 TResult result = handler.get(table, get);
339 assertArrayEquals(rowName, result.getRow());
340 List<TColumnValue> returnedColumnValues = result.getColumnValues();
341 assertTColumnValuesEqual(columnValues, returnedColumnValues);
342 }
343
344 @Test
345 public void testIncrementWithTags() throws Exception {
346 ThriftHBaseServiceHandler handler = createHandler();
347 byte[] rowName = "testIncrementWithTags".getBytes();
348 ByteBuffer table = wrap(tableAname);
349
350 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
351 columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
352 wrap(Bytes.toBytes(1L))));
353 TPut put = new TPut(wrap(rowName), columnValues);
354 put.setColumnValues(columnValues);
355 put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
356 handler.put(table, put);
357
358 List<TColumnIncrement> incrementColumns = new ArrayList<TColumnIncrement>();
359 incrementColumns.add(new TColumnIncrement(wrap(familyAname),
360 wrap(qualifierAname)));
361 TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
362 increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
363 handler.increment(table, increment);
364
365 TGet get = new TGet(wrap(rowName));
366 TAuthorization tauth = new TAuthorization();
367 List<String> labels = new ArrayList<String>();
368 labels.add(SECRET);
369 tauth.setLabels(labels);
370 get.setAuthorizations(tauth);
371 TResult result = handler.get(table, get);
372
373 assertArrayEquals(rowName, result.getRow());
374 assertEquals(1, result.getColumnValuesSize());
375 TColumnValue columnValue = result.getColumnValues().get(0);
376 assertArrayEquals(Bytes.toBytes(2L), columnValue.getValue());
377 }
378
379 @Test
380 public void testIncrementWithTagsWithNotMatchLabels() throws Exception {
381 ThriftHBaseServiceHandler handler = createHandler();
382 byte[] rowName = "testIncrementWithTagsWithNotMatchLabels".getBytes();
383 ByteBuffer table = wrap(tableAname);
384
385 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
386 columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
387 wrap(Bytes.toBytes(1L))));
388 TPut put = new TPut(wrap(rowName), columnValues);
389 put.setColumnValues(columnValues);
390 put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
391 handler.put(table, put);
392
393 List<TColumnIncrement> incrementColumns = new ArrayList<TColumnIncrement>();
394 incrementColumns.add(new TColumnIncrement(wrap(familyAname),
395 wrap(qualifierAname)));
396 TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
397 increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
398 handler.increment(table, increment);
399
400 TGet get = new TGet(wrap(rowName));
401 TAuthorization tauth = new TAuthorization();
402 List<String> labels = new ArrayList<String>();
403 labels.add(PUBLIC);
404 tauth.setLabels(labels);
405 get.setAuthorizations(tauth);
406 TResult result = handler.get(table, get);
407 assertNull(result.getRow());
408 }
409
410 @Test
411 public void testAppend() throws Exception {
412 ThriftHBaseServiceHandler handler = createHandler();
413 byte[] rowName = "testAppend".getBytes();
414 ByteBuffer table = wrap(tableAname);
415 byte[] v1 = Bytes.toBytes(1L);
416 byte[] v2 = Bytes.toBytes(5L);
417 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
418 columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
419 wrap(Bytes.toBytes(1L))));
420 TPut put = new TPut(wrap(rowName), columnValues);
421 put.setColumnValues(columnValues);
422 put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
423 handler.put(table, put);
424
425 List<TColumnValue> appendColumns = new ArrayList<TColumnValue>();
426 appendColumns.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
427 wrap(v2)));
428 TAppend append = new TAppend(wrap(rowName), appendColumns);
429 append.setCellVisibility(new TCellVisibility().setExpression(SECRET));
430 handler.append(table, append);
431
432 TGet get = new TGet(wrap(rowName));
433 TAuthorization tauth = new TAuthorization();
434 List<String> labels = new ArrayList<String>();
435 labels.add(SECRET);
436 tauth.setLabels(labels);
437 get.setAuthorizations(tauth);
438 TResult result = handler.get(table, get);
439
440 assertArrayEquals(rowName, result.getRow());
441 assertEquals(1, result.getColumnValuesSize());
442 TColumnValue columnValue = result.getColumnValues().get(0);
443 assertArrayEquals(Bytes.add(v1, v2), columnValue.getValue());
444 }
445
446
447
448
449
450
451
452
453
454
455 private String pad(int n, byte pad) {
456 String res = Integer.toString(n);
457 while (res.length() < pad)
458 res = "0" + res;
459 return res;
460 }
461 }