1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.thrift2;
19
20 import static java.nio.ByteBuffer.wrap;
21 import static org.junit.Assert.assertArrayEquals;
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertNull;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.nio.ByteBuffer;
28 import java.security.PrivilegedExceptionAction;
29 import java.util.ArrayList;
30 import java.util.Collections;
31 import java.util.Comparator;
32 import java.util.List;
33
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.hbase.HBaseTestingUtility;
36 import org.apache.hadoop.hbase.HColumnDescriptor;
37 import org.apache.hadoop.hbase.HTableDescriptor;
38 import org.apache.hadoop.hbase.security.visibility.VisibilityTestUtil;
39 import org.apache.hadoop.hbase.testclassification.MediumTests;
40 import org.apache.hadoop.hbase.TableName;
41 import org.apache.hadoop.hbase.client.Admin;
42 import org.apache.hadoop.hbase.client.Connection;
43 import org.apache.hadoop.hbase.client.ConnectionFactory;
44 import org.apache.hadoop.hbase.client.HBaseAdmin;
45 import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
46 import org.apache.hadoop.hbase.security.User;
47 import org.apache.hadoop.hbase.security.UserProvider;
48 import org.apache.hadoop.hbase.security.visibility.ScanLabelGenerator;
49 import org.apache.hadoop.hbase.security.visibility.SimpleScanLabelGenerator;
50 import org.apache.hadoop.hbase.security.visibility.VisibilityClient;
51 import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
52 import org.apache.hadoop.hbase.security.visibility.VisibilityController;
53 import org.apache.hadoop.hbase.security.visibility.VisibilityUtils;
54 import org.apache.hadoop.hbase.thrift2.generated.TAppend;
55 import org.apache.hadoop.hbase.thrift2.generated.TAuthorization;
56 import org.apache.hadoop.hbase.thrift2.generated.TCellVisibility;
57 import org.apache.hadoop.hbase.thrift2.generated.TColumn;
58 import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement;
59 import org.apache.hadoop.hbase.thrift2.generated.TColumnValue;
60 import org.apache.hadoop.hbase.thrift2.generated.TGet;
61 import org.apache.hadoop.hbase.thrift2.generated.TIllegalArgument;
62 import org.apache.hadoop.hbase.thrift2.generated.TIncrement;
63 import org.apache.hadoop.hbase.thrift2.generated.TPut;
64 import org.apache.hadoop.hbase.thrift2.generated.TResult;
65 import org.apache.hadoop.hbase.thrift2.generated.TScan;
66 import org.apache.hadoop.hbase.util.Bytes;
67 import org.junit.AfterClass;
68 import org.junit.Assert;
69 import org.junit.Before;
70 import org.junit.BeforeClass;
71 import org.junit.Test;
72 import org.junit.experimental.categories.Category;
73
74 @Category(MediumTests.class)
75 public class TestThriftHBaseServiceHandlerWithLabels {
76
77 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
78
79
80 private static byte[] tableAname = Bytes.toBytes("tableA");
81 private static byte[] familyAname = Bytes.toBytes("familyA");
82 private static byte[] familyBname = Bytes.toBytes("familyB");
83 private static byte[] qualifierAname = Bytes.toBytes("qualifierA");
84 private static byte[] qualifierBname = Bytes.toBytes("qualifierB");
85 private static byte[] valueAname = Bytes.toBytes("valueA");
86 private static byte[] valueBname = Bytes.toBytes("valueB");
87 private static HColumnDescriptor[] families = new HColumnDescriptor[] {
88 new HColumnDescriptor(familyAname).setMaxVersions(3),
89 new HColumnDescriptor(familyBname).setMaxVersions(2) };
90
91 private final static String TOPSECRET = "topsecret";
92 private final static String PUBLIC = "public";
93 private final static String PRIVATE = "private";
94 private final static String CONFIDENTIAL = "confidential";
95 private final static String SECRET = "secret";
96 private static User SUPERUSER;
97
98 private static Configuration conf;
99
100 public void assertTColumnValuesEqual(List<TColumnValue> columnValuesA,
101 List<TColumnValue> columnValuesB) {
102 assertEquals(columnValuesA.size(), columnValuesB.size());
103 Comparator<TColumnValue> comparator = new Comparator<TColumnValue>() {
104 @Override
105 public int compare(TColumnValue o1, TColumnValue o2) {
106 return Bytes.compareTo(Bytes.add(o1.getFamily(), o1.getQualifier()),
107 Bytes.add(o2.getFamily(), o2.getQualifier()));
108 }
109 };
110 Collections.sort(columnValuesA, comparator);
111 Collections.sort(columnValuesB, comparator);
112
113 for (int i = 0; i < columnValuesA.size(); i++) {
114 TColumnValue a = columnValuesA.get(i);
115 TColumnValue b = columnValuesB.get(i);
116 assertArrayEquals(a.getFamily(), b.getFamily());
117 assertArrayEquals(a.getQualifier(), b.getQualifier());
118 assertArrayEquals(a.getValue(), b.getValue());
119 }
120 }
121
122 @BeforeClass
123 public static void beforeClass() throws Exception {
124 SUPERUSER = User.createUserForTesting(conf, "admin",
125 new String[] { "supergroup" });
126 conf = UTIL.getConfiguration();
127 conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS,
128 SimpleScanLabelGenerator.class, ScanLabelGenerator.class);
129 conf.set("hbase.superuser", SUPERUSER.getShortName());
130 VisibilityTestUtil.enableVisiblityLabels(conf);
131 UTIL.startMiniCluster(1);
132
133 UTIL.waitTableEnabled(VisibilityConstants.LABELS_TABLE_NAME.getName(), 50000);
134 createLabels();
135 Admin admin = new HBaseAdmin(UTIL.getConfiguration());
136 HTableDescriptor tableDescriptor = new HTableDescriptor(
137 TableName.valueOf(tableAname));
138 for (HColumnDescriptor family : families) {
139 tableDescriptor.addFamily(family);
140 }
141 admin.createTable(tableDescriptor);
142 admin.close();
143 setAuths();
144 }
145
146 private static void createLabels() throws IOException, InterruptedException {
147 PrivilegedExceptionAction<VisibilityLabelsResponse> action = new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
148 public VisibilityLabelsResponse run() throws Exception {
149 String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
150 try (Connection conn = ConnectionFactory.createConnection(conf)) {
151 VisibilityClient.addLabels(conn, labels);
152 } catch (Throwable t) {
153 throw new IOException(t);
154 }
155 return null;
156 }
157 };
158 SUPERUSER.runAs(action);
159 }
160
161 private static void setAuths() throws IOException {
162 String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
163 try {
164 VisibilityClient.setAuths(UTIL.getConnection(), labels, User.getCurrent().getShortName());
165 } catch (Throwable t) {
166 throw new IOException(t);
167 }
168 }
169
170 @AfterClass
171 public static void afterClass() throws Exception {
172 UTIL.shutdownMiniCluster();
173 }
174
175 @Before
176 public void setup() throws Exception {
177
178 }
179
180 private ThriftHBaseServiceHandler createHandler() throws IOException {
181 return new ThriftHBaseServiceHandler(conf, UserProvider.instantiate(conf));
182 }
183
184 @Test
185 public void testScanWithVisibilityLabels() throws Exception {
186 ThriftHBaseServiceHandler handler = createHandler();
187 ByteBuffer table = wrap(tableAname);
188
189
190 TColumnValue columnValue = new TColumnValue(wrap(familyAname),
191 wrap(qualifierAname), wrap(valueAname));
192 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
193 columnValues.add(columnValue);
194 for (int i = 0; i < 10; i++) {
195 TPut put = new TPut(wrap(("testScan" + i).getBytes()), columnValues);
196 if (i == 5) {
197 put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
198 } else {
199 put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET
200 + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
201 }
202 handler.put(table, put);
203 }
204
205
206 TScan scan = new TScan();
207 List<TColumn> columns = new ArrayList<TColumn>();
208 TColumn column = new TColumn();
209 column.setFamily(familyAname);
210 column.setQualifier(qualifierAname);
211 columns.add(column);
212 scan.setColumns(columns);
213 scan.setStartRow("testScan".getBytes());
214 scan.setStopRow("testScan\uffff".getBytes());
215
216 TAuthorization tauth = new TAuthorization();
217 List<String> labels = new ArrayList<String>();
218 labels.add(SECRET);
219 labels.add(PRIVATE);
220 tauth.setLabels(labels);
221 scan.setAuthorizations(tauth);
222
223 int scanId = handler.openScanner(table, scan);
224 List<TResult> results = handler.getScannerRows(scanId, 10);
225 assertEquals(9, results.size());
226 Assert.assertFalse(Bytes.equals(results.get(5).getRow(),
227 ("testScan" + 5).getBytes()));
228 for (int i = 0; i < 9; i++) {
229 if (i < 5) {
230 assertArrayEquals(("testScan" + i).getBytes(), results.get(i).getRow());
231 } else if (i == 5) {
232 continue;
233 } else {
234 assertArrayEquals(("testScan" + (i + 1)).getBytes(), results.get(i)
235 .getRow());
236 }
237 }
238
239
240 results = handler.getScannerRows(scanId, 9);
241 assertEquals(0, results.size());
242
243
244 handler.closeScanner(scanId);
245 try {
246 handler.getScannerRows(scanId, 9);
247 fail("Scanner id should be invalid");
248 } catch (TIllegalArgument e) {
249 }
250 }
251
252 @Test
253 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="UC_USELESS_CONDITION",
254 justification="Intended")
255 public void testGetScannerResultsWithAuthorizations() throws Exception {
256 ThriftHBaseServiceHandler handler = createHandler();
257 ByteBuffer table = wrap(tableAname);
258
259
260 TColumnValue columnValue = new TColumnValue(wrap(familyAname),
261 wrap(qualifierAname), wrap(valueAname));
262 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
263 columnValues.add(columnValue);
264 for (int i = 0; i < 20; i++) {
265 TPut put = new TPut(
266 wrap(("testGetScannerResults" + pad(i, (byte) 2)).getBytes()),
267 columnValues);
268 if (i == 3) {
269 put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
270 } else {
271 put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET
272 + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
273 }
274 handler.put(table, put);
275 }
276
277
278 TScan scan = new TScan();
279 List<TColumn> columns = new ArrayList<TColumn>();
280 TColumn column = new TColumn();
281 column.setFamily(familyAname);
282 column.setQualifier(qualifierAname);
283 columns.add(column);
284 scan.setColumns(columns);
285 scan.setStartRow("testGetScannerResults".getBytes());
286
287
288 scan.setStopRow("testGetScannerResults05".getBytes());
289 TAuthorization tauth = new TAuthorization();
290 List<String> labels = new ArrayList<String>();
291 labels.add(SECRET);
292 labels.add(PRIVATE);
293 tauth.setLabels(labels);
294 scan.setAuthorizations(tauth);
295 List<TResult> results = handler.getScannerResults(table, scan, 5);
296 assertEquals(4, results.size());
297 for (int i = 0; i < 4; i++) {
298 if (i < 3) {
299 assertArrayEquals(
300 ("testGetScannerResults" + pad(i, (byte) 2)).getBytes(),
301 results.get(i).getRow());
302 } else if (i == 3) {
303 continue;
304 } else {
305 assertArrayEquals(
306 ("testGetScannerResults" + pad(i + 1, (byte) 2)).getBytes(), results
307 .get(i).getRow());
308 }
309 }
310 }
311
312 @Test
313 public void testGetsWithLabels() throws Exception {
314 ThriftHBaseServiceHandler handler = createHandler();
315 byte[] rowName = "testPutGet".getBytes();
316 ByteBuffer table = wrap(tableAname);
317
318 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
319 columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
320 wrap(valueAname)));
321 columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname),
322 wrap(valueBname)));
323 TPut put = new TPut(wrap(rowName), columnValues);
324
325 put.setColumnValues(columnValues);
326 put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET + "|"
327 + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
328 handler.put(table, put);
329 TGet get = new TGet(wrap(rowName));
330 TAuthorization tauth = new TAuthorization();
331 List<String> labels = new ArrayList<String>();
332 labels.add(SECRET);
333 labels.add(PRIVATE);
334 tauth.setLabels(labels);
335 get.setAuthorizations(tauth);
336 TResult result = handler.get(table, get);
337 assertArrayEquals(rowName, result.getRow());
338 List<TColumnValue> returnedColumnValues = result.getColumnValues();
339 assertTColumnValuesEqual(columnValues, returnedColumnValues);
340 }
341
342 @Test
343 public void testIncrementWithTags() throws Exception {
344 ThriftHBaseServiceHandler handler = createHandler();
345 byte[] rowName = "testIncrementWithTags".getBytes();
346 ByteBuffer table = wrap(tableAname);
347
348 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
349 columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
350 wrap(Bytes.toBytes(1L))));
351 TPut put = new TPut(wrap(rowName), columnValues);
352 put.setColumnValues(columnValues);
353 put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
354 handler.put(table, put);
355
356 List<TColumnIncrement> incrementColumns = new ArrayList<TColumnIncrement>();
357 incrementColumns.add(new TColumnIncrement(wrap(familyAname),
358 wrap(qualifierAname)));
359 TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
360 increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
361 handler.increment(table, increment);
362
363 TGet get = new TGet(wrap(rowName));
364 TAuthorization tauth = new TAuthorization();
365 List<String> labels = new ArrayList<String>();
366 labels.add(SECRET);
367 tauth.setLabels(labels);
368 get.setAuthorizations(tauth);
369 TResult result = handler.get(table, get);
370
371 assertArrayEquals(rowName, result.getRow());
372 assertEquals(1, result.getColumnValuesSize());
373 TColumnValue columnValue = result.getColumnValues().get(0);
374 assertArrayEquals(Bytes.toBytes(2L), columnValue.getValue());
375 }
376
377 @Test
378 public void testIncrementWithTagsWithNotMatchLabels() throws Exception {
379 ThriftHBaseServiceHandler handler = createHandler();
380 byte[] rowName = "testIncrementWithTagsWithNotMatchLabels".getBytes();
381 ByteBuffer table = wrap(tableAname);
382
383 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
384 columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
385 wrap(Bytes.toBytes(1L))));
386 TPut put = new TPut(wrap(rowName), columnValues);
387 put.setColumnValues(columnValues);
388 put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
389 handler.put(table, put);
390
391 List<TColumnIncrement> incrementColumns = new ArrayList<TColumnIncrement>();
392 incrementColumns.add(new TColumnIncrement(wrap(familyAname),
393 wrap(qualifierAname)));
394 TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
395 increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
396 handler.increment(table, increment);
397
398 TGet get = new TGet(wrap(rowName));
399 TAuthorization tauth = new TAuthorization();
400 List<String> labels = new ArrayList<String>();
401 labels.add(PUBLIC);
402 tauth.setLabels(labels);
403 get.setAuthorizations(tauth);
404 TResult result = handler.get(table, get);
405 assertNull(result.getRow());
406 }
407
408 @Test
409 public void testAppend() throws Exception {
410 ThriftHBaseServiceHandler handler = createHandler();
411 byte[] rowName = "testAppend".getBytes();
412 ByteBuffer table = wrap(tableAname);
413 byte[] v1 = Bytes.toBytes(1L);
414 byte[] v2 = Bytes.toBytes(5L);
415 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
416 columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
417 wrap(Bytes.toBytes(1L))));
418 TPut put = new TPut(wrap(rowName), columnValues);
419 put.setColumnValues(columnValues);
420 put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
421 handler.put(table, put);
422
423 List<TColumnValue> appendColumns = new ArrayList<TColumnValue>();
424 appendColumns.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
425 wrap(v2)));
426 TAppend append = new TAppend(wrap(rowName), appendColumns);
427 append.setCellVisibility(new TCellVisibility().setExpression(SECRET));
428 handler.append(table, append);
429
430 TGet get = new TGet(wrap(rowName));
431 TAuthorization tauth = new TAuthorization();
432 List<String> labels = new ArrayList<String>();
433 labels.add(SECRET);
434 tauth.setLabels(labels);
435 get.setAuthorizations(tauth);
436 TResult result = handler.get(table, get);
437
438 assertArrayEquals(rowName, result.getRow());
439 assertEquals(1, result.getColumnValuesSize());
440 TColumnValue columnValue = result.getColumnValues().get(0);
441 assertArrayEquals(Bytes.add(v1, v2), columnValue.getValue());
442 }
443
444
445
446
447
448
449
450
451
452
453 private String pad(int n, byte pad) {
454 String res = Integer.toString(n);
455 while (res.length() < pad)
456 res = "0" + res;
457 return res;
458 }
459 }