1
2
3
4
5
6
7
8
9
10
11 package org.apache.hadoop.hbase.namespace;
12
13 import static org.junit.Assert.assertEquals;
14 import static org.junit.Assert.assertFalse;
15 import static org.junit.Assert.assertNotNull;
16 import static org.junit.Assert.assertNull;
17 import static org.junit.Assert.assertTrue;
18 import static org.junit.Assert.fail;
19
20 import java.io.IOException;
21 import java.util.Collections;
22 import java.util.List;
23 import java.util.Set;
24 import java.util.concurrent.CountDownLatch;
25
26 import org.apache.commons.lang.StringUtils;
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.FileSystem;
31 import org.apache.hadoop.fs.Path;
32 import org.apache.hadoop.hbase.Coprocessor;
33 import org.apache.hadoop.hbase.CoprocessorEnvironment;
34 import org.apache.hadoop.hbase.DoNotRetryIOException;
35 import org.apache.hadoop.hbase.HBaseTestingUtility;
36 import org.apache.hadoop.hbase.HColumnDescriptor;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.HRegionInfo;
39 import org.apache.hadoop.hbase.HTableDescriptor;
40 import org.apache.hadoop.hbase.MiniHBaseCluster;
41 import org.apache.hadoop.hbase.NamespaceDescriptor;
42 import org.apache.hadoop.hbase.TableName;
43 import org.apache.hadoop.hbase.Waiter;
44 import org.apache.hadoop.hbase.client.Connection;
45 import org.apache.hadoop.hbase.client.ConnectionFactory;
46 import org.apache.hadoop.hbase.client.HBaseAdmin;
47 import org.apache.hadoop.hbase.client.HTable;
48 import org.apache.hadoop.hbase.client.Mutation;
49 import org.apache.hadoop.hbase.client.RegionLocator;
50 import org.apache.hadoop.hbase.client.Table;
51 import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver;
52 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
53 import org.apache.hadoop.hbase.coprocessor.BaseRegionServerObserver;
54 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
55 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
56 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
57 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
58 import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
59 import org.apache.hadoop.hbase.coprocessor.RegionServerObserver;
60 import org.apache.hadoop.hbase.mapreduce.TableInputFormatBase;
61 import org.apache.hadoop.hbase.master.HMaster;
62 import org.apache.hadoop.hbase.master.RegionState;
63 import org.apache.hadoop.hbase.master.RegionStates;
64 import org.apache.hadoop.hbase.master.TableNamespaceManager;
65 import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
66 import org.apache.hadoop.hbase.quotas.QuotaExceededException;
67 import org.apache.hadoop.hbase.quotas.QuotaUtil;
68 import org.apache.hadoop.hbase.regionserver.HRegion;
69 import org.apache.hadoop.hbase.regionserver.HRegionServer;
70 import org.apache.hadoop.hbase.regionserver.Region;
71 import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost;
72 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
73 import org.apache.hadoop.hbase.testclassification.MediumTests;
74 import org.apache.hadoop.hbase.util.Bytes;
75 import org.apache.hadoop.hbase.util.FSUtils;
76 import org.apache.zookeeper.KeeperException;
77 import org.junit.After;
78 import org.junit.AfterClass;
79 import org.junit.BeforeClass;
80 import org.junit.Test;
81 import org.junit.experimental.categories.Category;
82
83 import com.google.common.collect.Sets;
84
85 @Category(MediumTests.class)
86 public class TestNamespaceAuditor {
87 private static final Log LOG = LogFactory.getLog(TestNamespaceAuditor.class);
88 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
89 private static HBaseAdmin ADMIN;
90 private String prefix = "TestNamespaceAuditor";
91
92 @BeforeClass
93 public static void before() throws Exception {
94 UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
95 setupOnce();
96 }
97
98 public static void setupOnce() throws Exception, IOException {
99 Configuration conf = UTIL.getConfiguration();
100 conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
101 conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, CustomObserver.class.getName());
102 conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MasterSyncObserver.class.getName());
103 conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
104 conf.setClass("hbase.coprocessor.regionserver.classes", CPRegionServerObserver.class,
105 RegionServerObserver.class);
106 UTIL.startMiniCluster(1, 1);
107 waitForQuotaEnabled();
108 ADMIN = UTIL.getHBaseAdmin();
109 }
110
111 @AfterClass
112 public static void tearDown() throws Exception {
113 UTIL.shutdownMiniCluster();
114 }
115
116 @After
117 public void cleanup() throws Exception, KeeperException {
118 for (HTableDescriptor table : ADMIN.listTables()) {
119 ADMIN.disableTable(table.getTableName());
120 deleteTable(table.getTableName());
121 }
122 for (NamespaceDescriptor ns : ADMIN.listNamespaceDescriptors()) {
123 if (ns.getName().startsWith(prefix)) {
124 ADMIN.deleteNamespace(ns.getName());
125 }
126 }
127 assertTrue("Quota manager not enabled", UTIL.getHBaseCluster().getMaster()
128 .getMasterQuotaManager().isQuotaEnabled());
129 }
130
131 @Test(timeout = 60000)
132 public void testTableOperations() throws Exception {
133 String nsp = prefix + "_np2";
134 NamespaceDescriptor nspDesc =
135 NamespaceDescriptor.create(nsp)
136 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5")
137 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
138 ADMIN.createNamespace(nspDesc);
139 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
140 assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
141 HTableDescriptor tableDescOne =
142 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
143 HTableDescriptor tableDescTwo =
144 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
145 HTableDescriptor tableDescThree =
146 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table3"));
147 ADMIN.createTable(tableDescOne);
148 boolean constraintViolated = false;
149 try {
150 ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
151 } catch (Exception exp) {
152 assertTrue(exp instanceof IOException);
153 constraintViolated = true;
154 } finally {
155 assertTrue("Constraint not violated for table " + tableDescTwo.getTableName(),
156 constraintViolated);
157 }
158 ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
159 NamespaceTableAndRegionInfo nspState = getQuotaManager().getState(nsp);
160 assertNotNull(nspState);
161 assertTrue(nspState.getTables().size() == 2);
162 assertTrue(nspState.getRegionCount() == 5);
163 constraintViolated = false;
164 try {
165 ADMIN.createTable(tableDescThree);
166 } catch (Exception exp) {
167 assertTrue(exp instanceof IOException);
168 constraintViolated = true;
169 } finally {
170 assertTrue("Constraint not violated for table " + tableDescThree.getTableName(),
171 constraintViolated);
172 }
173 }
174
175 @Test
176 public void testValidQuotas() throws Exception {
177 boolean exceptionCaught = false;
178 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
179 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
180 NamespaceDescriptor nspDesc =
181 NamespaceDescriptor.create(prefix + "vq1")
182 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "hihdufh")
183 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
184 try {
185 ADMIN.createNamespace(nspDesc);
186 } catch (Exception exp) {
187 LOG.warn(exp);
188 exceptionCaught = true;
189 } finally {
190 assertTrue(exceptionCaught);
191 assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
192 }
193 nspDesc =
194 NamespaceDescriptor.create(prefix + "vq2")
195 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "-456")
196 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
197 try {
198 ADMIN.createNamespace(nspDesc);
199 } catch (Exception exp) {
200 LOG.warn(exp);
201 exceptionCaught = true;
202 } finally {
203 assertTrue(exceptionCaught);
204 assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
205 }
206 nspDesc =
207 NamespaceDescriptor.create(prefix + "vq3")
208 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
209 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "sciigd").build();
210 try {
211 ADMIN.createNamespace(nspDesc);
212 } catch (Exception exp) {
213 LOG.warn(exp);
214 exceptionCaught = true;
215 } finally {
216 assertTrue(exceptionCaught);
217 assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
218 }
219 nspDesc =
220 NamespaceDescriptor.create(prefix + "vq4")
221 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
222 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "-1500").build();
223 try {
224 ADMIN.createNamespace(nspDesc);
225 } catch (Exception exp) {
226 LOG.warn(exp);
227 exceptionCaught = true;
228 } finally {
229 assertTrue(exceptionCaught);
230 assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
231 }
232 }
233
234 @Test
235 public void testDeleteTable() throws Exception {
236 String namespace = prefix + "_dummy";
237 NamespaceDescriptor nspDesc =
238 NamespaceDescriptor.create(namespace)
239 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "100")
240 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "3").build();
241 ADMIN.createNamespace(nspDesc);
242 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(namespace));
243 NamespaceTableAndRegionInfo stateInfo = getNamespaceState(nspDesc.getName());
244 assertNotNull("Namespace state found null for " + namespace, stateInfo);
245 HTableDescriptor tableDescOne =
246 new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table1"));
247 HTableDescriptor tableDescTwo =
248 new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table2"));
249 ADMIN.createTable(tableDescOne);
250 ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
251 stateInfo = getNamespaceState(nspDesc.getName());
252 assertNotNull("Namespace state found to be null.", stateInfo);
253 assertEquals(2, stateInfo.getTables().size());
254 assertEquals(5, stateInfo.getRegionCountOfTable(tableDescTwo.getTableName()));
255 assertEquals(6, stateInfo.getRegionCount());
256 ADMIN.disableTable(tableDescOne.getTableName());
257 deleteTable(tableDescOne.getTableName());
258 stateInfo = getNamespaceState(nspDesc.getName());
259 assertNotNull("Namespace state found to be null.", stateInfo);
260 assertEquals(5, stateInfo.getRegionCount());
261 assertEquals(1, stateInfo.getTables().size());
262 ADMIN.disableTable(tableDescTwo.getTableName());
263 deleteTable(tableDescTwo.getTableName());
264 ADMIN.deleteNamespace(namespace);
265 stateInfo = getNamespaceState(namespace);
266 assertNull("Namespace state not found to be null.", stateInfo);
267 }
268
269 public static class CPRegionServerObserver extends BaseRegionServerObserver {
270 private volatile boolean shouldFailMerge = false;
271
272 public void failMerge(boolean fail) {
273 shouldFailMerge = fail;
274 }
275
276 private boolean triggered = false;
277
278 public synchronized void waitUtilTriggered() throws InterruptedException {
279 while (!triggered) {
280 wait();
281 }
282 }
283
284 @Override
285 public synchronized void preMerge(ObserverContext<RegionServerCoprocessorEnvironment> ctx,
286 Region regionA, Region regionB) throws IOException {
287 triggered = true;
288 notifyAll();
289 if (shouldFailMerge) {
290 throw new IOException("fail merge");
291 }
292 }
293 }
294
295 @Test
296 public void testRegionMerge() throws Exception {
297 String nsp1 = prefix + "_regiontest";
298 NamespaceDescriptor nspDesc =
299 NamespaceDescriptor.create(nsp1)
300 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "3")
301 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
302 ADMIN.createNamespace(nspDesc);
303 final TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
304 byte[] columnFamily = Bytes.toBytes("info");
305 HTableDescriptor tableDescOne = new HTableDescriptor(tableTwo);
306 tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
307 final int initialRegions = 3;
308 ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("2000"), initialRegions);
309 try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
310 Table table = connection.getTable(tableTwo)) {
311 UTIL.loadNumericRows(table, Bytes.toBytes("info"), 1000, 1999);
312 }
313 ADMIN.flush(tableTwo);
314 List<HRegionInfo> hris = ADMIN.getTableRegions(tableTwo);
315 Collections.sort(hris);
316
317 final Set<String> encodedRegionNamesToMerge =
318 Sets.newHashSet(hris.get(0).getEncodedName(), hris.get(1).getEncodedName());
319 ADMIN.mergeRegions(hris.get(0).getEncodedNameAsBytes(), hris.get(1).getEncodedNameAsBytes(),
320 false);
321 waitForMergeToComplete(tableTwo, encodedRegionNamesToMerge);
322 hris = ADMIN.getTableRegions(tableTwo);
323 assertEquals(initialRegions - 1, hris.size());
324 Collections.sort(hris);
325
326 final HRegionInfo hriToSplit = hris.get(1);
327 ADMIN.split(tableTwo, Bytes.toBytes("500"));
328
329 UTIL.waitFor(10000, 100, new Waiter.ExplainingPredicate<Exception>() {
330
331 @Override
332 public boolean evaluate() throws Exception {
333 RegionStates regionStates =
334 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
335 for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
336 if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
337 return false;
338 }
339 if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
340 return false;
341 }
342 }
343 return true;
344 }
345
346 @Override
347 public String explainFailure() throws Exception {
348 RegionStates regionStates =
349 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
350 for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
351 if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
352 return hriToSplit + " which is expected to be split is still online";
353 }
354 if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
355 return hri + " is still in not opened";
356 }
357 }
358 return "Unknown";
359 }
360 });
361 hris = ADMIN.getTableRegions(tableTwo);
362 assertEquals(initialRegions, hris.size());
363 Collections.sort(hris);
364
365
366 MiniHBaseCluster cluster = UTIL.getHBaseCluster();
367 HRegionServer regionServer = cluster.getRegionServer(0);
368 RegionServerCoprocessorHost cpHost = regionServer.getRegionServerCoprocessorHost();
369 Coprocessor coprocessor = cpHost.findCoprocessor(CPRegionServerObserver.class.getName());
370 CPRegionServerObserver regionServerObserver = (CPRegionServerObserver) coprocessor;
371 regionServerObserver.failMerge(true);
372 regionServerObserver.triggered = false;
373
374 ADMIN.mergeRegions(hris.get(1).getEncodedNameAsBytes(), hris.get(2).getEncodedNameAsBytes(),
375 false);
376 regionServerObserver.waitUtilTriggered();
377 hris = ADMIN.getTableRegions(tableTwo);
378 assertEquals(initialRegions, hris.size());
379 Collections.sort(hris);
380
381 HRegionInfo hriToSplit2 = hris.get(1);
382 ADMIN.split(tableTwo,
383 TableInputFormatBase.getSplitKey(hriToSplit2.getStartKey(), hriToSplit2.getEndKey(), true));
384 waitForMergeToComplete(tableTwo, encodedRegionNamesToMerge);
385 assertEquals(initialRegions, ADMIN.getTableRegions(tableTwo).size());
386 }
387
388 private void waitForMergeToComplete(final TableName tableTwo,
389 final Set<String> encodedRegionNamesToMerge) throws Exception {
390 UTIL.waitFor(10000, 100, new Waiter.ExplainingPredicate<Exception>() {
391
392 @Override
393 public boolean evaluate() throws Exception {
394 RegionStates regionStates =
395 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
396 for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
397 if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
398 return false;
399 }
400 if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
401 return false;
402 }
403 }
404 return true;
405 }
406
407 @Override
408 public String explainFailure() throws Exception {
409 RegionStates regionStates =
410 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
411 for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
412 if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
413 return hri + " which is expected to be merged is still online";
414 }
415 if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
416 return hri + " is still in not opened";
417 }
418 }
419 return "Unknown";
420 }
421 });
422 }
423
424 @Test
425 public void testRegionOperations() throws Exception {
426 String nsp1 = prefix + "_regiontest";
427 NamespaceDescriptor nspDesc =
428 NamespaceDescriptor.create(nsp1)
429 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "2")
430 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
431 ADMIN.createNamespace(nspDesc);
432 boolean constraintViolated = false;
433 final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
434 byte[] columnFamily = Bytes.toBytes("info");
435 HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
436 tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
437 NamespaceTableAndRegionInfo stateInfo;
438 try {
439 ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 7);
440 } catch (Exception exp) {
441 assertTrue(exp instanceof DoNotRetryIOException);
442 LOG.info(exp);
443 constraintViolated = true;
444 } finally {
445 assertTrue(constraintViolated);
446 }
447 assertFalse(ADMIN.tableExists(tableOne));
448
449 ADMIN.createTable(tableDescOne);
450 Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
451 HTable htable = (HTable) connection.getTable(tableOne);
452 UTIL.loadNumericRows(htable, Bytes.toBytes("info"), 1, 1000);
453 ADMIN.flush(tableOne);
454 stateInfo = getNamespaceState(nsp1);
455 assertEquals(1, stateInfo.getTables().size());
456 assertEquals(1, stateInfo.getRegionCount());
457 restartMaster();
458 ADMIN.split(tableOne, Bytes.toBytes("500"));
459 HRegion actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
460 CustomObserver observer =
461 (CustomObserver) actualRegion.getCoprocessorHost().findCoprocessor(
462 CustomObserver.class.getName());
463 assertNotNull(observer);
464 observer.postSplit.await();
465 assertEquals(2, ADMIN.getTableRegions(tableOne).size());
466 actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
467 observer =
468 (CustomObserver) actualRegion.getCoprocessorHost().findCoprocessor(
469 CustomObserver.class.getName());
470 assertNotNull(observer);
471 ADMIN.split(
472 tableOne,
473 getSplitKey(actualRegion.getRegionInfo().getStartKey(), actualRegion.getRegionInfo()
474 .getEndKey()));
475 observer.postSplit.await();
476
477 List<HRegionInfo> hris = ADMIN.getTableRegions(tableOne);
478 assertEquals(2, hris.size());
479 assertTrue("split completed", observer.preSplitBeforePONR.getCount() == 1);
480
481 htable.close();
482 }
483
484
485
486
487
488
489 @Test(timeout = 180000)
490 public void testRecreateTableWithSameNameAfterFirstTimeFailure() throws Exception {
491 String nsp1 = prefix + "_testRecreateTable";
492 NamespaceDescriptor nspDesc =
493 NamespaceDescriptor.create(nsp1)
494 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
495 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1").build();
496 ADMIN.createNamespace(nspDesc);
497 final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
498 byte[] columnFamily = Bytes.toBytes("info");
499 HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
500 tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
501 MasterSyncObserver.throwExceptionInPreCreateTable = true;
502 try {
503 try {
504 ADMIN.createTable(tableDescOne);
505 fail("Table " + tableOne.toString() + "creation should fail.");
506 } catch (Exception exp) {
507 LOG.error(exp);
508 }
509 assertFalse(ADMIN.tableExists(tableOne));
510
511 NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp1);
512 assertEquals("First table creation failed in namespace so number of tables in namespace "
513 + "should be 0.", 0, nstate.getTables().size());
514
515 MasterSyncObserver.throwExceptionInPreCreateTable = false;
516 try {
517 ADMIN.createTable(tableDescOne);
518 } catch (Exception e) {
519 fail("Table " + tableOne.toString() + "creation should succeed.");
520 LOG.error(e);
521 }
522 assertTrue(ADMIN.tableExists(tableOne));
523 nstate = getNamespaceState(nsp1);
524 assertEquals("First table was created successfully so table size in namespace should "
525 + "be one now.", 1, nstate.getTables().size());
526 } finally {
527 MasterSyncObserver.throwExceptionInPreCreateTable = false;
528 if (ADMIN.tableExists(tableOne)) {
529 ADMIN.disableTable(tableOne);
530 deleteTable(tableOne);
531 }
532 ADMIN.deleteNamespace(nsp1);
533 }
534 }
535
536 private NamespaceTableAndRegionInfo getNamespaceState(String namespace) throws KeeperException,
537 IOException {
538 return getQuotaManager().getState(namespace);
539 }
540
541 byte[] getSplitKey(byte[] startKey, byte[] endKey) {
542 String skey = Bytes.toString(startKey);
543 int key;
544 if (StringUtils.isBlank(skey)) {
545 key = Integer.parseInt(Bytes.toString(endKey)) / 2;
546 } else {
547 key = (int) (Integer.parseInt(skey) * 1.5);
548 }
549 return Bytes.toBytes("" + key);
550 }
551
552 public static class CustomObserver extends BaseRegionObserver {
553 volatile CountDownLatch postSplit;
554 volatile CountDownLatch preSplitBeforePONR;
555
556 @Override
557 public void postCompleteSplit(ObserverContext<RegionCoprocessorEnvironment> ctx)
558 throws IOException {
559 postSplit.countDown();
560 }
561
562 @Override
563 public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
564 byte[] splitKey, List<Mutation> metaEntries) throws IOException {
565 preSplitBeforePONR.countDown();
566 }
567
568 @Override
569 public void start(CoprocessorEnvironment e) throws IOException {
570 postSplit = new CountDownLatch(1);
571 preSplitBeforePONR = new CountDownLatch(1);
572 }
573 }
574
575 @Test
576 public void testStatePreserve() throws Exception {
577 final String nsp1 = prefix + "_testStatePreserve";
578 NamespaceDescriptor nspDesc =
579 NamespaceDescriptor.create(nsp1)
580 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
581 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "10").build();
582 ADMIN.createNamespace(nspDesc);
583 TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
584 TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
585 TableName tableThree = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table3");
586 HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
587 HTableDescriptor tableDescTwo = new HTableDescriptor(tableTwo);
588 HTableDescriptor tableDescThree = new HTableDescriptor(tableThree);
589 ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
590 ADMIN.createTable(tableDescTwo, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
591 ADMIN.createTable(tableDescThree, Bytes.toBytes("1"), Bytes.toBytes("1000"), 4);
592 ADMIN.disableTable(tableThree);
593 deleteTable(tableThree);
594
595 UTIL.waitFor(1000, new Waiter.Predicate<Exception>() {
596 @Override
597 public boolean evaluate() throws Exception {
598 return (getNamespaceState(nsp1).getTables().size() == 2);
599 }
600 });
601 NamespaceTableAndRegionInfo before = getNamespaceState(nsp1);
602 restartMaster();
603 NamespaceTableAndRegionInfo after = getNamespaceState(nsp1);
604 assertEquals("Expected: " + before.getTables() + " Found: " + after.getTables(), before
605 .getTables().size(), after.getTables().size());
606 }
607
608 private static void waitForQuotaEnabled() throws Exception {
609 UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
610 @Override
611 public boolean evaluate() throws Exception {
612 HMaster master = UTIL.getHBaseCluster().getMaster();
613 if (master == null) {
614 return false;
615 }
616 MasterQuotaManager quotaManager = master.getMasterQuotaManager();
617 return quotaManager != null && quotaManager.isQuotaEnabled();
618 }
619 });
620 }
621
622 private void restartMaster() throws Exception {
623 UTIL.getHBaseCluster().getMaster(0).stop("Stopping to start again");
624 UTIL.getHBaseCluster().waitOnMaster(0);
625 UTIL.getHBaseCluster().startMaster();
626 waitForQuotaEnabled();
627 }
628
629 private NamespaceAuditor getQuotaManager() {
630 return UTIL.getHBaseCluster().getMaster().getMasterQuotaManager().getNamespaceQuotaManager();
631 }
632
633 public static class MasterSyncObserver extends BaseMasterObserver {
634 volatile CountDownLatch tableDeletionLatch;
635 static boolean throwExceptionInPreCreateTable;
636
637 @Override
638 public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
639 TableName tableName) throws IOException {
640 tableDeletionLatch = new CountDownLatch(1);
641 }
642
643 @Override
644 public void postDeleteTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
645 TableName tableName) throws IOException {
646 tableDeletionLatch.countDown();
647 }
648
649 @Override
650 public void preCreateTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
651 HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
652 if (throwExceptionInPreCreateTable) {
653 throw new IOException("Throw exception as it is demanded.");
654 }
655 }
656 }
657
658 private void deleteTable(final TableName tableName) throws Exception {
659
660
661 MasterSyncObserver observer =
662 (MasterSyncObserver) UTIL.getHBaseCluster().getMaster().getMasterCoprocessorHost()
663 .findCoprocessor(MasterSyncObserver.class.getName());
664 ADMIN.deleteTable(tableName);
665 observer.tableDeletionLatch.await();
666 }
667
668 @Test(expected = QuotaExceededException.class, timeout = 30000)
669 public void testExceedTableQuotaInNamespace() throws Exception {
670 String nsp = prefix + "_testExceedTableQuotaInNamespace";
671 NamespaceDescriptor nspDesc =
672 NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
673 .build();
674 ADMIN.createNamespace(nspDesc);
675 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
676 assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
677 HTableDescriptor tableDescOne =
678 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
679 HTableDescriptor tableDescTwo =
680 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
681 ADMIN.createTable(tableDescOne);
682 ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
683 }
684
685 @Test(expected = QuotaExceededException.class, timeout = 30000)
686 public void testCloneSnapshotQuotaExceed() throws Exception {
687 String nsp = prefix + "_testTableQuotaExceedWithCloneSnapshot";
688 NamespaceDescriptor nspDesc =
689 NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
690 .build();
691 ADMIN.createNamespace(nspDesc);
692 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
693 TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
694 TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
695 HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
696 ADMIN.createTable(tableDescOne);
697 String snapshot = "snapshot_testTableQuotaExceedWithCloneSnapshot";
698 ADMIN.snapshot(snapshot, tableName);
699 ADMIN.cloneSnapshot(snapshot, cloneTableName);
700 ADMIN.deleteSnapshot(snapshot);
701 }
702
703 @Test(timeout = 180000)
704 public void testCloneSnapshot() throws Exception {
705 String nsp = prefix + "_testCloneSnapshot";
706 NamespaceDescriptor nspDesc =
707 NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2")
708 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20").build();
709 ADMIN.createNamespace(nspDesc);
710 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
711 TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
712 TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
713 HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
714
715 ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
716 String snapshot = "snapshot_testCloneSnapshot";
717 ADMIN.snapshot(snapshot, tableName);
718 ADMIN.cloneSnapshot(snapshot, cloneTableName);
719
720 int tableLength;
721 try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(tableName)) {
722 tableLength = locator.getStartKeys().length;
723 }
724 assertEquals(tableName.getNameAsString() + " should have four regions.", 4, tableLength);
725
726 try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(cloneTableName)) {
727 tableLength = locator.getStartKeys().length;
728 }
729 assertEquals(cloneTableName.getNameAsString() + " should have four regions.", 4, tableLength);
730
731 NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
732 assertEquals("Total tables count should be 2.", 2, nstate.getTables().size());
733 assertEquals("Total regions count should be.", 8, nstate.getRegionCount());
734
735 ADMIN.deleteSnapshot(snapshot);
736 }
737
738 @Test(timeout = 180000)
739 public void testRestoreSnapshot() throws Exception {
740 String nsp = prefix + "_testRestoreSnapshot";
741 NamespaceDescriptor nspDesc =
742 NamespaceDescriptor.create(nsp)
743 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
744 ADMIN.createNamespace(nspDesc);
745 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
746 TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
747 HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
748 ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
749
750 NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
751 assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
752
753 String snapshot = "snapshot_testRestoreSnapshot";
754 ADMIN.snapshot(snapshot, tableName1);
755
756 List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1);
757 Collections.sort(regions);
758
759 ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
760 Thread.sleep(2000);
761 assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());
762
763 ADMIN.disableTable(tableName1);
764 ADMIN.restoreSnapshot(snapshot);
765
766 assertEquals("Total regions count should be 4 after restore.", 4, nstate.getRegionCount());
767
768 ADMIN.enableTable(tableName1);
769 ADMIN.deleteSnapshot(snapshot);
770 }
771
772 @Test(timeout = 180000)
773 public void testRestoreSnapshotQuotaExceed() throws Exception {
774 String nsp = prefix + "_testRestoreSnapshotQuotaExceed";
775 NamespaceDescriptor nspDesc =
776 NamespaceDescriptor.create(nsp)
777 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
778 ADMIN.createNamespace(nspDesc);
779 NamespaceDescriptor ndesc = ADMIN.getNamespaceDescriptor(nsp);
780 assertNotNull("Namespace descriptor found null.", ndesc);
781 TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
782 HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
783 ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
784
785 NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
786 assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
787
788 String snapshot = "snapshot_testRestoreSnapshotQuotaExceed";
789 ADMIN.snapshot(snapshot, tableName1);
790
791 List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1);
792 Collections.sort(regions);
793
794 ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
795 Thread.sleep(2000);
796 assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());
797
798 ndesc.setConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "2");
799 ADMIN.modifyNamespace(ndesc);
800
801 ADMIN.disableTable(tableName1);
802 try {
803 ADMIN.restoreSnapshot(snapshot);
804 fail("Region quota is exceeded so QuotaExceededException should be thrown but HBaseAdmin"
805 + " wraps IOException into RestoreSnapshotException");
806 } catch (RestoreSnapshotException ignore) {
807 }
808 ADMIN.enableTable(tableName1);
809 ADMIN.deleteSnapshot(snapshot);
810 }
811 }