1
2
3
4
5
6
7
8
9
10
11 package org.apache.hadoop.hbase.namespace;
12
13 import static org.junit.Assert.assertEquals;
14 import static org.junit.Assert.assertFalse;
15 import static org.junit.Assert.assertNotNull;
16 import static org.junit.Assert.assertNull;
17 import static org.junit.Assert.assertTrue;
18 import static org.junit.Assert.fail;
19
20 import java.io.IOException;
21 import java.util.Collections;
22 import java.util.List;
23 import java.util.Set;
24 import java.util.concurrent.CountDownLatch;
25
26 import org.apache.commons.lang.StringUtils;
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.FileSystem;
31 import org.apache.hadoop.fs.Path;
32 import org.apache.hadoop.hbase.CategoryBasedTimeout;
33 import org.apache.hadoop.hbase.Coprocessor;
34 import org.apache.hadoop.hbase.CoprocessorEnvironment;
35 import org.apache.hadoop.hbase.DoNotRetryIOException;
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.HColumnDescriptor;
38 import org.apache.hadoop.hbase.HConstants;
39 import org.apache.hadoop.hbase.HRegionInfo;
40 import org.apache.hadoop.hbase.HTableDescriptor;
41 import org.apache.hadoop.hbase.MiniHBaseCluster;
42 import org.apache.hadoop.hbase.NamespaceDescriptor;
43 import org.apache.hadoop.hbase.TableName;
44 import org.apache.hadoop.hbase.Waiter;
45 import org.apache.hadoop.hbase.client.Connection;
46 import org.apache.hadoop.hbase.client.ConnectionFactory;
47 import org.apache.hadoop.hbase.client.HBaseAdmin;
48 import org.apache.hadoop.hbase.client.HTable;
49 import org.apache.hadoop.hbase.client.Mutation;
50 import org.apache.hadoop.hbase.client.RegionLocator;
51 import org.apache.hadoop.hbase.client.Table;
52 import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver;
53 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
54 import org.apache.hadoop.hbase.coprocessor.BaseRegionServerObserver;
55 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
56 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
57 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
58 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
59 import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
60 import org.apache.hadoop.hbase.coprocessor.RegionServerObserver;
61 import org.apache.hadoop.hbase.mapreduce.TableInputFormatBase;
62 import org.apache.hadoop.hbase.master.HMaster;
63 import org.apache.hadoop.hbase.master.RegionState;
64 import org.apache.hadoop.hbase.master.RegionStates;
65 import org.apache.hadoop.hbase.master.TableNamespaceManager;
66 import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
67 import org.apache.hadoop.hbase.quotas.QuotaExceededException;
68 import org.apache.hadoop.hbase.quotas.QuotaUtil;
69 import org.apache.hadoop.hbase.regionserver.HRegion;
70 import org.apache.hadoop.hbase.regionserver.HRegionServer;
71 import org.apache.hadoop.hbase.regionserver.Region;
72 import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost;
73 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
74 import org.apache.hadoop.hbase.testclassification.MediumTests;
75 import org.apache.hadoop.hbase.util.Bytes;
76 import org.apache.hadoop.hbase.util.FSUtils;
77 import org.apache.zookeeper.KeeperException;
78 import org.junit.After;
79 import org.junit.AfterClass;
80 import org.junit.BeforeClass;
81 import org.junit.Ignore;
82 import org.junit.Rule;
83 import org.junit.Test;
84 import org.junit.experimental.categories.Category;
85 import org.junit.rules.TestRule;
86
87 import com.google.common.collect.Sets;
88
89 @Category(MediumTests.class)
90 public class TestNamespaceAuditor {
91 @Rule public final TestRule timeout = CategoryBasedTimeout.builder().
92 withTimeout(this.getClass()).withLookingForStuckThread(true).build();
93 private static final Log LOG = LogFactory.getLog(TestNamespaceAuditor.class);
94 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
95 private static HBaseAdmin ADMIN;
96 private String prefix = "TestNamespaceAuditor";
97
98 @BeforeClass
99 public static void before() throws Exception {
100 UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
101 setupOnce();
102 }
103
104 public static void setupOnce() throws Exception, IOException {
105 Configuration conf = UTIL.getConfiguration();
106 conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, CustomObserver.class.getName());
107 conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MasterSyncObserver.class.getName());
108 conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
109 conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
110 conf.setClass("hbase.coprocessor.regionserver.classes", CPRegionServerObserver.class,
111 RegionServerObserver.class);
112 UTIL.startMiniCluster(1, 1);
113 waitForQuotaEnabled();
114 ADMIN = UTIL.getHBaseAdmin();
115 }
116
117 @AfterClass
118 public static void tearDown() throws Exception {
119 UTIL.shutdownMiniCluster();
120 }
121
122 @After
123 public void cleanup() throws Exception, KeeperException {
124 for (HTableDescriptor table : ADMIN.listTables()) {
125 ADMIN.disableTable(table.getTableName());
126 deleteTable(table.getTableName());
127 }
128 for (NamespaceDescriptor ns : ADMIN.listNamespaceDescriptors()) {
129 if (ns.getName().startsWith(prefix)) {
130 ADMIN.deleteNamespace(ns.getName());
131 }
132 }
133 assertTrue("Quota manager not initialized", UTIL.getHBaseCluster().getMaster()
134 .getMasterQuotaManager().isQuotaInitialized());
135 }
136
137 @Test
138 public void testTableOperations() throws Exception {
139 String nsp = prefix + "_np2";
140 NamespaceDescriptor nspDesc =
141 NamespaceDescriptor.create(nsp)
142 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5")
143 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
144 ADMIN.createNamespace(nspDesc);
145 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
146 assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
147 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
148
149 HTableDescriptor tableDescOne =
150 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
151 tableDescOne.addFamily(fam1);
152 HTableDescriptor tableDescTwo =
153 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
154 tableDescTwo.addFamily(fam1);
155 HTableDescriptor tableDescThree =
156 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table3"));
157 tableDescThree.addFamily(fam1);
158 ADMIN.createTable(tableDescOne);
159 boolean constraintViolated = false;
160 try {
161 ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
162 } catch (Exception exp) {
163 assertTrue(exp instanceof IOException);
164 constraintViolated = true;
165 } finally {
166 assertTrue("Constraint not violated for table " + tableDescTwo.getTableName(),
167 constraintViolated);
168 }
169 ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
170 NamespaceTableAndRegionInfo nspState = getQuotaManager().getState(nsp);
171 assertNotNull(nspState);
172 assertTrue(nspState.getTables().size() == 2);
173 assertTrue(nspState.getRegionCount() == 5);
174 constraintViolated = false;
175 try {
176 ADMIN.createTable(tableDescThree);
177 } catch (Exception exp) {
178 assertTrue(exp instanceof IOException);
179 constraintViolated = true;
180 } finally {
181 assertTrue("Constraint not violated for table " + tableDescThree.getTableName(),
182 constraintViolated);
183 }
184 }
185
186 @Test
187 public void testValidQuotas() throws Exception {
188 boolean exceptionCaught = false;
189 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
190 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
191 NamespaceDescriptor nspDesc =
192 NamespaceDescriptor.create(prefix + "vq1")
193 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "hihdufh")
194 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
195 try {
196 ADMIN.createNamespace(nspDesc);
197 } catch (Exception exp) {
198 LOG.warn(exp);
199 exceptionCaught = true;
200 } finally {
201 assertTrue(exceptionCaught);
202 assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
203 }
204 nspDesc =
205 NamespaceDescriptor.create(prefix + "vq2")
206 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "-456")
207 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
208 try {
209 ADMIN.createNamespace(nspDesc);
210 } catch (Exception exp) {
211 LOG.warn(exp);
212 exceptionCaught = true;
213 } finally {
214 assertTrue(exceptionCaught);
215 assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
216 }
217 nspDesc =
218 NamespaceDescriptor.create(prefix + "vq3")
219 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
220 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "sciigd").build();
221 try {
222 ADMIN.createNamespace(nspDesc);
223 } catch (Exception exp) {
224 LOG.warn(exp);
225 exceptionCaught = true;
226 } finally {
227 assertTrue(exceptionCaught);
228 assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
229 }
230 nspDesc =
231 NamespaceDescriptor.create(prefix + "vq4")
232 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
233 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "-1500").build();
234 try {
235 ADMIN.createNamespace(nspDesc);
236 } catch (Exception exp) {
237 LOG.warn(exp);
238 exceptionCaught = true;
239 } finally {
240 assertTrue(exceptionCaught);
241 assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
242 }
243 }
244
245 @Test
246 public void testDeleteTable() throws Exception {
247 String namespace = prefix + "_dummy";
248 NamespaceDescriptor nspDesc =
249 NamespaceDescriptor.create(namespace)
250 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "100")
251 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "3").build();
252 ADMIN.createNamespace(nspDesc);
253 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(namespace));
254 NamespaceTableAndRegionInfo stateInfo = getNamespaceState(nspDesc.getName());
255 assertNotNull("Namespace state found null for " + namespace, stateInfo);
256 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
257 HTableDescriptor tableDescOne =
258 new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table1"));
259 tableDescOne.addFamily(fam1);
260 HTableDescriptor tableDescTwo =
261 new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table2"));
262 tableDescTwo.addFamily(fam1);
263 ADMIN.createTable(tableDescOne);
264 ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
265 stateInfo = getNamespaceState(nspDesc.getName());
266 assertNotNull("Namespace state found to be null.", stateInfo);
267 assertEquals(2, stateInfo.getTables().size());
268 assertEquals(5, stateInfo.getRegionCountOfTable(tableDescTwo.getTableName()));
269 assertEquals(6, stateInfo.getRegionCount());
270 ADMIN.disableTable(tableDescOne.getTableName());
271 deleteTable(tableDescOne.getTableName());
272 stateInfo = getNamespaceState(nspDesc.getName());
273 assertNotNull("Namespace state found to be null.", stateInfo);
274 assertEquals(5, stateInfo.getRegionCount());
275 assertEquals(1, stateInfo.getTables().size());
276 ADMIN.disableTable(tableDescTwo.getTableName());
277 deleteTable(tableDescTwo.getTableName());
278 ADMIN.deleteNamespace(namespace);
279 stateInfo = getNamespaceState(namespace);
280 assertNull("Namespace state not found to be null.", stateInfo);
281 }
282
283 public static class CPRegionServerObserver extends BaseRegionServerObserver {
284 private volatile boolean shouldFailMerge = false;
285
286 public void failMerge(boolean fail) {
287 shouldFailMerge = fail;
288 }
289
290 private boolean triggered = false;
291
292 public synchronized void waitUtilTriggered() throws InterruptedException {
293 while (!triggered) {
294 wait();
295 }
296 }
297
298 @Override
299 public synchronized void preMerge(ObserverContext<RegionServerCoprocessorEnvironment> ctx,
300 Region regionA, Region regionB) throws IOException {
301 triggered = true;
302 notifyAll();
303 if (shouldFailMerge) {
304 throw new IOException("fail merge");
305 }
306 }
307 }
308
309 @Test
310 public void testRegionMerge() throws Exception {
311 String nsp1 = prefix + "_regiontest";
312 NamespaceDescriptor nspDesc =
313 NamespaceDescriptor.create(nsp1)
314 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "3")
315 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
316 ADMIN.createNamespace(nspDesc);
317 final TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
318 byte[] columnFamily = Bytes.toBytes("info");
319 HTableDescriptor tableDescOne = new HTableDescriptor(tableTwo);
320 tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
321 final int initialRegions = 3;
322 ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("2000"), initialRegions);
323 try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
324 Table table = connection.getTable(tableTwo)) {
325 UTIL.loadNumericRows(table, Bytes.toBytes("info"), 1000, 1999);
326 }
327 ADMIN.flush(tableTwo);
328 List<HRegionInfo> hris = ADMIN.getTableRegions(tableTwo);
329 Collections.sort(hris);
330
331 final Set<String> encodedRegionNamesToMerge =
332 Sets.newHashSet(hris.get(0).getEncodedName(), hris.get(1).getEncodedName());
333 ADMIN.mergeRegions(hris.get(0).getEncodedNameAsBytes(), hris.get(1).getEncodedNameAsBytes(),
334 false);
335 waitForMergeToComplete(tableTwo, encodedRegionNamesToMerge);
336 hris = ADMIN.getTableRegions(tableTwo);
337 assertEquals(initialRegions - 1, hris.size());
338 Collections.sort(hris);
339
340 final HRegionInfo hriToSplit = hris.get(1);
341 ADMIN.split(tableTwo, Bytes.toBytes("500"));
342
343 UTIL.waitFor(10000, 100, new Waiter.ExplainingPredicate<Exception>() {
344
345 @Override
346 public boolean evaluate() throws Exception {
347 RegionStates regionStates =
348 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
349 for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
350 if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
351 return false;
352 }
353 if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
354 return false;
355 }
356 }
357 return true;
358 }
359
360 @Override
361 public String explainFailure() throws Exception {
362 RegionStates regionStates =
363 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
364 for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
365 if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
366 return hriToSplit + " which is expected to be split is still online";
367 }
368 if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
369 return hri + " is still in not opened";
370 }
371 }
372 return "Unknown";
373 }
374 });
375 hris = ADMIN.getTableRegions(tableTwo);
376 assertEquals(initialRegions, hris.size());
377 Collections.sort(hris);
378
379
380 MiniHBaseCluster cluster = UTIL.getHBaseCluster();
381 HRegionServer regionServer = cluster.getRegionServer(0);
382 RegionServerCoprocessorHost cpHost = regionServer.getRegionServerCoprocessorHost();
383 Coprocessor coprocessor = cpHost.findCoprocessor(CPRegionServerObserver.class.getName());
384 CPRegionServerObserver regionServerObserver = (CPRegionServerObserver) coprocessor;
385 regionServerObserver.failMerge(true);
386 regionServerObserver.triggered = false;
387
388 ADMIN.mergeRegions(hris.get(1).getEncodedNameAsBytes(), hris.get(2).getEncodedNameAsBytes(),
389 false);
390 regionServerObserver.waitUtilTriggered();
391 hris = ADMIN.getTableRegions(tableTwo);
392 assertEquals(initialRegions, hris.size());
393 Collections.sort(hris);
394
395 ADMIN.split(tableTwo, Bytes.toBytes("6"));
396 waitForMergeToComplete(tableTwo, encodedRegionNamesToMerge);
397 assertEquals(initialRegions, ADMIN.getTableRegions(tableTwo).size());
398 }
399
400 private void waitForMergeToComplete(final TableName tableTwo,
401 final Set<String> encodedRegionNamesToMerge) throws Exception {
402 UTIL.waitFor(10000, 100, new Waiter.ExplainingPredicate<Exception>() {
403
404 @Override
405 public boolean evaluate() throws Exception {
406 RegionStates regionStates =
407 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
408 for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
409 if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
410 return false;
411 }
412 if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
413 return false;
414 }
415 }
416 return true;
417 }
418
419 @Override
420 public String explainFailure() throws Exception {
421 RegionStates regionStates =
422 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
423 for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
424 if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
425 return hri + " which is expected to be merged is still online";
426 }
427 if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
428 return hri + " is still in not opened";
429 }
430 }
431 return "Unknown";
432 }
433 });
434 }
435
436 @Ignore("Hangs on occasion waiting on countdown latch") @Test
437 public void testRegionOperations() throws Exception {
438 String nsp1 = prefix + "_regiontest";
439 NamespaceDescriptor nspDesc =
440 NamespaceDescriptor.create(nsp1)
441 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "2")
442 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
443 ADMIN.createNamespace(nspDesc);
444 boolean constraintViolated = false;
445 final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
446 byte[] columnFamily = Bytes.toBytes("info");
447 HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
448 tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
449 NamespaceTableAndRegionInfo stateInfo;
450 try {
451 ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 7);
452 } catch (Exception exp) {
453 assertTrue(exp instanceof DoNotRetryIOException);
454 LOG.info(exp);
455 constraintViolated = true;
456 } finally {
457 assertTrue(constraintViolated);
458 }
459 assertFalse(ADMIN.tableExists(tableOne));
460
461 ADMIN.createTable(tableDescOne);
462 Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
463 HTable htable = (HTable) connection.getTable(tableOne);
464 UTIL.loadNumericRows(htable, Bytes.toBytes("info"), 1, 1000);
465 ADMIN.flush(tableOne);
466 stateInfo = getNamespaceState(nsp1);
467 assertEquals(1, stateInfo.getTables().size());
468 assertEquals(1, stateInfo.getRegionCount());
469 restartMaster();
470 ADMIN.split(tableOne, Bytes.toBytes("500"));
471 HRegion actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
472 CustomObserver observer =
473 (CustomObserver) actualRegion.getCoprocessorHost().findCoprocessor(
474 CustomObserver.class.getName());
475 assertNotNull(observer);
476 observer.postSplit.await();
477 assertEquals(2, ADMIN.getTableRegions(tableOne).size());
478 actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
479 observer =
480 (CustomObserver) actualRegion.getCoprocessorHost().findCoprocessor(
481 CustomObserver.class.getName());
482 assertNotNull(observer);
483 ADMIN.split(
484 tableOne,
485 getSplitKey(actualRegion.getRegionInfo().getStartKey(), actualRegion.getRegionInfo()
486 .getEndKey()));
487 observer.postSplit.await();
488
489 List<HRegionInfo> hris = ADMIN.getTableRegions(tableOne);
490 assertEquals(2, hris.size());
491 assertTrue("split completed", observer.preSplitBeforePONR.getCount() == 1);
492
493 htable.close();
494 }
495
496
497
498
499
500
501 @Test
502 public void testRecreateTableWithSameNameAfterFirstTimeFailure() throws Exception {
503 String nsp1 = prefix + "_testRecreateTable";
504 NamespaceDescriptor nspDesc =
505 NamespaceDescriptor.create(nsp1)
506 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
507 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1").build();
508 ADMIN.createNamespace(nspDesc);
509 final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
510 byte[] columnFamily = Bytes.toBytes("info");
511 HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
512 tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
513 MasterSyncObserver.throwExceptionInPreCreateTableHandler = true;
514 try {
515 try {
516 ADMIN.createTable(tableDescOne);
517 fail("Table " + tableOne.toString() + "creation should fail.");
518 } catch (Exception exp) {
519 LOG.error(exp);
520 }
521 assertFalse(ADMIN.tableExists(tableOne));
522
523 NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp1);
524 assertEquals("First table creation failed in namespace so number of tables in namespace "
525 + "should be 0.", 0, nstate.getTables().size());
526
527 MasterSyncObserver.throwExceptionInPreCreateTableHandler = false;
528 try {
529 ADMIN.createTable(tableDescOne);
530 } catch (Exception e) {
531 fail("Table " + tableOne.toString() + "creation should succeed.");
532 LOG.error(e);
533 }
534 assertTrue(ADMIN.tableExists(tableOne));
535 nstate = getNamespaceState(nsp1);
536 assertEquals("First table was created successfully so table size in namespace should "
537 + "be one now.", 1, nstate.getTables().size());
538 } finally {
539 MasterSyncObserver.throwExceptionInPreCreateTableHandler = false;
540 if (ADMIN.tableExists(tableOne)) {
541 ADMIN.disableTable(tableOne);
542 deleteTable(tableOne);
543 }
544 ADMIN.deleteNamespace(nsp1);
545 }
546 }
547
548 private NamespaceTableAndRegionInfo getNamespaceState(String namespace) throws KeeperException,
549 IOException {
550 return getQuotaManager().getState(namespace);
551 }
552
553 byte[] getSplitKey(byte[] startKey, byte[] endKey) {
554 String skey = Bytes.toString(startKey);
555 int key;
556 if (StringUtils.isBlank(skey)) {
557 key = Integer.parseInt(Bytes.toString(endKey)) / 2;
558 } else {
559 key = (int) (Integer.parseInt(skey) * 1.5);
560 }
561 return Bytes.toBytes("" + key);
562 }
563
564 public static class CustomObserver extends BaseRegionObserver {
565 volatile CountDownLatch postSplit;
566 volatile CountDownLatch preSplitBeforePONR;
567
568 @Override
569 public void postCompleteSplit(ObserverContext<RegionCoprocessorEnvironment> ctx)
570 throws IOException {
571 postSplit.countDown();
572 }
573
574 @Override
575 public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
576 byte[] splitKey, List<Mutation> metaEntries) throws IOException {
577 preSplitBeforePONR.countDown();
578 }
579
580 @Override
581 public void start(CoprocessorEnvironment e) throws IOException {
582 postSplit = new CountDownLatch(1);
583 preSplitBeforePONR = new CountDownLatch(1);
584 }
585 }
586
587 @Test
588 public void testStatePreserve() throws Exception {
589 final String nsp1 = prefix + "_testStatePreserve";
590 NamespaceDescriptor nspDesc =
591 NamespaceDescriptor.create(nsp1)
592 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
593 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "10").build();
594 ADMIN.createNamespace(nspDesc);
595 TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
596 TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
597 TableName tableThree = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table3");
598 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
599 HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
600 tableDescOne.addFamily(fam1);
601 HTableDescriptor tableDescTwo = new HTableDescriptor(tableTwo);
602 tableDescTwo.addFamily(fam1);
603 HTableDescriptor tableDescThree = new HTableDescriptor(tableThree);
604 tableDescThree.addFamily(fam1);
605 ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
606 ADMIN.createTable(tableDescTwo, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
607 ADMIN.createTable(tableDescThree, Bytes.toBytes("1"), Bytes.toBytes("1000"), 4);
608 ADMIN.disableTable(tableThree);
609 deleteTable(tableThree);
610
611 UTIL.waitFor(1000, new Waiter.Predicate<Exception>() {
612 @Override
613 public boolean evaluate() throws Exception {
614 return (getNamespaceState(nsp1).getTables().size() == 2);
615 }
616 });
617 NamespaceTableAndRegionInfo before = getNamespaceState(nsp1);
618 restartMaster();
619 NamespaceTableAndRegionInfo after = getNamespaceState(nsp1);
620 assertEquals("Expected: " + before.getTables() + " Found: " + after.getTables(), before
621 .getTables().size(), after.getTables().size());
622 }
623
624 private static void waitForQuotaEnabled() throws Exception {
625 UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
626 @Override
627 public boolean evaluate() throws Exception {
628 HMaster master = UTIL.getHBaseCluster().getMaster();
629 if (master == null) {
630 return false;
631 }
632 MasterQuotaManager quotaManager = master.getMasterQuotaManager();
633 return quotaManager != null && quotaManager.isQuotaInitialized();
634 }
635 });
636 }
637
638 private void restartMaster() throws Exception {
639 UTIL.getHBaseCluster().getMaster(0).stop("Stopping to start again");
640 UTIL.getHBaseCluster().waitOnMaster(0);
641 UTIL.getHBaseCluster().startMaster();
642 waitForQuotaEnabled();
643 }
644
645 private NamespaceAuditor getQuotaManager() {
646 return UTIL.getHBaseCluster().getMaster().getMasterQuotaManager().getNamespaceQuotaManager();
647 }
648
649 public static class MasterSyncObserver extends BaseMasterObserver {
650 volatile CountDownLatch tableDeletionLatch;
651 static boolean throwExceptionInPreCreateTableHandler;
652
653 @Override
654 public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
655 TableName tableName) throws IOException {
656 tableDeletionLatch = new CountDownLatch(1);
657 }
658
659 @Override
660 public void postDeleteTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
661 TableName tableName) throws IOException {
662 tableDeletionLatch.countDown();
663 }
664
665 @Override
666 public void preCreateTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
667 HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
668 if (throwExceptionInPreCreateTableHandler) {
669 throw new IOException("Throw exception as it is demanded.");
670 }
671 }
672 }
673
674 private void deleteTable(final TableName tableName) throws Exception {
675
676
677 MasterSyncObserver observer =
678 (MasterSyncObserver) UTIL.getHBaseCluster().getMaster().getMasterCoprocessorHost()
679 .findCoprocessor(MasterSyncObserver.class.getName());
680 ADMIN.deleteTable(tableName);
681 observer.tableDeletionLatch.await();
682 }
683
684 @Test(expected = QuotaExceededException.class)
685 public void testExceedTableQuotaInNamespace() throws Exception {
686 String nsp = prefix + "_testExceedTableQuotaInNamespace";
687 NamespaceDescriptor nspDesc =
688 NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
689 .build();
690 ADMIN.createNamespace(nspDesc);
691 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
692 assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
693 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
694 HTableDescriptor tableDescOne =
695 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
696 tableDescOne.addFamily(fam1);
697 HTableDescriptor tableDescTwo =
698 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
699 tableDescTwo.addFamily(fam1);
700 ADMIN.createTable(tableDescOne);
701 ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
702 }
703
704 @Test(expected = QuotaExceededException.class)
705 public void testCloneSnapshotQuotaExceed() throws Exception {
706 String nsp = prefix + "_testTableQuotaExceedWithCloneSnapshot";
707 NamespaceDescriptor nspDesc =
708 NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
709 .build();
710 ADMIN.createNamespace(nspDesc);
711 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
712 TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
713 TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
714 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
715 HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
716 tableDescOne.addFamily(fam1);
717 ADMIN.createTable(tableDescOne);
718 String snapshot = "snapshot_testTableQuotaExceedWithCloneSnapshot";
719 ADMIN.snapshot(snapshot, tableName);
720 ADMIN.cloneSnapshot(snapshot, cloneTableName);
721 ADMIN.deleteSnapshot(snapshot);
722 }
723
724 @Test
725 public void testCloneSnapshot() throws Exception {
726 String nsp = prefix + "_testCloneSnapshot";
727 NamespaceDescriptor nspDesc =
728 NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2")
729 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20").build();
730 ADMIN.createNamespace(nspDesc);
731 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
732 TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
733 TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
734
735 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
736 HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
737 tableDescOne.addFamily(fam1);
738
739 ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
740 String snapshot = "snapshot_testCloneSnapshot";
741 ADMIN.snapshot(snapshot, tableName);
742 ADMIN.cloneSnapshot(snapshot, cloneTableName);
743
744 int tableLength;
745 try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(tableName)) {
746 tableLength = locator.getStartKeys().length;
747 }
748 assertEquals(tableName.getNameAsString() + " should have four regions.", 4, tableLength);
749
750 try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(cloneTableName)) {
751 tableLength = locator.getStartKeys().length;
752 }
753 assertEquals(cloneTableName.getNameAsString() + " should have four regions.", 4, tableLength);
754
755 NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
756 assertEquals("Total tables count should be 2.", 2, nstate.getTables().size());
757 assertEquals("Total regions count should be.", 8, nstate.getRegionCount());
758
759 ADMIN.deleteSnapshot(snapshot);
760 }
761
762 @Test
763 public void testRestoreSnapshot() throws Exception {
764 String nsp = prefix + "_testRestoreSnapshot";
765 NamespaceDescriptor nspDesc =
766 NamespaceDescriptor.create(nsp)
767 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
768 ADMIN.createNamespace(nspDesc);
769 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
770 TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
771 HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
772 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
773 tableDescOne.addFamily(fam1);
774 ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
775
776 NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
777 assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
778
779 String snapshot = "snapshot_testRestoreSnapshot";
780 ADMIN.snapshot(snapshot, tableName1);
781
782 List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1);
783 Collections.sort(regions);
784
785 ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
786 Thread.sleep(2000);
787 assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());
788
789 ADMIN.disableTable(tableName1);
790 ADMIN.restoreSnapshot(snapshot);
791
792 assertEquals("Total regions count should be 4 after restore.", 4, nstate.getRegionCount());
793
794 ADMIN.enableTable(tableName1);
795 ADMIN.deleteSnapshot(snapshot);
796 }
797
798 @Test
799 public void testRestoreSnapshotQuotaExceed() throws Exception {
800 String nsp = prefix + "_testRestoreSnapshotQuotaExceed";
801 NamespaceDescriptor nspDesc =
802 NamespaceDescriptor.create(nsp)
803 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
804 ADMIN.createNamespace(nspDesc);
805 NamespaceDescriptor ndesc = ADMIN.getNamespaceDescriptor(nsp);
806 assertNotNull("Namespace descriptor found null.", ndesc);
807 TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
808 HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
809 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
810 tableDescOne.addFamily(fam1);
811
812 ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
813
814 NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
815 assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
816
817 String snapshot = "snapshot_testRestoreSnapshotQuotaExceed";
818
819 ADMIN.snapshot(snapshot, tableName1);
820
821 ADMIN.disableTable(tableName1);
822 ADMIN.deleteTable(tableName1);
823 ADMIN.createTable(tableDescOne);
824 ndesc.setConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "3");
825 ADMIN.modifyNamespace(ndesc);
826
827 ADMIN.disableTable(tableName1);
828 try {
829 ADMIN.restoreSnapshot(snapshot);
830 fail("Region quota is exceeded so QuotaExceededException should be thrown but HBaseAdmin"
831 + " wraps IOException into RestoreSnapshotException");
832 } catch (RestoreSnapshotException ignore) {
833 assertTrue(ignore.getCause() instanceof QuotaExceededException);
834 }
835 assertEquals(1, getNamespaceState(nsp).getRegionCount());
836 ADMIN.enableTable(tableName1);
837 ADMIN.deleteSnapshot(snapshot);
838 }
839 }