1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.client;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.util.ArrayList;
28 import java.util.HashMap;
29 import java.util.Iterator;
30 import java.util.List;
31 import java.util.Map;
32 import java.util.Set;
33 import java.util.concurrent.atomic.AtomicBoolean;
34 import java.util.concurrent.atomic.AtomicInteger;
35
36 import org.apache.commons.logging.Log;
37 import org.apache.commons.logging.LogFactory;
38 import org.apache.hadoop.hbase.HBaseTestingUtility;
39 import org.apache.hadoop.hbase.HColumnDescriptor;
40 import org.apache.hadoop.hbase.HConstants;
41 import org.apache.hadoop.hbase.HRegionInfo;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.InvalidFamilyOperationException;
44 import org.apache.hadoop.hbase.testclassification.LargeTests;
45 import org.apache.hadoop.hbase.MasterNotRunningException;
46 import org.apache.hadoop.hbase.MetaTableAccessor;
47 import org.apache.hadoop.hbase.ServerName;
48 import org.apache.hadoop.hbase.TableName;
49 import org.apache.hadoop.hbase.TableNotDisabledException;
50 import org.apache.hadoop.hbase.TableNotEnabledException;
51 import org.apache.hadoop.hbase.TableNotFoundException;
52 import org.apache.hadoop.hbase.Waiter;
53 import org.apache.hadoop.hbase.executor.EventHandler;
54 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
55 import org.apache.hadoop.hbase.util.Bytes;
56 import org.apache.hadoop.hbase.util.FSUtils;
57 import org.apache.hadoop.hbase.zookeeper.ZKTableStateClientSideReader;
58 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
59 import org.apache.hadoop.hbase.exceptions.MergeRegionException;
60 import org.apache.hadoop.hbase.master.AssignmentManager;
61 import org.apache.hadoop.hbase.master.HMaster;
62 import org.apache.hadoop.hbase.master.RegionState;
63 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
64 import org.apache.hadoop.hbase.protobuf.RequestConverter;
65 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
66 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DispatchMergingRegionsRequest;
67 import org.apache.hadoop.hbase.regionserver.HRegion;
68 import org.apache.hadoop.hbase.regionserver.Store;
69 import org.apache.hadoop.hbase.regionserver.StoreFile;
70 import org.apache.hadoop.hbase.util.Pair;
71 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
72 import org.junit.After;
73 import org.junit.AfterClass;
74 import org.junit.Before;
75 import org.junit.BeforeClass;
76 import org.junit.Test;
77 import org.junit.experimental.categories.Category;
78
79 import com.google.protobuf.ServiceException;
80
81
82
83
84
85
86 @Category(LargeTests.class)
87 public class TestAdmin1 {
88 private static final Log LOG = LogFactory.getLog(TestAdmin1.class);
89 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
90 private Admin admin;
91
92 @BeforeClass
93 public static void setUpBeforeClass() throws Exception {
94 TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
95 TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
96 TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
97 TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
98 TEST_UTIL.getConfiguration().setBoolean(
99 "hbase.master.enabletable.roundrobin", true);
100 TEST_UTIL.startMiniCluster(3);
101 }
102
103 @AfterClass
104 public static void tearDownAfterClass() throws Exception {
105 TEST_UTIL.shutdownMiniCluster();
106 }
107
108 @Before
109 public void setUp() throws Exception {
110 this.admin = TEST_UTIL.getHBaseAdmin();
111 }
112
113 @After
114 public void tearDown() throws Exception {
115 for (HTableDescriptor htd : this.admin.listTables()) {
116 TEST_UTIL.deleteTable(htd.getName());
117 }
118 }
119
120 @Test (timeout=300000)
121 public void testSplitFlushCompactUnknownTable() throws InterruptedException {
122 final TableName unknowntable = TableName.valueOf("fubar");
123 Exception exception = null;
124 try {
125 this.admin.compact(unknowntable);
126 } catch (IOException e) {
127 exception = e;
128 }
129 assertTrue(exception instanceof TableNotFoundException);
130
131 exception = null;
132 try {
133 this.admin.flush(unknowntable);
134 } catch (IOException e) {
135 exception = e;
136 }
137 assertTrue(exception instanceof TableNotFoundException);
138
139 exception = null;
140 try {
141 this.admin.split(unknowntable);
142 } catch (IOException e) {
143 exception = e;
144 }
145 assertTrue(exception instanceof TableNotFoundException);
146 }
147
148 @Test (timeout=300000)
149 public void testDeleteEditUnknownColumnFamilyAndOrTable() throws IOException {
150
151 final TableName nonexistentTable = TableName.valueOf("nonexistent");
152 final byte[] nonexistentColumn = Bytes.toBytes("nonexistent");
153 HColumnDescriptor nonexistentHcd = new HColumnDescriptor(nonexistentColumn);
154 Exception exception = null;
155 try {
156 this.admin.addColumn(nonexistentTable, nonexistentHcd);
157 } catch (IOException e) {
158 exception = e;
159 }
160 assertTrue(exception instanceof TableNotFoundException);
161
162 exception = null;
163 try {
164 this.admin.deleteTable(nonexistentTable);
165 } catch (IOException e) {
166 exception = e;
167 }
168 assertTrue(exception instanceof TableNotFoundException);
169
170 exception = null;
171 try {
172 this.admin.deleteColumn(nonexistentTable, nonexistentColumn);
173 } catch (IOException e) {
174 exception = e;
175 }
176 assertTrue(exception instanceof TableNotFoundException);
177
178 exception = null;
179 try {
180 this.admin.disableTable(nonexistentTable);
181 } catch (IOException e) {
182 exception = e;
183 }
184 assertTrue(exception instanceof TableNotFoundException);
185
186 exception = null;
187 try {
188 this.admin.enableTable(nonexistentTable);
189 } catch (IOException e) {
190 exception = e;
191 }
192 assertTrue(exception instanceof TableNotFoundException);
193
194 exception = null;
195 try {
196 this.admin.modifyColumn(nonexistentTable, nonexistentHcd);
197 } catch (IOException e) {
198 exception = e;
199 }
200 assertTrue(exception instanceof TableNotFoundException);
201
202 exception = null;
203 try {
204 HTableDescriptor htd = new HTableDescriptor(nonexistentTable);
205 htd.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
206 this.admin.modifyTable(htd.getTableName(), htd);
207 } catch (IOException e) {
208 exception = e;
209 }
210 assertTrue(exception instanceof TableNotFoundException);
211
212
213
214 final TableName tableName =
215 TableName.valueOf("testDeleteEditUnknownColumnFamilyAndOrTable" + System.currentTimeMillis());
216 HTableDescriptor htd = new HTableDescriptor(tableName);
217 htd.addFamily(new HColumnDescriptor("cf"));
218 this.admin.createTable(htd);
219 try {
220 exception = null;
221 try {
222 this.admin.deleteColumn(htd.getTableName(), nonexistentHcd.getName());
223 } catch (IOException e) {
224 exception = e;
225 }
226 assertTrue("found=" + exception.getClass().getName(),
227 exception instanceof InvalidFamilyOperationException);
228
229 exception = null;
230 try {
231 this.admin.modifyColumn(htd.getTableName(), nonexistentHcd);
232 } catch (IOException e) {
233 exception = e;
234 }
235 assertTrue("found=" + exception.getClass().getName(),
236 exception instanceof InvalidFamilyOperationException);
237 } finally {
238 this.admin.disableTable(tableName);
239 this.admin.deleteTable(tableName);
240 }
241 }
242
243 @Test (timeout=300000)
244 public void testDisableAndEnableTable() throws IOException {
245 final byte [] row = Bytes.toBytes("row");
246 final byte [] qualifier = Bytes.toBytes("qualifier");
247 final byte [] value = Bytes.toBytes("value");
248 final TableName table = TableName.valueOf("testDisableAndEnableTable");
249 Table ht = TEST_UTIL.createTable(table, HConstants.CATALOG_FAMILY);
250 Put put = new Put(row);
251 put.add(HConstants.CATALOG_FAMILY, qualifier, value);
252 ht.put(put);
253 Get get = new Get(row);
254 get.addColumn(HConstants.CATALOG_FAMILY, qualifier);
255 ht.get(get);
256
257 this.admin.disableTable(ht.getName());
258 assertTrue("Table must be disabled.", TEST_UTIL.getHBaseCluster()
259 .getMaster().getAssignmentManager().getTableStateManager().isTableState(
260 ht.getName(), ZooKeeperProtos.Table.State.DISABLED));
261
262
263 get = new Get(row);
264 get.addColumn(HConstants.CATALOG_FAMILY, qualifier);
265 boolean ok = false;
266 try {
267 ht.get(get);
268 } catch (TableNotEnabledException e) {
269 ok = true;
270 }
271 ok = false;
272
273 Scan scan = new Scan();
274 try {
275 ResultScanner scanner = ht.getScanner(scan);
276 Result res = null;
277 do {
278 res = scanner.next();
279 } while (res != null);
280 } catch (TableNotEnabledException e) {
281 ok = true;
282 }
283 assertTrue(ok);
284 this.admin.enableTable(table);
285 assertTrue("Table must be enabled.", TEST_UTIL.getHBaseCluster()
286 .getMaster().getAssignmentManager().getTableStateManager().isTableState(
287 ht.getName(), ZooKeeperProtos.Table.State.ENABLED));
288
289
290 try {
291 ht.get(get);
292 } catch (RetriesExhaustedException e) {
293 ok = false;
294 }
295 assertTrue(ok);
296 ht.close();
297 }
298
299 @Test (timeout=300000)
300 public void testDisableAndEnableTables() throws IOException {
301 final byte [] row = Bytes.toBytes("row");
302 final byte [] qualifier = Bytes.toBytes("qualifier");
303 final byte [] value = Bytes.toBytes("value");
304 final TableName table1 = TableName.valueOf("testDisableAndEnableTable1");
305 final TableName table2 = TableName.valueOf("testDisableAndEnableTable2");
306 Table ht1 = TEST_UTIL.createTable(table1, HConstants.CATALOG_FAMILY);
307 Table ht2 = TEST_UTIL.createTable(table2, HConstants.CATALOG_FAMILY);
308 Put put = new Put(row);
309 put.add(HConstants.CATALOG_FAMILY, qualifier, value);
310 ht1.put(put);
311 ht2.put(put);
312 Get get = new Get(row);
313 get.addColumn(HConstants.CATALOG_FAMILY, qualifier);
314 ht1.get(get);
315 ht2.get(get);
316
317 this.admin.disableTables("testDisableAndEnableTable.*");
318
319
320 get = new Get(row);
321 get.addColumn(HConstants.CATALOG_FAMILY, qualifier);
322 boolean ok = false;
323 try {
324 ht1.get(get);
325 ht2.get(get);
326 } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
327 ok = true;
328 }
329
330 assertTrue(ok);
331 this.admin.enableTables("testDisableAndEnableTable.*");
332
333
334 try {
335 ht1.get(get);
336 } catch (IOException e) {
337 ok = false;
338 }
339 try {
340 ht2.get(get);
341 } catch (IOException e) {
342 ok = false;
343 }
344 assertTrue(ok);
345
346 ht1.close();
347 ht2.close();
348 }
349
350 @Test (timeout=300000)
351 public void testCreateTable() throws IOException {
352 HTableDescriptor [] tables = admin.listTables();
353 int numTables = tables.length;
354 TEST_UTIL.createTable(TableName.valueOf("testCreateTable"), HConstants.CATALOG_FAMILY).close();
355 tables = this.admin.listTables();
356 assertEquals(numTables + 1, tables.length);
357 assertTrue("Table must be enabled.", TEST_UTIL.getHBaseCluster()
358 .getMaster().getAssignmentManager().getTableStateManager().isTableState(
359 TableName.valueOf("testCreateTable"), ZooKeeperProtos.Table.State.ENABLED));
360 }
361
362 @Test (timeout=300000)
363 public void testTruncateTable() throws IOException {
364 testTruncateTable(TableName.valueOf("testTruncateTable"), false);
365 }
366
367 @Test (timeout=300000)
368 public void testTruncateTablePreservingSplits() throws IOException {
369 testTruncateTable(TableName.valueOf("testTruncateTablePreservingSplits"), true);
370 }
371
372 private void testTruncateTable(final TableName tableName, boolean preserveSplits)
373 throws IOException {
374 byte[][] splitKeys = new byte[2][];
375 splitKeys[0] = Bytes.toBytes(4);
376 splitKeys[1] = Bytes.toBytes(8);
377
378
379 HTable table = TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY, splitKeys);
380 try {
381 TEST_UTIL.loadNumericRows(table, HConstants.CATALOG_FAMILY, 0, 10);
382 assertEquals(10, TEST_UTIL.countRows(table));
383 } finally {
384 table.close();
385 }
386 assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
387
388
389 this.admin.disableTable(tableName);
390 this.admin.truncateTable(tableName, preserveSplits);
391 table = new HTable(TEST_UTIL.getConfiguration(), tableName);
392 try {
393 assertEquals(0, TEST_UTIL.countRows(table));
394 } finally {
395 table.close();
396 }
397 if (preserveSplits) {
398 assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
399 } else {
400 assertEquals(1, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
401 }
402 }
403
404 @Test (timeout=300000)
405 public void testGetTableDescriptor() throws IOException {
406 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
407 HColumnDescriptor fam2 = new HColumnDescriptor("fam2");
408 HColumnDescriptor fam3 = new HColumnDescriptor("fam3");
409 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("myTestTable"));
410 htd.addFamily(fam1);
411 htd.addFamily(fam2);
412 htd.addFamily(fam3);
413 this.admin.createTable(htd);
414 Table table = new HTable(TEST_UTIL.getConfiguration(), htd.getTableName());
415 HTableDescriptor confirmedHtd = table.getTableDescriptor();
416 assertEquals(htd.compareTo(confirmedHtd), 0);
417 table.close();
418 }
419
420 @Test (timeout=300000)
421 public void testCompactionTimestamps() throws Exception {
422 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
423 TableName tableName = TableName.valueOf("testCompactionTimestampsTable");
424 HTableDescriptor htd = new HTableDescriptor(tableName);
425 htd.addFamily(fam1);
426 this.admin.createTable(htd);
427 HTable table = (HTable)TEST_UTIL.getConnection().getTable(htd.getTableName());
428 long ts = this.admin.getLastMajorCompactionTimestamp(tableName);
429 assertEquals(0, ts);
430 Put p = new Put(Bytes.toBytes("row1"));
431 p.add(Bytes.toBytes("fam1"), Bytes.toBytes("fam1"), Bytes.toBytes("fam1"));
432 table.put(p);
433 ts = this.admin.getLastMajorCompactionTimestamp(tableName);
434
435 assertEquals(0, ts);
436
437 this.admin.flush(tableName);
438 ts = this.admin.getLastMajorCompactionTimestamp(tableName);
439
440 assertEquals(0, ts);
441
442 byte[] regionName =
443 table.getRegionLocator().getAllRegionLocations().get(0).getRegionInfo().getRegionName();
444 long ts1 = this.admin.getLastMajorCompactionTimestampForRegion(regionName);
445 assertEquals(ts, ts1);
446 p = new Put(Bytes.toBytes("row2"));
447 p.add(Bytes.toBytes("fam1"), Bytes.toBytes("fam1"), Bytes.toBytes("fam1"));
448 table.put(p);
449 this.admin.flush(tableName);
450 ts = this.admin.getLastMajorCompactionTimestamp(tableName);
451
452 assertEquals(ts1, ts);
453
454 TEST_UTIL.compact(tableName, true);
455 table.put(p);
456
457 this.admin.flush(tableName);
458 ts = this.admin.getLastMajorCompactionTimestamp(tableName);
459
460 assertTrue(ts > ts1);
461
462
463 ts1 = this.admin.getLastMajorCompactionTimestampForRegion(regionName);
464 assertEquals(ts, ts1);
465 table.put(p);
466 this.admin.flush(tableName);
467 ts = this.admin.getLastMajorCompactionTimestamp(tableName);
468 assertEquals(ts, ts1);
469 table.close();
470 }
471
472 @Test (timeout=300000)
473 public void testHColumnValidName() {
474 boolean exceptionThrown;
475 try {
476 new HColumnDescriptor("\\test\\abc");
477 } catch(IllegalArgumentException iae) {
478 exceptionThrown = true;
479 assertTrue(exceptionThrown);
480 }
481 }
482
483
484
485
486
487
488 @Test (timeout=300000)
489 public void testOnlineChangeTableSchema() throws IOException, InterruptedException {
490 final TableName tableName =
491 TableName.valueOf("changeTableSchemaOnline");
492 TEST_UTIL.getMiniHBaseCluster().getMaster().getConfiguration().setBoolean(
493 "hbase.online.schema.update.enable", true);
494 HTableDescriptor [] tables = admin.listTables();
495 int numTables = tables.length;
496 TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close();
497 tables = this.admin.listTables();
498 assertEquals(numTables + 1, tables.length);
499
500
501 HTableDescriptor htd = this.admin.getTableDescriptor(tableName);
502
503 HTableDescriptor copy = new HTableDescriptor(htd);
504 assertTrue(htd.equals(copy));
505
506 long newFlushSize = htd.getMemStoreFlushSize() / 2;
507 if (newFlushSize <=0) {
508 newFlushSize = HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE / 2;
509 }
510 copy.setMemStoreFlushSize(newFlushSize);
511 final String key = "anyoldkey";
512 assertTrue(htd.getValue(key) == null);
513 copy.setValue(key, key);
514 boolean expectedException = false;
515 try {
516 admin.modifyTable(tableName, copy);
517 } catch (TableNotDisabledException re) {
518 expectedException = true;
519 }
520 assertFalse(expectedException);
521 HTableDescriptor modifiedHtd = this.admin.getTableDescriptor(tableName);
522 assertFalse(htd.equals(modifiedHtd));
523 assertTrue(copy.equals(modifiedHtd));
524 assertEquals(newFlushSize, modifiedHtd.getMemStoreFlushSize());
525 assertEquals(key, modifiedHtd.getValue(key));
526
527
528 int countOfFamilies = modifiedHtd.getFamilies().size();
529 assertTrue(countOfFamilies > 0);
530 HColumnDescriptor hcd = modifiedHtd.getFamilies().iterator().next();
531 int maxversions = hcd.getMaxVersions();
532 final int newMaxVersions = maxversions + 1;
533 hcd.setMaxVersions(newMaxVersions);
534 final byte [] hcdName = hcd.getName();
535 expectedException = false;
536 try {
537 this.admin.modifyColumn(tableName, hcd);
538 } catch (TableNotDisabledException re) {
539 expectedException = true;
540 }
541 assertFalse(expectedException);
542 modifiedHtd = this.admin.getTableDescriptor(tableName);
543 HColumnDescriptor modifiedHcd = modifiedHtd.getFamily(hcdName);
544 assertEquals(newMaxVersions, modifiedHcd.getMaxVersions());
545
546
547 assertFalse(this.admin.isTableDisabled(tableName));
548 final String xtracolName = "xtracol";
549 HColumnDescriptor xtracol = new HColumnDescriptor(xtracolName);
550 xtracol.setValue(xtracolName, xtracolName);
551 expectedException = false;
552 try {
553 this.admin.addColumn(tableName, xtracol);
554 } catch (TableNotDisabledException re) {
555 expectedException = true;
556 }
557
558 assertFalse(expectedException);
559 modifiedHtd = this.admin.getTableDescriptor(tableName);
560 hcd = modifiedHtd.getFamily(xtracol.getName());
561 assertTrue(hcd != null);
562 assertTrue(hcd.getValue(xtracolName).equals(xtracolName));
563
564
565 this.admin.deleteColumn(tableName, xtracol.getName());
566 modifiedHtd = this.admin.getTableDescriptor(tableName);
567 hcd = modifiedHtd.getFamily(xtracol.getName());
568 assertTrue(hcd == null);
569
570
571 this.admin.disableTable(tableName);
572 this.admin.deleteTable(tableName);
573 this.admin.listTables();
574 assertFalse(this.admin.tableExists(tableName));
575 }
576
577 @Test (timeout=300000)
578 public void testShouldFailOnlineSchemaUpdateIfOnlineSchemaIsNotEnabled()
579 throws Exception {
580 final TableName tableName = TableName.valueOf("changeTableSchemaOnlineFailure");
581 TEST_UTIL.getMiniHBaseCluster().getMaster().getConfiguration().setBoolean(
582 "hbase.online.schema.update.enable", false);
583 HTableDescriptor[] tables = admin.listTables();
584 int numTables = tables.length;
585 TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close();
586 tables = this.admin.listTables();
587 assertEquals(numTables + 1, tables.length);
588
589
590 HTableDescriptor htd = this.admin.getTableDescriptor(tableName);
591
592 HTableDescriptor copy = new HTableDescriptor(htd);
593 assertTrue(htd.equals(copy));
594
595 long newFlushSize = htd.getMemStoreFlushSize() / 2;
596 if (newFlushSize <=0) {
597 newFlushSize = HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE / 2;
598 }
599 copy.setMemStoreFlushSize(newFlushSize);
600 final String key = "anyoldkey";
601 assertTrue(htd.getValue(key) == null);
602 copy.setValue(key, key);
603 boolean expectedException = false;
604 try {
605 admin.modifyTable(tableName, copy);
606 } catch (TableNotDisabledException re) {
607 expectedException = true;
608 }
609 assertTrue("Online schema update should not happen.", expectedException);
610
611
612 TEST_UTIL.getMiniHBaseCluster().getMaster().getConfiguration().setBoolean(
613 "hbase.online.schema.update.enable", true);
614 }
615
616
617
618
619 static class DoneListener implements EventHandler.EventHandlerListener {
620 private final AtomicBoolean done;
621
622 DoneListener(final AtomicBoolean done) {
623 super();
624 this.done = done;
625 }
626
627 @Override
628 public void afterProcess(EventHandler event) {
629 this.done.set(true);
630 synchronized (this.done) {
631
632 this.done.notifyAll();
633 }
634 }
635
636 @Override
637 public void beforeProcess(EventHandler event) {
638
639 }
640 }
641
642 @SuppressWarnings("deprecation")
643 protected void verifyRoundRobinDistribution(HTable ht, int expectedRegions) throws IOException {
644 int numRS = ht.getConnection().getCurrentNrHRS();
645 Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
646 Map<ServerName, List<HRegionInfo>> server2Regions = new HashMap<ServerName, List<HRegionInfo>>();
647 for (Map.Entry<HRegionInfo, ServerName> entry : regions.entrySet()) {
648 ServerName server = entry.getValue();
649 List<HRegionInfo> regs = server2Regions.get(server);
650 if (regs == null) {
651 regs = new ArrayList<HRegionInfo>();
652 server2Regions.put(server, regs);
653 }
654 regs.add(entry.getKey());
655 }
656 float average = (float) expectedRegions/numRS;
657 int min = (int)Math.floor(average);
658 int max = (int)Math.ceil(average);
659 for (List<HRegionInfo> regionList : server2Regions.values()) {
660 assertTrue(regionList.size() == min || regionList.size() == max);
661 }
662 }
663
664 @Test (timeout=300000)
665 public void testCreateTableNumberOfRegions() throws IOException, InterruptedException {
666 TableName tableName = TableName.valueOf("testCreateTableNumberOfRegions");
667 HTableDescriptor desc = new HTableDescriptor(tableName);
668 desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
669 admin.createTable(desc);
670 HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
671 Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
672 assertEquals("Table should have only 1 region", 1, regions.size());
673 ht.close();
674
675 TableName TABLE_2 = TableName.valueOf(tableName.getNameAsString() + "_2");
676 desc = new HTableDescriptor(TABLE_2);
677 desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
678 admin.createTable(desc, new byte[][]{new byte[]{42}});
679 HTable ht2 = new HTable(TEST_UTIL.getConfiguration(), TABLE_2);
680 regions = ht2.getRegionLocations();
681 assertEquals("Table should have only 2 region", 2, regions.size());
682 ht2.close();
683
684 TableName TABLE_3 = TableName.valueOf(tableName.getNameAsString() + "_3");
685 desc = new HTableDescriptor(TABLE_3);
686 desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
687 admin.createTable(desc, "a".getBytes(), "z".getBytes(), 3);
688 HTable ht3 = new HTable(TEST_UTIL.getConfiguration(), TABLE_3);
689 regions = ht3.getRegionLocations();
690 assertEquals("Table should have only 3 region", 3, regions.size());
691 ht3.close();
692
693 TableName TABLE_4 = TableName.valueOf(tableName.getNameAsString() + "_4");
694 desc = new HTableDescriptor(TABLE_4);
695 desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
696 try {
697 admin.createTable(desc, "a".getBytes(), "z".getBytes(), 2);
698 fail("Should not be able to create a table with only 2 regions using this API.");
699 } catch (IllegalArgumentException eae) {
700
701 }
702
703 TableName TABLE_5 = TableName.valueOf(tableName.getNameAsString() + "_5");
704 desc = new HTableDescriptor(TABLE_5);
705 desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
706 admin.createTable(desc, new byte[] {1}, new byte[] {127}, 16);
707 HTable ht5 = new HTable(TEST_UTIL.getConfiguration(), TABLE_5);
708 regions = ht5.getRegionLocations();
709 assertEquals("Table should have 16 region", 16, regions.size());
710 ht5.close();
711 }
712
713 @Test (timeout=300000)
714 public void testCreateTableWithRegions() throws IOException, InterruptedException {
715
716 TableName tableName = TableName.valueOf("testCreateTableWithRegions");
717
718 byte [][] splitKeys = {
719 new byte [] { 1, 1, 1 },
720 new byte [] { 2, 2, 2 },
721 new byte [] { 3, 3, 3 },
722 new byte [] { 4, 4, 4 },
723 new byte [] { 5, 5, 5 },
724 new byte [] { 6, 6, 6 },
725 new byte [] { 7, 7, 7 },
726 new byte [] { 8, 8, 8 },
727 new byte [] { 9, 9, 9 },
728 };
729 int expectedRegions = splitKeys.length + 1;
730
731 HTableDescriptor desc = new HTableDescriptor(tableName);
732 desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
733 admin.createTable(desc, splitKeys);
734
735 boolean tableAvailable = admin.isTableAvailable(tableName, splitKeys);
736 assertTrue("Table should be created with splitKyes + 1 rows in META", tableAvailable);
737
738 HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
739 Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
740 assertEquals("Tried to create " + expectedRegions + " regions " +
741 "but only found " + regions.size(),
742 expectedRegions, regions.size());
743 System.err.println("Found " + regions.size() + " regions");
744
745 Iterator<HRegionInfo> hris = regions.keySet().iterator();
746 HRegionInfo hri = hris.next();
747 assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0);
748 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[0]));
749 hri = hris.next();
750 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[0]));
751 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[1]));
752 hri = hris.next();
753 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[1]));
754 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[2]));
755 hri = hris.next();
756 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[2]));
757 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[3]));
758 hri = hris.next();
759 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[3]));
760 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[4]));
761 hri = hris.next();
762 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[4]));
763 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[5]));
764 hri = hris.next();
765 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[5]));
766 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[6]));
767 hri = hris.next();
768 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[6]));
769 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[7]));
770 hri = hris.next();
771 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[7]));
772 assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[8]));
773 hri = hris.next();
774 assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[8]));
775 assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0);
776
777 verifyRoundRobinDistribution(ht, expectedRegions);
778 ht.close();
779
780
781
782
783 byte [] startKey = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
784 byte [] endKey = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 };
785
786
787
788
789 expectedRegions = 10;
790
791 TableName TABLE_2 = TableName.valueOf(tableName.getNameAsString() + "_2");
792
793 desc = new HTableDescriptor(TABLE_2);
794 desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
795 admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
796 admin.createTable(desc, startKey, endKey, expectedRegions);
797
798 HTable ht2 = new HTable(TEST_UTIL.getConfiguration(), TABLE_2);
799 regions = ht2.getRegionLocations();
800 assertEquals("Tried to create " + expectedRegions + " regions " +
801 "but only found " + regions.size(),
802 expectedRegions, regions.size());
803 System.err.println("Found " + regions.size() + " regions");
804
805 hris = regions.keySet().iterator();
806 hri = hris.next();
807 assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0);
808 assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {1,1,1,1,1,1,1,1,1,1}));
809 hri = hris.next();
810 assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {1,1,1,1,1,1,1,1,1,1}));
811 assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {2,2,2,2,2,2,2,2,2,2}));
812 hri = hris.next();
813 assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {2,2,2,2,2,2,2,2,2,2}));
814 assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {3,3,3,3,3,3,3,3,3,3}));
815 hri = hris.next();
816 assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {3,3,3,3,3,3,3,3,3,3}));
817 assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {4,4,4,4,4,4,4,4,4,4}));
818 hri = hris.next();
819 assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {4,4,4,4,4,4,4,4,4,4}));
820 assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {5,5,5,5,5,5,5,5,5,5}));
821 hri = hris.next();
822 assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {5,5,5,5,5,5,5,5,5,5}));
823 assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {6,6,6,6,6,6,6,6,6,6}));
824 hri = hris.next();
825 assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {6,6,6,6,6,6,6,6,6,6}));
826 assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {7,7,7,7,7,7,7,7,7,7}));
827 hri = hris.next();
828 assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {7,7,7,7,7,7,7,7,7,7}));
829 assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {8,8,8,8,8,8,8,8,8,8}));
830 hri = hris.next();
831 assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {8,8,8,8,8,8,8,8,8,8}));
832 assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {9,9,9,9,9,9,9,9,9,9}));
833 hri = hris.next();
834 assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {9,9,9,9,9,9,9,9,9,9}));
835 assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0);
836
837 verifyRoundRobinDistribution(ht2, expectedRegions);
838 ht2.close();
839
840
841
842 startKey = new byte [] { 0, 0, 0, 0, 0, 0 };
843 endKey = new byte [] { 1, 0, 0, 0, 0, 0 };
844
845 expectedRegions = 5;
846
847 TableName TABLE_3 = TableName.valueOf(tableName.getNameAsString() + "_3");
848
849 desc = new HTableDescriptor(TABLE_3);
850 desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
851 admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
852 admin.createTable(desc, startKey, endKey, expectedRegions);
853
854
855 HTable ht3 = new HTable(TEST_UTIL.getConfiguration(), TABLE_3);
856 regions = ht3.getRegionLocations();
857 assertEquals("Tried to create " + expectedRegions + " regions " +
858 "but only found " + regions.size(),
859 expectedRegions, regions.size());
860 System.err.println("Found " + regions.size() + " regions");
861
862 verifyRoundRobinDistribution(ht3, expectedRegions);
863 ht3.close();
864
865
866
867 splitKeys = new byte [][] {
868 new byte [] { 1, 1, 1 },
869 new byte [] { 2, 2, 2 },
870 new byte [] { 3, 3, 3 },
871 new byte [] { 2, 2, 2 }
872 };
873
874 TableName TABLE_4 = TableName.valueOf(tableName.getNameAsString() + "_4");
875 desc = new HTableDescriptor(TABLE_4);
876 desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
877 Admin ladmin = new HBaseAdmin(TEST_UTIL.getConfiguration());
878 try {
879 ladmin.createTable(desc, splitKeys);
880 assertTrue("Should not be able to create this table because of " +
881 "duplicate split keys", false);
882 } catch(IllegalArgumentException iae) {
883
884 }
885 ladmin.close();
886 }
887
888 @Test (timeout=300000)
889 public void testTableAvailableWithRandomSplitKeys() throws Exception {
890 TableName tableName = TableName.valueOf("testTableAvailableWithRandomSplitKeys");
891 HTableDescriptor desc = new HTableDescriptor(tableName);
892 desc.addFamily(new HColumnDescriptor("col"));
893 byte[][] splitKeys = new byte[1][];
894 splitKeys = new byte [][] {
895 new byte [] { 1, 1, 1 },
896 new byte [] { 2, 2, 2 }
897 };
898 admin.createTable(desc);
899 boolean tableAvailable = admin.isTableAvailable(tableName, splitKeys);
900 assertFalse("Table should be created with 1 row in META", tableAvailable);
901 }
902
903 @Test (timeout=300000)
904 public void testCreateTableWithOnlyEmptyStartRow() throws IOException {
905 byte[] tableName = Bytes.toBytes("testCreateTableWithOnlyEmptyStartRow");
906 byte[][] splitKeys = new byte[1][];
907 splitKeys[0] = HConstants.EMPTY_BYTE_ARRAY;
908 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
909 desc.addFamily(new HColumnDescriptor("col"));
910 try {
911 admin.createTable(desc, splitKeys);
912 fail("Test case should fail as empty split key is passed.");
913 } catch (IllegalArgumentException e) {
914 }
915 }
916
917 @Test (timeout=300000)
918 public void testCreateTableWithEmptyRowInTheSplitKeys() throws IOException{
919 byte[] tableName = Bytes.toBytes("testCreateTableWithEmptyRowInTheSplitKeys");
920 byte[][] splitKeys = new byte[3][];
921 splitKeys[0] = "region1".getBytes();
922 splitKeys[1] = HConstants.EMPTY_BYTE_ARRAY;
923 splitKeys[2] = "region2".getBytes();
924 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
925 desc.addFamily(new HColumnDescriptor("col"));
926 try {
927 admin.createTable(desc, splitKeys);
928 fail("Test case should fail as empty split key is passed.");
929 } catch (IllegalArgumentException e) {
930 LOG.info("Expected ", e);
931 }
932 }
933
934 @Test (timeout=120000)
935 public void testTableExist() throws IOException {
936 final TableName table = TableName.valueOf("testTableExist");
937 boolean exist;
938 exist = this.admin.tableExists(table);
939 assertEquals(false, exist);
940 TEST_UTIL.createTable(table, HConstants.CATALOG_FAMILY);
941 exist = this.admin.tableExists(table);
942 assertEquals(true, exist);
943 }
944
945
946
947
948
949
950 @Test (timeout=400000)
951 public void testForceSplit() throws Exception {
952 byte[][] familyNames = new byte[][] { Bytes.toBytes("cf") };
953 int[] rowCounts = new int[] { 6000 };
954 int numVersions = HColumnDescriptor.DEFAULT_VERSIONS;
955 int blockSize = 256;
956 splitTest(null, familyNames, rowCounts, numVersions, blockSize);
957
958 byte[] splitKey = Bytes.toBytes(3500);
959 splitTest(splitKey, familyNames, rowCounts, numVersions, blockSize);
960 }
961
962
963
964
965
966
967 @Test (timeout=300000)
968 public void testEnableTableRetainAssignment() throws IOException {
969 final TableName tableName = TableName.valueOf("testEnableTableAssignment");
970 byte[][] splitKeys = { new byte[] { 1, 1, 1 }, new byte[] { 2, 2, 2 },
971 new byte[] { 3, 3, 3 }, new byte[] { 4, 4, 4 }, new byte[] { 5, 5, 5 },
972 new byte[] { 6, 6, 6 }, new byte[] { 7, 7, 7 }, new byte[] { 8, 8, 8 },
973 new byte[] { 9, 9, 9 } };
974 int expectedRegions = splitKeys.length + 1;
975 HTableDescriptor desc = new HTableDescriptor(tableName);
976 desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
977 admin.createTable(desc, splitKeys);
978 HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
979 Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
980 assertEquals("Tried to create " + expectedRegions + " regions "
981 + "but only found " + regions.size(), expectedRegions, regions.size());
982
983 admin.disableTable(tableName);
984
985 admin.enableTable(tableName);
986 Map<HRegionInfo, ServerName> regions2 = ht.getRegionLocations();
987
988
989 assertEquals(regions.size(), regions2.size());
990 for (Map.Entry<HRegionInfo, ServerName> entry : regions.entrySet()) {
991 assertEquals(regions2.get(entry.getKey()), entry.getValue());
992 }
993 }
994
995
996
997
998
999
1000
1001 @Test (timeout=800000)
1002 public void testForceSplitMultiFamily() throws Exception {
1003 int numVersions = HColumnDescriptor.DEFAULT_VERSIONS;
1004
1005
1006
1007
1008 int blockSize = 256;
1009 byte[][] familyNames = new byte[][] { Bytes.toBytes("cf1"),
1010 Bytes.toBytes("cf2") };
1011
1012
1013 int[] rowCounts = new int[] { 6000, 1 };
1014 splitTest(null, familyNames, rowCounts, numVersions, blockSize);
1015
1016 rowCounts = new int[] { 1, 6000 };
1017 splitTest(null, familyNames, rowCounts, numVersions, blockSize);
1018
1019
1020
1021 rowCounts = new int[] { 6000, 300 };
1022 splitTest(null, familyNames, rowCounts, numVersions, blockSize);
1023
1024 rowCounts = new int[] { 300, 6000 };
1025 splitTest(null, familyNames, rowCounts, numVersions, blockSize);
1026
1027 }
1028
1029 void splitTest(byte[] splitPoint, byte[][] familyNames, int[] rowCounts,
1030 int numVersions, int blockSize) throws Exception {
1031 TableName tableName = TableName.valueOf("testForceSplit");
1032 StringBuilder sb = new StringBuilder();
1033
1034 for (int i = 0; i < rowCounts.length; i++) {
1035 sb.append("_").append(Integer.toString(rowCounts[i]));
1036 }
1037 assertFalse(admin.tableExists(tableName));
1038 final HTable table = TEST_UTIL.createTable(tableName, familyNames,
1039 numVersions, blockSize);
1040
1041 int rowCount = 0;
1042 byte[] q = new byte[0];
1043
1044
1045
1046 for (int index = 0; index < familyNames.length; index++) {
1047 ArrayList<Put> puts = new ArrayList<Put>(rowCounts[index]);
1048 for (int i = 0; i < rowCounts[index]; i++) {
1049 byte[] k = Bytes.toBytes(i);
1050 Put put = new Put(k);
1051 put.add(familyNames[index], q, k);
1052 puts.add(put);
1053 }
1054 table.put(puts);
1055
1056 if ( rowCount < rowCounts[index] ) {
1057 rowCount = rowCounts[index];
1058 }
1059 }
1060
1061
1062 Map<HRegionInfo, ServerName> m = table.getRegionLocations();
1063 LOG.info("Initial regions (" + m.size() + "): " + m);
1064 assertTrue(m.size() == 1);
1065
1066
1067 Scan scan = new Scan();
1068 ResultScanner scanner = table.getScanner(scan);
1069 int rows = 0;
1070 for(@SuppressWarnings("unused") Result result : scanner) {
1071 rows++;
1072 }
1073 scanner.close();
1074 assertEquals(rowCount, rows);
1075
1076
1077 scan = new Scan();
1078 scanner = table.getScanner(scan);
1079
1080 scanner.next();
1081
1082
1083 this.admin.split(tableName, splitPoint);
1084
1085 final AtomicInteger count = new AtomicInteger(0);
1086 Thread t = new Thread("CheckForSplit") {
1087 @Override
1088 public void run() {
1089 for (int i = 0; i < 45; i++) {
1090 try {
1091 sleep(1000);
1092 } catch (InterruptedException e) {
1093 continue;
1094 }
1095
1096 Map<HRegionInfo, ServerName> regions = null;
1097 try {
1098 regions = table.getRegionLocations();
1099 } catch (IOException e) {
1100 e.printStackTrace();
1101 }
1102 if (regions == null) continue;
1103 count.set(regions.size());
1104 if (count.get() >= 2) {
1105 LOG.info("Found: " + regions);
1106 break;
1107 }
1108 LOG.debug("Cycle waiting on split");
1109 }
1110 LOG.debug("CheckForSplit thread exited, current region count: " + count.get());
1111 }
1112 };
1113 t.setPriority(Thread.NORM_PRIORITY - 2);
1114 t.start();
1115 t.join();
1116
1117
1118 rows = 1;
1119 for (@SuppressWarnings("unused") Result result : scanner) {
1120 rows++;
1121 if (rows > rowCount) {
1122 scanner.close();
1123 assertTrue("Scanned more than expected (" + rowCount + ")", false);
1124 }
1125 }
1126 scanner.close();
1127 assertEquals(rowCount, rows);
1128
1129 Map<HRegionInfo, ServerName> regions = null;
1130 try {
1131 regions = table.getRegionLocations();
1132 } catch (IOException e) {
1133 e.printStackTrace();
1134 }
1135 assertEquals(2, regions.size());
1136 Set<HRegionInfo> hRegionInfos = regions.keySet();
1137 HRegionInfo[] r = hRegionInfos.toArray(new HRegionInfo[hRegionInfos.size()]);
1138 if (splitPoint != null) {
1139
1140 assertEquals(Bytes.toString(splitPoint),
1141 Bytes.toString(r[0].getEndKey()));
1142 assertEquals(Bytes.toString(splitPoint),
1143 Bytes.toString(r[1].getStartKey()));
1144 LOG.debug("Properly split on " + Bytes.toString(splitPoint));
1145 } else {
1146 if (familyNames.length > 1) {
1147 int splitKey = Bytes.toInt(r[0].getEndKey());
1148
1149
1150 int deltaForLargestFamily = Math.abs(rowCount/2 - splitKey);
1151 LOG.debug("SplitKey=" + splitKey + "&deltaForLargestFamily=" + deltaForLargestFamily +
1152 ", r=" + r[0]);
1153 for (int index = 0; index < familyNames.length; index++) {
1154 int delta = Math.abs(rowCounts[index]/2 - splitKey);
1155 if (delta < deltaForLargestFamily) {
1156 assertTrue("Delta " + delta + " for family " + index
1157 + " should be at least deltaForLargestFamily " + deltaForLargestFamily,
1158 false);
1159 }
1160 }
1161 }
1162 }
1163 TEST_UTIL.deleteTable(tableName);
1164 table.close();
1165 }
1166
1167 @Test
1168 public void testSplitAndMergeWithReplicaTable() throws Exception {
1169
1170
1171
1172
1173 TableName tableName = TableName.valueOf("testSplitAndMergeWithReplicaTable");
1174 byte[] cf = "f".getBytes();
1175 createReplicaTable(tableName, cf);
1176 List<HRegion> oldRegions;
1177 do {
1178 oldRegions = TEST_UTIL.getHBaseCluster().getRegions(tableName);
1179 Thread.sleep(10);
1180 } while (oldRegions.size() != 9);
1181
1182 HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
1183 List<Put> puts = new ArrayList<Put>();
1184 byte[] qualifier = "c".getBytes();
1185 Put put = new Put(new byte[]{(byte)'1'});
1186 put.add(cf, qualifier, "100".getBytes());
1187 puts.add(put);
1188 put = new Put(new byte[]{(byte)'6'});
1189 put.add(cf, qualifier, "100".getBytes());
1190 puts.add(put);
1191 put = new Put(new byte[]{(byte)'8'});
1192 put.add(cf, qualifier, "100".getBytes());
1193 puts.add(put);
1194 ht.put(puts);
1195 ht.flushCommits();
1196 ht.close();
1197 List<Pair<HRegionInfo, ServerName>> regions =
1198 MetaTableAccessor.getTableRegionsAndLocations(TEST_UTIL.getZooKeeperWatcher(),
1199 TEST_UTIL.getConnection(), tableName);
1200 boolean gotException = false;
1201
1202
1203 try {
1204 TEST_UTIL.getHBaseAdmin().split(regions.get(1).getFirst().getRegionName());
1205 } catch (IllegalArgumentException ex) {
1206 gotException = true;
1207 }
1208 assertTrue(gotException);
1209 gotException = false;
1210
1211
1212
1213 try {
1214 TEST_UTIL.getHBaseAdmin().split(regions.get(1).getSecond(), regions.get(1).getFirst(),
1215 new byte[]{(byte)'1'});
1216 } catch (IOException ex) {
1217 gotException = true;
1218 }
1219 assertTrue(gotException);
1220 gotException = false;
1221
1222 try {
1223 TEST_UTIL.getHBaseAdmin().mergeRegions(regions.get(1).getFirst().getEncodedNameAsBytes(),
1224 regions.get(2).getFirst().getEncodedNameAsBytes(), true);
1225 } catch (IllegalArgumentException m) {
1226 gotException = true;
1227 }
1228 assertTrue(gotException);
1229
1230 try {
1231 DispatchMergingRegionsRequest request = RequestConverter
1232 .buildDispatchMergingRegionsRequest(regions.get(1).getFirst().getEncodedNameAsBytes(),
1233 regions.get(2).getFirst().getEncodedNameAsBytes(), true);
1234 TEST_UTIL.getHBaseAdmin().getConnection().getMaster().dispatchMergingRegions(null, request);
1235 } catch (ServiceException m) {
1236 Throwable t = m.getCause();
1237 do {
1238 if (t instanceof MergeRegionException) {
1239 gotException = true;
1240 break;
1241 }
1242 t = t.getCause();
1243 } while (t != null);
1244 }
1245 assertTrue(gotException);
1246 gotException = false;
1247
1248
1249 if (!regions.get(2).getSecond().equals(regions.get(1).getSecond())) {
1250 moveRegionAndWait(regions.get(2).getFirst(), regions.get(1).getSecond());
1251 }
1252 try {
1253 AdminService.BlockingInterface admin = TEST_UTIL.getHBaseAdmin().getConnection()
1254 .getAdmin(regions.get(1).getSecond());
1255 ProtobufUtil.mergeRegions(null, admin, regions.get(1).getFirst(), regions.get(2).getFirst(),
1256 true, null);
1257 } catch (MergeRegionException mm) {
1258 gotException = true;
1259 }
1260 assertTrue(gotException);
1261 }
1262
1263 private void moveRegionAndWait(HRegionInfo destRegion, ServerName destServer)
1264 throws InterruptedException, MasterNotRunningException,
1265 ZooKeeperConnectionException, IOException {
1266 HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
1267 TEST_UTIL.getHBaseAdmin().move(
1268 destRegion.getEncodedNameAsBytes(),
1269 Bytes.toBytes(destServer.getServerName()));
1270 while (true) {
1271 ServerName serverName = master.getAssignmentManager()
1272 .getRegionStates().getRegionServerOfRegion(destRegion);
1273 if (serverName != null && serverName.equals(destServer)) {
1274 TEST_UTIL.assertRegionOnServer(
1275 destRegion, serverName, 200);
1276 break;
1277 }
1278 Thread.sleep(10);
1279 }
1280 }
1281
1282
1283
1284
1285
1286 @Test
1287 public void testRegionStateCleanupFromAMMemoryAfterRegionSplitAndMerge() throws Exception {
1288 final TableName tableName =
1289 TableName.valueOf("testRegionStateCleanupFromAMMemoryAfterRegionSplitAndMerge");
1290 createReplicaTable(tableName, "f".getBytes());
1291 final int regionReplication = admin.getTableDescriptor(tableName).getRegionReplication();
1292
1293 List<Pair<HRegionInfo, ServerName>> regions = MetaTableAccessor.getTableRegionsAndLocations(
1294 TEST_UTIL.getZooKeeperWatcher(), TEST_UTIL.getConnection(), tableName);
1295 assertEquals(9, regions.size());
1296 final int primaryRegionCount = regions.size() / regionReplication;
1297
1298 final AssignmentManager am = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager();
1299 List<HRegionInfo> splitRegions =
1300 am.getRegionStates().getRegionByStateOfTable(tableName).get(RegionState.State.SPLIT);
1301 assertEquals(0, splitRegions.size());
1302
1303
1304 byte[] regionName = regions.get(0).getFirst().getRegionName();
1305 try {
1306 TEST_UTIL.getHBaseAdmin().split(regionName, Bytes.toBytes('2'));
1307 } catch (IllegalArgumentException ex) {
1308 fail("Exception occured during region split" + ex);
1309 }
1310
1311
1312 TEST_UTIL.waitFor(60000, 500, new Waiter.Predicate<IOException>() {
1313 @Override
1314 public boolean evaluate() throws IOException {
1315 return am.getRegionStates().getRegionByStateOfTable(tableName).get(RegionState.State.OPEN)
1316 .size() == (primaryRegionCount + 1) * regionReplication;
1317 }
1318 });
1319
1320 regions = MetaTableAccessor.getTableRegionsAndLocations(TEST_UTIL.getZooKeeperWatcher(),
1321 TEST_UTIL.getConnection(), tableName);
1322 assertEquals(12, regions.size());
1323 final int primaryRegionCountAfterSplit = regions.size() / regionReplication;
1324
1325
1326 splitRegions =
1327 am.getRegionStates().getRegionByStateOfTable(tableName).get(RegionState.State.SPLIT);
1328
1329 assertEquals(1, splitRegions.size());
1330
1331
1332 HRegionInfo regionA = regions.get(3).getFirst();
1333 HRegionInfo regionB = regions.get(6).getFirst();
1334 try {
1335 TEST_UTIL.getHBaseAdmin().mergeRegions(regionA.getRegionName(), regionB.getRegionName(),
1336 true);
1337 } catch (IllegalArgumentException ex) {
1338 fail("Exception occured during region merge" + ex);
1339 }
1340
1341
1342 TEST_UTIL.waitFor(60000, 500, new Waiter.Predicate<IOException>() {
1343 @Override
1344 public boolean evaluate() throws IOException {
1345 return am.getRegionStates().getRegionByStateOfTable(tableName).get(RegionState.State.OPEN)
1346 .size() == (primaryRegionCountAfterSplit - 1) * regionReplication;
1347 }
1348 });
1349
1350 regions = MetaTableAccessor.getTableRegionsAndLocations(TEST_UTIL.getZooKeeperWatcher(),
1351 TEST_UTIL.getConnection(), tableName);
1352 assertEquals(9, regions.size());
1353
1354 List<HRegionInfo> offlineRegions =
1355 am.getRegionStates().getRegionByStateOfTable(tableName).get(RegionState.State.OFFLINE);
1356
1357 assertEquals(0, offlineRegions.size());
1358 }
1359
1360 private byte[] createReplicaTable(TableName tableName, byte[] cf) throws IOException {
1361 HTableDescriptor desc = new HTableDescriptor(tableName);
1362 desc.setRegionReplication(3);
1363 HColumnDescriptor hcd = new HColumnDescriptor(cf);
1364 desc.addFamily(hcd);
1365 byte[][] splitRows = new byte[2][];
1366 splitRows[0] = new byte[] { (byte) '4' };
1367 splitRows[1] = new byte[] { (byte) '7' };
1368 TEST_UTIL.getHBaseAdmin().createTable(desc, splitRows);
1369 return cf;
1370 }
1371
1372
1373
1374
1375
1376 @SuppressWarnings("deprecation")
1377 @Test (expected=IllegalArgumentException.class, timeout=300000)
1378 public void testEmptyHTableDescriptor() throws IOException {
1379 this.admin.createTable(new HTableDescriptor());
1380 }
1381
1382 @Test (expected=IllegalArgumentException.class, timeout=300000)
1383 public void testInvalidHColumnDescriptor() throws IOException {
1384 new HColumnDescriptor("/cfamily/name");
1385 }
1386
1387 @Test (timeout=300000)
1388 public void testEnableDisableAddColumnDeleteColumn() throws Exception {
1389 ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
1390 TableName tableName = TableName.valueOf("testEnableDisableAddColumnDeleteColumn");
1391 TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close();
1392 while (!ZKTableStateClientSideReader.isEnabledTable(zkw,
1393 TableName.valueOf("testEnableDisableAddColumnDeleteColumn"))) {
1394 Thread.sleep(10);
1395 }
1396 this.admin.disableTable(tableName);
1397 try {
1398 new HTable(TEST_UTIL.getConfiguration(), tableName);
1399 } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
1400
1401 }
1402
1403 this.admin.addColumn(tableName, new HColumnDescriptor("col2"));
1404 this.admin.enableTable(tableName);
1405 try {
1406 this.admin.deleteColumn(tableName, Bytes.toBytes("col2"));
1407 } catch (TableNotDisabledException e) {
1408 LOG.info(e);
1409 }
1410 this.admin.disableTable(tableName);
1411 this.admin.deleteTable(tableName);
1412 }
1413
1414 @Test (timeout=300000)
1415 public void testDeleteLastColumnFamily() throws Exception {
1416 TableName tableName = TableName.valueOf("testDeleteLastColumnFamily");
1417 TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close();
1418 while (!this.admin.isTableEnabled(TableName.valueOf("testDeleteLastColumnFamily"))) {
1419 Thread.sleep(10);
1420 }
1421
1422
1423 try {
1424 this.admin.deleteColumn(tableName, HConstants.CATALOG_FAMILY);
1425 fail("Should have failed to delete the only column family of a table");
1426 } catch (InvalidFamilyOperationException ex) {
1427
1428 }
1429
1430
1431 this.admin.disableTable(tableName);
1432
1433 try {
1434 this.admin.deleteColumn(tableName, HConstants.CATALOG_FAMILY);
1435 fail("Should have failed to delete the only column family of a table");
1436 } catch (InvalidFamilyOperationException ex) {
1437
1438 }
1439
1440 this.admin.deleteTable(tableName);
1441 }
1442
1443
1444
1445
1446
1447 @Test(timeout = 300000)
1448 public void testHFileReplication() throws Exception {
1449 TableName name = TableName.valueOf("testHFileReplication");
1450 String fn1 = "rep1";
1451 HColumnDescriptor hcd1 = new HColumnDescriptor(fn1);
1452 hcd1.setDFSReplication((short) 1);
1453 String fn = "defaultRep";
1454 HColumnDescriptor hcd = new HColumnDescriptor(fn);
1455 HTableDescriptor htd = new HTableDescriptor(name);
1456 htd.addFamily(hcd);
1457 htd.addFamily(hcd1);
1458 Table table = TEST_UTIL.createTable(htd, null);
1459 TEST_UTIL.waitTableAvailable(name);
1460 Put p = new Put(Bytes.toBytes("defaultRep_rk"));
1461 byte[] q1 = Bytes.toBytes("q1");
1462 byte[] v1 = Bytes.toBytes("v1");
1463 p.addColumn(Bytes.toBytes(fn), q1, v1);
1464 List<Put> puts = new ArrayList<Put>(2);
1465 puts.add(p);
1466 p = new Put(Bytes.toBytes("rep1_rk"));
1467 p.addColumn(Bytes.toBytes(fn1), q1, v1);
1468 puts.add(p);
1469 try {
1470 table.put(puts);
1471 admin.flush(name);
1472
1473 List<HRegion> regions = TEST_UTIL.getMiniHBaseCluster().getRegions(name);
1474 for (HRegion r : regions) {
1475 Store store = r.getStore(Bytes.toBytes(fn));
1476 for (StoreFile sf : store.getStorefiles()) {
1477 assertTrue(sf.toString().contains(fn));
1478 assertTrue("Column family " + fn + " should have 3 copies",
1479 FSUtils.getDefaultReplication(TEST_UTIL.getTestFileSystem(), sf.getPath()) == (sf
1480 .getFileInfo().getFileStatus().getReplication()));
1481 }
1482
1483 store = r.getStore(Bytes.toBytes(fn1));
1484 for (StoreFile sf : store.getStorefiles()) {
1485 assertTrue(sf.toString().contains(fn1));
1486 assertTrue("Column family " + fn1 + " should have only 1 copy", 1 == sf.getFileInfo()
1487 .getFileStatus().getReplication());
1488 }
1489 }
1490 } finally {
1491 if (admin.isTableEnabled(name)) {
1492 this.admin.disableTable(name);
1493 this.admin.deleteTable(name);
1494 }
1495 }
1496 }
1497
1498 @Test
1499 public void testMergeRegions() throws Exception {
1500 TableName tableName = TableName.valueOf("testMergeWithFullRegionName");
1501 HColumnDescriptor cd = new HColumnDescriptor("d");
1502 HTableDescriptor td = new HTableDescriptor(tableName);
1503 td.addFamily(cd);
1504 byte[][] splitRows = new byte[2][];
1505 splitRows[0] = new byte[]{(byte)'3'};
1506 splitRows[1] = new byte[]{(byte)'6'};
1507 try {
1508 TEST_UTIL.createTable(td, splitRows);
1509 TEST_UTIL.waitTableAvailable(tableName);
1510
1511 List<HRegionInfo> tableRegions;
1512 HRegionInfo regionA;
1513 HRegionInfo regionB;
1514
1515
1516 tableRegions = admin.getTableRegions(tableName);
1517 assertEquals(3, admin.getTableRegions(tableName).size());
1518 regionA = tableRegions.get(0);
1519 regionB = tableRegions.get(1);
1520 admin.mergeRegions(regionA.getRegionName(), regionB.getRegionName(), false);
1521 Thread.sleep(1000);
1522 assertEquals(2, admin.getTableRegions(tableName).size());
1523
1524
1525 tableRegions = admin.getTableRegions(tableName);
1526 regionA = tableRegions.get(0);
1527 regionB = tableRegions.get(1);
1528 admin.mergeRegions(regionA.getEncodedNameAsBytes(), regionB.getEncodedNameAsBytes(), false);
1529 Thread.sleep(1000);
1530 assertEquals(1, admin.getTableRegions(tableName).size());
1531 } finally {
1532 this.admin.disableTable(tableName);
1533 this.admin.deleteTable(tableName);
1534 }
1535 }
1536 }