1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertNotSame;
25 import static org.junit.Assert.assertNull;
26 import static org.junit.Assert.assertTrue;
27 import static org.junit.Assert.fail;
28
29 import java.io.IOException;
30 import java.io.InterruptedIOException;
31 import java.util.Collection;
32 import java.util.List;
33 import java.util.Map;
34 import java.util.concurrent.CountDownLatch;
35
36 import org.apache.commons.logging.Log;
37 import org.apache.commons.logging.LogFactory;
38 import org.apache.hadoop.conf.Configuration;
39 import org.apache.hadoop.fs.FileSystem;
40 import org.apache.hadoop.fs.Path;
41 import org.apache.hadoop.hbase.Abortable;
42 import org.apache.hadoop.hbase.CoordinatedStateManager;
43 import org.apache.hadoop.hbase.Coprocessor;
44 import org.apache.hadoop.hbase.CoprocessorEnvironment;
45 import org.apache.hadoop.hbase.HBaseTestingUtility;
46 import org.apache.hadoop.hbase.HColumnDescriptor;
47 import org.apache.hadoop.hbase.HConstants;
48 import org.apache.hadoop.hbase.HRegionInfo;
49 import org.apache.hadoop.hbase.HTableDescriptor;
50 import org.apache.hadoop.hbase.MasterNotRunningException;
51 import org.apache.hadoop.hbase.MetaTableAccessor;
52 import org.apache.hadoop.hbase.MiniHBaseCluster;
53 import org.apache.hadoop.hbase.RegionTransition;
54 import org.apache.hadoop.hbase.Server;
55 import org.apache.hadoop.hbase.ServerName;
56 import org.apache.hadoop.hbase.TableName;
57 import org.apache.hadoop.hbase.UnknownRegionException;
58 import org.apache.hadoop.hbase.Waiter;
59 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
60 import org.apache.hadoop.hbase.client.Admin;
61 import org.apache.hadoop.hbase.client.Connection;
62 import org.apache.hadoop.hbase.client.ConnectionFactory;
63 import org.apache.hadoop.hbase.client.Consistency;
64 import org.apache.hadoop.hbase.client.Delete;
65 import org.apache.hadoop.hbase.client.Get;
66 import org.apache.hadoop.hbase.client.HBaseAdmin;
67 import org.apache.hadoop.hbase.client.HTable;
68 import org.apache.hadoop.hbase.client.Mutation;
69 import org.apache.hadoop.hbase.client.Put;
70 import org.apache.hadoop.hbase.client.Result;
71 import org.apache.hadoop.hbase.client.ResultScanner;
72 import org.apache.hadoop.hbase.client.Scan;
73 import org.apache.hadoop.hbase.client.Table;
74 import org.apache.hadoop.hbase.client.TestReplicasClient.SlowMeCopro;
75 import org.apache.hadoop.hbase.coordination.ZKSplitTransactionCoordination;
76 import org.apache.hadoop.hbase.coordination.ZkCloseRegionCoordination;
77 import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
78 import org.apache.hadoop.hbase.coordination.ZkOpenRegionCoordination;
79 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
80 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
81 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
82 import org.apache.hadoop.hbase.exceptions.DeserializationException;
83 import org.apache.hadoop.hbase.executor.EventType;
84 import org.apache.hadoop.hbase.master.AssignmentManager;
85 import org.apache.hadoop.hbase.master.HMaster;
86 import org.apache.hadoop.hbase.master.RegionState;
87 import org.apache.hadoop.hbase.master.RegionState.State;
88 import org.apache.hadoop.hbase.master.RegionStates;
89 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
90 import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
91 import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
92 import org.apache.hadoop.hbase.testclassification.LargeTests;
93 import org.apache.hadoop.hbase.util.Bytes;
94 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
95 import org.apache.hadoop.hbase.util.FSUtils;
96 import org.apache.hadoop.hbase.util.HBaseFsck;
97 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
98 import org.apache.hadoop.hbase.util.PairOfSameType;
99 import org.apache.hadoop.hbase.util.Threads;
100 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
101 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
102 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
103 import org.apache.zookeeper.KeeperException;
104 import org.apache.zookeeper.KeeperException.NodeExistsException;
105 import org.apache.zookeeper.data.Stat;
106 import org.junit.After;
107 import org.junit.AfterClass;
108 import org.junit.Assert;
109 import org.junit.Before;
110 import org.junit.BeforeClass;
111 import org.junit.Test;
112 import org.junit.experimental.categories.Category;
113
114 import com.google.protobuf.ServiceException;
115
116
117
118
119
120
121 @Category(LargeTests.class)
122 @SuppressWarnings("deprecation")
123 public class TestSplitTransactionOnCluster {
124 private static final Log LOG =
125 LogFactory.getLog(TestSplitTransactionOnCluster.class);
126 private HBaseAdmin admin = null;
127 private MiniHBaseCluster cluster = null;
128 private static final int NB_SERVERS = 3;
129 private static CountDownLatch latch = new CountDownLatch(1);
130 private static volatile boolean secondSplit = false;
131 private static volatile boolean callRollBack = false;
132 private static volatile boolean firstSplitCompleted = false;
133 private static boolean useZKForAssignment;
134
135 static final HBaseTestingUtility TESTING_UTIL =
136 new HBaseTestingUtility();
137
138 static void setupOnce() throws Exception {
139 TESTING_UTIL.getConfiguration().setInt("hbase.balancer.period", 60000);
140 useZKForAssignment = TESTING_UTIL.getConfiguration().getBoolean(
141 "hbase.assignment.usezk", true);
142 TESTING_UTIL.startMiniCluster(NB_SERVERS);
143 }
144
145 @BeforeClass public static void before() throws Exception {
146
147 TESTING_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
148 setupOnce();
149 }
150
151 @AfterClass public static void after() throws Exception {
152 TESTING_UTIL.shutdownMiniCluster();
153 }
154
155 @Before public void setup() throws IOException {
156 TESTING_UTIL.ensureSomeNonStoppedRegionServersAvailable(NB_SERVERS);
157 this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
158 this.cluster = TESTING_UTIL.getMiniHBaseCluster();
159 }
160
161 @After
162 public void tearDown() throws Exception {
163 this.admin.close();
164 }
165
166 private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions)
167 throws IOException, InterruptedException {
168 assertEquals(1, regions.size());
169 HRegionInfo hri = regions.get(0).getRegionInfo();
170 TESTING_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
171 .waitOnRegionToClearRegionsInTransition(hri, 600000);
172 return hri;
173 }
174
175 @Test(timeout = 60000)
176 public void testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack() throws Exception {
177 final TableName tableName =
178 TableName.valueOf("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack");
179
180 if (!useZKForAssignment) {
181
182 return;
183 }
184
185 try {
186
187 HTable t = createTableAndWait(tableName, Bytes.toBytes("cf"));
188 final List<HRegion> regions = cluster.getRegions(tableName);
189 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
190 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionInfo()
191 .getRegionName());
192 final HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
193 insertData(tableName, admin, t);
194 t.close();
195
196
197 this.admin.setBalancerRunning(false, true);
198
199 cluster.getMaster().setCatalogJanitorEnabled(false);
200
201
202 final HRegion region = findSplittableRegion(regions);
203 assertTrue("not able to find a splittable region", region != null);
204 MockedCoordinatedStateManager cp = new MockedCoordinatedStateManager();
205 cp.initialize(regionServer, region);
206 cp.start();
207 regionServer.csm = cp;
208
209 new Thread() {
210 @Override
211 public void run() {
212 SplitTransaction st = null;
213 st = new MockedSplitTransaction(region, Bytes.toBytes("row2"));
214 try {
215 st.prepare();
216 st.execute(regionServer, regionServer);
217 } catch (IOException e) {
218
219 }
220 }
221 }.start();
222 for (int i = 0; !callRollBack && i < 100; i++) {
223 Thread.sleep(100);
224 }
225 assertTrue("Waited too long for rollback", callRollBack);
226 SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row3"));
227 try {
228 secondSplit = true;
229
230 region.initialize();
231 st.prepare();
232 st.execute(regionServer, regionServer);
233 } catch (IOException e) {
234 LOG.debug("Rollback started :"+ e.getMessage());
235 st.rollback(regionServer, regionServer);
236 }
237 for (int i=0; !firstSplitCompleted && i<100; i++) {
238 Thread.sleep(100);
239 }
240 assertTrue("fist split did not complete", firstSplitCompleted);
241
242 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
243 Map<String, RegionState> rit = regionStates.getRegionsInTransition();
244
245 for (int i=0; rit.containsKey(hri.getTable()) && i<100; i++) {
246 Thread.sleep(100);
247 }
248 assertFalse("region still in transition", rit.containsKey(
249 rit.containsKey(hri.getTable())));
250
251 List<Region> onlineRegions = regionServer.getOnlineRegions(tableName);
252
253 assertEquals("The parent region should be splitted", 2, onlineRegions.size());
254
255 List<HRegionInfo> regionsOfTable = cluster.getMaster().getAssignmentManager()
256 .getRegionStates().getRegionsOfTable(tableName);
257
258 assertEquals("No of regions in master", 2, regionsOfTable.size());
259 } finally {
260 admin.setBalancerRunning(true, false);
261 secondSplit = false;
262 firstSplitCompleted = false;
263 callRollBack = false;
264 cluster.getMaster().setCatalogJanitorEnabled(true);
265 TESTING_UTIL.deleteTable(tableName);
266 }
267 }
268
269 @Test(timeout = 60000)
270 public void testRITStateForRollback() throws Exception {
271 final TableName tableName =
272 TableName.valueOf("testRITStateForRollback");
273 try {
274
275 Table t = createTableAndWait(tableName, Bytes.toBytes("cf"));
276 final List<HRegion> regions = cluster.getRegions(tableName);
277 final HRegionInfo hri = getAndCheckSingleTableRegion(regions);
278 insertData(tableName, admin, t);
279 t.close();
280
281
282 this.admin.setBalancerRunning(false, true);
283
284 cluster.getMaster().setCatalogJanitorEnabled(false);
285
286
287 final HRegion region = findSplittableRegion(regions);
288 assertTrue("not able to find a splittable region", region != null);
289
290
291 region.getCoprocessorHost().load(FailingSplitRegionObserver.class,
292 Coprocessor.PRIORITY_USER, region.getBaseConf());
293
294
295 this.admin.split(region.getRegionInfo().getRegionName(), new byte[] {42});
296
297
298 FailingSplitRegionObserver observer = (FailingSplitRegionObserver) region
299 .getCoprocessorHost().findCoprocessor(FailingSplitRegionObserver.class.getName());
300 assertNotNull(observer);
301 observer.latch.await();
302
303 LOG.info("Waiting for region to come out of RIT");
304 TESTING_UTIL.waitUntilNoRegionsInTransition(60000);
305 } finally {
306 admin.setBalancerRunning(true, false);
307 cluster.getMaster().setCatalogJanitorEnabled(true);
308 TESTING_UTIL.deleteTable(tableName);
309 }
310 }
311 @Test(timeout = 60000)
312 public void testSplitFailedCompactionAndSplit() throws Exception {
313 final TableName tableName = TableName.valueOf("testSplitFailedCompactionAndSplit");
314 Configuration conf = TESTING_UTIL.getConfiguration();
315 try {
316 HBaseAdmin admin = new HBaseAdmin(conf);
317
318 HTableDescriptor htd = new HTableDescriptor(tableName);
319 byte[] cf = Bytes.toBytes("cf");
320 htd.addFamily(new HColumnDescriptor(cf));
321 admin.createTable(htd);
322
323 for (int i = 0; cluster.getRegions(tableName).size() == 0 && i < 100; i++) {
324 Thread.sleep(100);
325 }
326 assertEquals(1, cluster.getRegions(tableName).size());
327
328 HRegion region = cluster.getRegions(tableName).get(0);
329 Store store = region.getStore(cf);
330 int regionServerIndex = cluster.getServerWith(region.getRegionInfo().getRegionName());
331 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
332
333 Table t = new HTable(conf, tableName);
334
335 insertData(tableName, admin, t);
336 insertData(tableName, admin, t);
337
338 int fileNum = store.getStorefiles().size();
339
340 store.triggerMajorCompaction();
341 CompactionContext cc = store.requestCompaction();
342 assertNotNull(cc);
343
344
345 assertEquals(2, region.close(false).get(cf).size());
346
347 region.initialize();
348
349
350 assertFalse(region.compact(cc, store, NoLimitThroughputController.INSTANCE));
351 assertTrue(fileNum > store.getStorefiles().size());
352
353
354 SplitTransaction st = new SplitTransactionImpl(region, Bytes.toBytes("row3"));
355 assertTrue(st.prepare());
356 st.execute(regionServer, regionServer);
357 LOG.info("Waiting for region to come out of RIT");
358 TESTING_UTIL.waitFor(60000, 1000, new Waiter.Predicate<Exception>() {
359 @Override
360 public boolean evaluate() throws Exception {
361 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
362 Map<String, RegionState> rit = regionStates.getRegionsInTransition();
363 return (rit.size() == 0);
364 }
365 });
366 assertEquals(2, cluster.getRegions(tableName).size());
367 } finally {
368 TESTING_UTIL.deleteTable(tableName);
369 }
370 }
371
372 public static class FailingSplitRegionObserver extends BaseRegionObserver {
373 volatile CountDownLatch latch;
374 volatile CountDownLatch postSplit;
375 @Override
376 public void start(CoprocessorEnvironment e) throws IOException {
377 latch = new CountDownLatch(1);
378 postSplit = new CountDownLatch(1);
379 }
380 @Override
381 public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
382 byte[] splitKey, List<Mutation> metaEntries) throws IOException {
383 latch.countDown();
384 LOG.info("Causing rollback of region split");
385 throw new IOException("Causing rollback of region split");
386 }
387 @Override
388 public void postCompleteSplit(ObserverContext<RegionCoprocessorEnvironment> ctx)
389 throws IOException {
390 postSplit.countDown();
391 LOG.info("postCompleteSplit called");
392 }
393 }
394
395
396
397
398
399
400
401
402
403
404
405
406 @Test (timeout = 300000) public void testRSSplitEphemeralsDisappearButDaughtersAreOnlinedAfterShutdownHandling()
407 throws IOException, InterruptedException, NodeExistsException, KeeperException,
408 DeserializationException, ServiceException {
409 final TableName tableName =
410 TableName.valueOf("testRSSplitEphemeralsDisappearButDaughtersAreOnlinedAfterShutdownHandling");
411
412
413 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
414 List<HRegion> regions = cluster.getRegions(tableName);
415 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
416
417 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
418
419
420 this.admin.setBalancerRunning(false, true);
421
422 cluster.getMaster().setCatalogJanitorEnabled(false);
423 try {
424
425 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
426
427 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
428 printOutRegions(server, "Initial regions: ");
429 int regionCount = ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size();
430
431
432 AssignmentManager.TEST_SKIP_SPLIT_HANDLING = true;
433
434 split(hri, server, regionCount);
435
436 String path = ZKAssign.getNodeName(TESTING_UTIL.getZooKeeperWatcher(),
437 hri.getEncodedName());
438 RegionTransition rt = null;
439 Stat stats = null;
440 List<HRegion> daughters = null;
441 if (useZKForAssignment) {
442 daughters = checkAndGetDaughters(tableName);
443
444
445 for (int i=0; i<100; i++) {
446 stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false);
447 rt = RegionTransition.parseFrom(ZKAssign.getData(TESTING_UTIL.getZooKeeperWatcher(),
448 hri.getEncodedName()));
449 if (rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)) break;
450 Thread.sleep(100);
451 }
452 LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats=" + stats);
453 assertTrue(rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT));
454
455 cluster.abortRegionServer(tableRegionIndex);
456 }
457 waitUntilRegionServerDead();
458 awaitDaughters(tableName, 2);
459 if (useZKForAssignment) {
460 regions = cluster.getRegions(tableName);
461 for (HRegion r: regions) {
462 assertTrue(daughters.contains(r));
463 }
464
465
466 for (int i=0; i<100; i++) {
467
468 stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false);
469 if (stats == null) break;
470 Thread.sleep(100);
471 }
472 LOG.info("EPHEMERAL NODE AFTER SERVER ABORT, path=" + path + ", stats=" + stats);
473 assertTrue(stats == null);
474 }
475 } finally {
476
477 AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
478 cluster.getMaster().getAssignmentManager().regionOffline(hri);
479 admin.setBalancerRunning(true, false);
480 cluster.getMaster().setCatalogJanitorEnabled(true);
481 cluster.startRegionServer();
482 t.close();
483 TESTING_UTIL.deleteTable(tableName);
484 }
485 }
486
487 @Test (timeout = 300000) public void testExistingZnodeBlocksSplitAndWeRollback()
488 throws IOException, InterruptedException, NodeExistsException, KeeperException, ServiceException {
489 final TableName tableName =
490 TableName.valueOf("testExistingZnodeBlocksSplitAndWeRollback");
491
492
493 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
494 List<HRegion> regions = cluster.getRegions(tableName);
495 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
496
497 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
498
499 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
500
501
502 this.admin.setBalancerRunning(false, true);
503
504 cluster.getMaster().setCatalogJanitorEnabled(false);
505 try {
506
507 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
508
509 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
510 printOutRegions(server, "Initial regions: ");
511 int regionCount = ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size();
512
513
514 ServerName fakedServer = ServerName.valueOf("any.old.server", 1234, -1);
515 if (useZKForAssignment) {
516 ZKAssign.createNodeClosing(TESTING_UTIL.getZooKeeperWatcher(),
517 hri, fakedServer);
518 } else {
519 regionStates.updateRegionState(hri, RegionState.State.CLOSING);
520 }
521
522
523 this.admin.split(hri.getRegionNameAsString());
524 this.admin.split(hri.getRegionNameAsString());
525 this.admin.split(hri.getRegionNameAsString());
526
527 for (int i = 0; i < 10; i++) {
528 Thread.sleep(100);
529 assertEquals(regionCount, ProtobufUtil.getOnlineRegions(
530 server.getRSRpcServices()).size());
531 }
532 if (useZKForAssignment) {
533
534 ZKAssign.deleteClosingNode(TESTING_UTIL.getZooKeeperWatcher(),
535 hri, fakedServer);
536 } else {
537 regionStates.regionOnline(hri, server.getServerName());
538 }
539
540 split(hri, server, regionCount);
541
542 checkAndGetDaughters(tableName);
543
544 } finally {
545 admin.setBalancerRunning(true, false);
546 cluster.getMaster().setCatalogJanitorEnabled(true);
547 t.close();
548 }
549 }
550
551
552
553
554
555
556
557 @Test (timeout=300000) public void testShutdownFixupWhenDaughterHasSplit()
558 throws IOException, InterruptedException, ServiceException {
559 final TableName tableName =
560 TableName.valueOf("testShutdownFixupWhenDaughterHasSplit");
561
562
563 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
564 List<HRegion> regions = cluster.getRegions(tableName);
565 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
566
567 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
568
569
570 this.admin.setBalancerRunning(false, true);
571
572 cluster.getMaster().setCatalogJanitorEnabled(false);
573 try {
574
575 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
576
577 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
578 printOutRegions(server, "Initial regions: ");
579 int regionCount = ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size();
580
581 split(hri, server, regionCount);
582
583 List<HRegion> daughters = checkAndGetDaughters(tableName);
584
585 regionCount = ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size();
586 HRegionInfo daughter = daughters.get(0).getRegionInfo();
587 LOG.info("Daughter we are going to split: " + daughter);
588
589
590 this.admin.compact(daughter.getRegionName());
591 daughters = cluster.getRegions(tableName);
592 HRegion daughterRegion = null;
593 for (HRegion r: daughters) {
594 if (r.getRegionInfo().equals(daughter)) {
595 daughterRegion = r;
596 LOG.info("Found matching HRI: " + daughterRegion);
597 break;
598 }
599 }
600 assertTrue(daughterRegion != null);
601 for (int i=0; i<100; i++) {
602 if (!daughterRegion.hasReferences()) break;
603 Threads.sleep(100);
604 }
605 assertFalse("Waiting for reference to be compacted", daughterRegion.hasReferences());
606 LOG.info("Daughter hri before split (has been compacted): " + daughter);
607 split(daughter, server, regionCount);
608
609 daughters = cluster.getRegions(tableName);
610 for (HRegion d: daughters) {
611 LOG.info("Regions before crash: " + d);
612 }
613
614 cluster.abortRegionServer(tableRegionIndex);
615 waitUntilRegionServerDead();
616 awaitDaughters(tableName, daughters.size());
617
618
619 regions = cluster.getRegions(tableName);
620 for (HRegion d: daughters) {
621 LOG.info("Regions after crash: " + d);
622 }
623 assertEquals(daughters.size(), regions.size());
624 for (HRegion r: regions) {
625 LOG.info("Regions post crash " + r);
626 assertTrue("Missing region post crash " + r, daughters.contains(r));
627 }
628 } finally {
629 admin.setBalancerRunning(true, false);
630 cluster.getMaster().setCatalogJanitorEnabled(true);
631 t.close();
632 }
633 }
634
635 @Test(timeout = 180000)
636 public void testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles() throws Exception {
637 TableName userTableName =
638 TableName.valueOf("testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles");
639 HTableDescriptor htd = new HTableDescriptor(userTableName);
640 HColumnDescriptor hcd = new HColumnDescriptor("col");
641 htd.addFamily(hcd);
642 admin.createTable(htd);
643 Table table = new HTable(TESTING_UTIL.getConfiguration(), userTableName);
644 try {
645 for (int i = 0; i <= 5; i++) {
646 String row = "row" + i;
647 Put p = new Put(row.getBytes());
648 String val = "Val" + i;
649 p.add("col".getBytes(), "ql".getBytes(), val.getBytes());
650 table.put(p);
651 admin.flush(userTableName.getName());
652 Delete d = new Delete(row.getBytes());
653
654 table.delete(d);
655 admin.flush(userTableName.getName());
656 }
657 admin.majorCompact(userTableName.getName());
658 List<HRegionInfo> regionsOfTable = TESTING_UTIL.getMiniHBaseCluster()
659 .getMaster().getAssignmentManager().getRegionStates()
660 .getRegionsOfTable(userTableName);
661 HRegionInfo hRegionInfo = regionsOfTable.get(0);
662 Put p = new Put("row6".getBytes());
663 p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
664 table.put(p);
665 p = new Put("row7".getBytes());
666 p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
667 table.put(p);
668 p = new Put("row8".getBytes());
669 p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
670 table.put(p);
671 admin.flush(userTableName.getName());
672 admin.split(hRegionInfo.getRegionName(), "row7".getBytes());
673 regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
674 .getAssignmentManager().getRegionStates()
675 .getRegionsOfTable(userTableName);
676
677 while (regionsOfTable.size() != 2) {
678 Thread.sleep(2000);
679 regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
680 .getAssignmentManager().getRegionStates()
681 .getRegionsOfTable(userTableName);
682 }
683 Assert.assertEquals(2, regionsOfTable.size());
684 Scan s = new Scan();
685 ResultScanner scanner = table.getScanner(s);
686 int mainTableCount = 0;
687 for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
688 mainTableCount++;
689 }
690 Assert.assertEquals(3, mainTableCount);
691 } finally {
692 table.close();
693 }
694 }
695
696
697
698
699 static class UselessTestAbortable implements Abortable {
700 boolean aborted = false;
701 @Override
702 public void abort(String why, Throwable e) {
703 LOG.warn("ABORTED (But nothing to abort): why=" + why, e);
704 aborted = true;
705 }
706
707 @Override
708 public boolean isAborted() {
709 return this.aborted;
710 }
711 }
712
713
714
715
716
717
718
719
720
721
722
723 @Test(timeout = 400000)
724 public void testMasterRestartWhenSplittingIsPartial()
725 throws IOException, InterruptedException, NodeExistsException,
726 KeeperException, DeserializationException, ServiceException {
727 final TableName tableName = TableName.valueOf("testMasterRestartWhenSplittingIsPartial");
728
729 if (!useZKForAssignment) {
730
731 return;
732 }
733
734
735 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
736 List<HRegion> regions = cluster.getRegions(tableName);
737 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
738
739 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
740
741
742 this.admin.setBalancerRunning(false, true);
743
744 cluster.getMaster().setCatalogJanitorEnabled(false);
745 ZooKeeperWatcher zkw = new ZooKeeperWatcher(t.getConfiguration(),
746 "testMasterRestartWhenSplittingIsPartial", new UselessTestAbortable());
747 try {
748
749 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
750
751 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
752 printOutRegions(server, "Initial regions: ");
753
754
755 AssignmentManager.TEST_SKIP_SPLIT_HANDLING = true;
756
757
758 this.admin.split(hri.getRegionNameAsString());
759 checkAndGetDaughters(tableName);
760
761 String path = ZKAssign.getNodeName(zkw, hri.getEncodedName());
762 Stat stats = zkw.getRecoverableZooKeeper().exists(path, false);
763 LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
764 + stats);
765 byte[] bytes = ZKAssign.getData(zkw, hri.getEncodedName());
766 RegionTransition rtd = RegionTransition.parseFrom(bytes);
767
768 assertTrue(rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)
769 || rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLITTING));
770
771
772 MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
773
774 this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
775
776
777
778 hri.setOffline(true);
779 hri.setSplit(true);
780 ServerName regionServerOfRegion = master.getAssignmentManager()
781 .getRegionStates().getRegionServerOfRegion(hri);
782 assertTrue(regionServerOfRegion != null);
783
784
785 AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
786 String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
787 Stat stat = new Stat();
788 byte[] data = ZKUtil.getDataNoWatch(zkw, node, stat);
789
790 for (int i=0; data != null && i<60; i++) {
791 Thread.sleep(1000);
792 data = ZKUtil.getDataNoWatch(zkw, node, stat);
793 }
794 assertNull("Waited too long for ZK node to be removed: "+node, data);
795 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
796 assertTrue("Split parent should be in SPLIT state",
797 regionStates.isRegionInState(hri, State.SPLIT));
798 regionServerOfRegion = regionStates.getRegionServerOfRegion(hri);
799 assertTrue(regionServerOfRegion == null);
800 } finally {
801
802 AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
803 admin.setBalancerRunning(true, false);
804 cluster.getMaster().setCatalogJanitorEnabled(true);
805 t.close();
806 zkw.close();
807 }
808 }
809
810
811
812
813
814
815
816
817
818 @Test (timeout = 300000)
819 public void testMasterRestartAtRegionSplitPendingCatalogJanitor()
820 throws IOException, InterruptedException, NodeExistsException,
821 KeeperException, ServiceException {
822 final TableName tableName = TableName
823 .valueOf("testMasterRestartAtRegionSplitPendingCatalogJanitor");
824
825
826 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
827 List<HRegion> regions = cluster.getRegions(tableName);
828 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
829
830 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
831
832
833 this.admin.setBalancerRunning(false, true);
834
835 cluster.getMaster().setCatalogJanitorEnabled(false);
836 ZooKeeperWatcher zkw = new ZooKeeperWatcher(t.getConfiguration(),
837 "testMasterRestartAtRegionSplitPendingCatalogJanitor", new UselessTestAbortable());
838 try {
839
840 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
841
842 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
843 printOutRegions(server, "Initial regions: ");
844
845 this.admin.split(hri.getRegionNameAsString());
846 checkAndGetDaughters(tableName);
847
848 String path = ZKAssign.getNodeName(zkw, hri.getEncodedName());
849 Stat stats = zkw.getRecoverableZooKeeper().exists(path, false);
850 LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
851 + stats);
852 String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
853 Stat stat = new Stat();
854 byte[] data = ZKUtil.getDataNoWatch(zkw, node, stat);
855
856 for (int i=0; data != null && i<60; i++) {
857 Thread.sleep(1000);
858 data = ZKUtil.getDataNoWatch(zkw, node, stat);
859 }
860 assertNull("Waited too long for ZK node to be removed: "+node, data);
861
862 MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
863
864 this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
865
866
867
868 hri.setOffline(true);
869 hri.setSplit(true);
870 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
871 assertTrue("Split parent should be in SPLIT state",
872 regionStates.isRegionInState(hri, State.SPLIT));
873 ServerName regionServerOfRegion = regionStates.getRegionServerOfRegion(hri);
874 assertTrue(regionServerOfRegion == null);
875 } finally {
876 this.admin.setBalancerRunning(true, false);
877 cluster.getMaster().setCatalogJanitorEnabled(true);
878 t.close();
879 zkw.close();
880 }
881 }
882
883
884
885
886
887
888
889
890
891
892
893
894 @Test(timeout = 60000)
895 public void testSplitBeforeSettingSplittingInZK() throws Exception,
896 InterruptedException, KeeperException {
897 testSplitBeforeSettingSplittingInZKInternals();
898 }
899
900 @Test(timeout = 60000)
901 public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Exception {
902 ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
903 final TableName tableName =
904 TableName.valueOf("testTableExistsIfTheSpecifiedTableRegionIsSplitParent");
905
906 Table t = createTableAndWait(tableName, Bytes.toBytes("cf"));
907 List<HRegion> regions = null;
908 try {
909 regions = cluster.getRegions(tableName);
910 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionInfo()
911 .getRegionName());
912 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
913 insertData(tableName, admin, t);
914
915 admin.setBalancerRunning(false, true);
916
917 cluster.getMaster().setCatalogJanitorEnabled(false);
918 boolean tableExists = MetaTableAccessor.tableExists(regionServer.getConnection(),
919 tableName);
920 assertEquals("The specified table should present.", true, tableExists);
921 final HRegion region = findSplittableRegion(regions);
922 assertTrue("not able to find a splittable region", region != null);
923 SplitTransactionImpl st = new SplitTransactionImpl(region, Bytes.toBytes("row2"));
924 try {
925 st.prepare();
926 st.createDaughters(regionServer, regionServer, null);
927 } catch (IOException e) {
928
929 }
930 tableExists = MetaTableAccessor.tableExists(regionServer.getConnection(),
931 tableName);
932 assertEquals("The specified table should present.", true, tableExists);
933 Map<String, RegionState> rit = cluster.getMaster().getAssignmentManager().getRegionStates()
934 .getRegionsInTransition();
935 assertTrue(rit.size() == 3);
936 cluster.getMaster().getAssignmentManager().regionOffline(st.getFirstDaughter());
937 cluster.getMaster().getAssignmentManager().regionOffline(st.getSecondDaughter());
938 cluster.getMaster().getAssignmentManager().regionOffline(region.getRegionInfo());
939 rit = cluster.getMaster().getAssignmentManager().getRegionStates().getRegionsInTransition();
940 assertTrue(rit.size() == 0);
941 }
942 finally {
943 admin.setBalancerRunning(true, false);
944 cluster.getMaster().setCatalogJanitorEnabled(true);
945 t.close();
946 TESTING_UTIL.deleteTable(tableName);
947 }
948 }
949
950 @Test
951 public void testSplitWithRegionReplicas() throws Exception {
952 ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
953 final TableName tableName =
954 TableName.valueOf("foobar");
955 HTableDescriptor htd = TESTING_UTIL.createTableDescriptor("foobar");
956 htd.setRegionReplication(2);
957 htd.addCoprocessor(SlowMeCopro.class.getName());
958
959 HTable t = TESTING_UTIL.createTable(htd, new byte[][]{Bytes.toBytes("cf")},
960 TESTING_UTIL.getConfiguration());
961 int count;
962 List<HRegion> oldRegions;
963 do {
964 oldRegions = cluster.getRegions(tableName);
965 Thread.sleep(10);
966 } while (oldRegions.size() != 2);
967 for (HRegion h : oldRegions) LOG.debug("OLDREGION " + h.getRegionInfo());
968 try {
969 int regionServerIndex = cluster.getServerWith(oldRegions.get(0).getRegionInfo()
970 .getRegionName());
971 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
972 insertData(tableName, admin, t);
973
974 admin.setBalancerRunning(false, true);
975
976 cluster.getMaster().setCatalogJanitorEnabled(false);
977 boolean tableExists = MetaTableAccessor.tableExists(regionServer.getConnection(),
978 tableName);
979 assertEquals("The specified table should be present.", true, tableExists);
980 final HRegion region = findSplittableRegion(oldRegions);
981 regionServerIndex = cluster.getServerWith(region.getRegionInfo().getRegionName());
982 regionServer = cluster.getRegionServer(regionServerIndex);
983 assertTrue("not able to find a splittable region", region != null);
984 String node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
985 region.getRegionInfo().getEncodedName());
986 regionServer.getZooKeeper().sync(node);
987 SplitTransactionImpl st = new SplitTransactionImpl(region, Bytes.toBytes("row2"));
988 try {
989 st.prepare();
990 st.execute(regionServer, regionServer);
991 } catch (IOException e) {
992 e.printStackTrace();
993 fail("Split execution should have succeeded with no exceptions thrown " + e);
994 }
995
996 List<HRegion> newRegions;
997 do {
998 newRegions = cluster.getRegions(tableName);
999 for (HRegion h : newRegions) LOG.debug("NEWREGION " + h.getRegionInfo());
1000 Thread.sleep(1000);
1001 } while ((newRegions.contains(oldRegions.get(0)) || newRegions.contains(oldRegions.get(1)))
1002 || newRegions.size() != 4);
1003 tableExists = MetaTableAccessor.tableExists(regionServer.getConnection(),
1004 tableName);
1005 assertEquals("The specified table should be present.", true, tableExists);
1006
1007 byte[] b1 = "row1".getBytes();
1008 Get g = new Get(b1);
1009 g.setConsistency(Consistency.STRONG);
1010
1011
1012
1013 Result r = t.get(g);
1014 Assert.assertFalse(r.isStale());
1015 LOG.info("exists stale after flush done");
1016
1017 SlowMeCopro.getCdl().set(new CountDownLatch(1));
1018 g = new Get(b1);
1019 g.setConsistency(Consistency.TIMELINE);
1020
1021 r = t.get(g);
1022 Assert.assertTrue(r.isStale());
1023 SlowMeCopro.getCdl().get().countDown();
1024 } finally {
1025 SlowMeCopro.getCdl().get().countDown();
1026 admin.setBalancerRunning(true, false);
1027 cluster.getMaster().setCatalogJanitorEnabled(true);
1028 t.close();
1029 }
1030 }
1031
1032 private void insertData(final TableName tableName, HBaseAdmin admin, Table t) throws IOException,
1033 InterruptedException {
1034 Put p = new Put(Bytes.toBytes("row1"));
1035 p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("1"));
1036 t.put(p);
1037 p = new Put(Bytes.toBytes("row2"));
1038 p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("2"));
1039 t.put(p);
1040 p = new Put(Bytes.toBytes("row3"));
1041 p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("3"));
1042 t.put(p);
1043 p = new Put(Bytes.toBytes("row4"));
1044 p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("4"));
1045 t.put(p);
1046 admin.flush(tableName);
1047 }
1048
1049
1050
1051
1052
1053 @Test(timeout = 60000)
1054 public void testSplitRegionWithNoStoreFiles()
1055 throws Exception {
1056 final TableName tableName =
1057 TableName.valueOf("testSplitRegionWithNoStoreFiles");
1058
1059 createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
1060 List<HRegion> regions = cluster.getRegions(tableName);
1061 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
1062 ensureTableRegionNotOnSameServerAsMeta(admin, hri);
1063 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionInfo()
1064 .getRegionName());
1065 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
1066
1067 this.admin.setBalancerRunning(false, true);
1068
1069 cluster.getMaster().setCatalogJanitorEnabled(false);
1070 try {
1071
1072 printOutRegions(regionServer, "Initial regions: ");
1073 Configuration conf = cluster.getConfiguration();
1074 HBaseFsck.debugLsr(conf, new Path("/"));
1075 Path rootDir = FSUtils.getRootDir(conf);
1076 FileSystem fs = TESTING_UTIL.getDFSCluster().getFileSystem();
1077 Map<String, Path> storefiles =
1078 FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
1079 assertEquals("Expected nothing but found " + storefiles.toString(), storefiles.size(), 0);
1080
1081
1082 regions = cluster.getRegions(tableName);
1083 final HRegion region = findSplittableRegion(regions);
1084 assertTrue("not able to find a splittable region", region != null);
1085
1086
1087 SplitTransactionImpl st = new MockedSplitTransaction(region, Bytes.toBytes("row2"));
1088 try {
1089 st.prepare();
1090 st.execute(regionServer, regionServer);
1091 } catch (IOException e) {
1092 fail("Split execution should have succeeded with no exceptions thrown");
1093 }
1094
1095
1096
1097 List<HRegion> daughters = cluster.getRegions(tableName);
1098 assertTrue(daughters.size() == 2);
1099
1100
1101 HBaseFsck.debugLsr(conf, new Path("/"));
1102 Map<String, Path> storefilesAfter =
1103 FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
1104 assertEquals("Expected nothing but found " + storefilesAfter.toString(),
1105 storefilesAfter.size(), 0);
1106
1107 hri = region.getRegionInfo();
1108 AssignmentManager am = cluster.getMaster().getAssignmentManager();
1109 RegionStates regionStates = am.getRegionStates();
1110 long start = EnvironmentEdgeManager.currentTime();
1111 while (!regionStates.isRegionInState(hri, State.SPLIT)) {
1112 assertFalse("Timed out in waiting split parent to be in state SPLIT",
1113 EnvironmentEdgeManager.currentTime() - start > 60000);
1114 Thread.sleep(500);
1115 }
1116
1117
1118 am.assign(hri, true, true);
1119 assertFalse("Split region can't be assigned",
1120 regionStates.isRegionInTransition(hri));
1121 assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
1122
1123
1124 am.unassign(hri, true, null);
1125 assertFalse("Split region can't be unassigned",
1126 regionStates.isRegionInTransition(hri));
1127 assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
1128 } finally {
1129 admin.setBalancerRunning(true, false);
1130 cluster.getMaster().setCatalogJanitorEnabled(true);
1131 }
1132 }
1133
1134 @Test(timeout = 180000)
1135 public void testSplitHooksBeforeAndAfterPONR() throws Exception {
1136 TableName firstTable = TableName.valueOf("testSplitHooksBeforeAndAfterPONR_1");
1137 TableName secondTable = TableName.valueOf("testSplitHooksBeforeAndAfterPONR_2");
1138 HColumnDescriptor hcd = new HColumnDescriptor("cf");
1139
1140 HTableDescriptor desc = new HTableDescriptor(firstTable);
1141 desc.addCoprocessor(MockedRegionObserver.class.getName());
1142 desc.addFamily(hcd);
1143 admin.createTable(desc);
1144 TESTING_UTIL.waitUntilAllRegionsAssigned(firstTable);
1145
1146 desc = new HTableDescriptor(secondTable);
1147 desc.addFamily(hcd);
1148 admin.createTable(desc);
1149 TESTING_UTIL.waitUntilAllRegionsAssigned(secondTable);
1150
1151 List<HRegion> firstTableRegions = cluster.getRegions(firstTable);
1152 List<HRegion> secondTableRegions = cluster.getRegions(secondTable);
1153
1154
1155 if (firstTableRegions.size() == 0 || secondTableRegions.size() == 0) {
1156 fail("Each table should have at least one region.");
1157 }
1158 ServerName serverName = cluster.getServerHoldingRegion(firstTable,
1159 firstTableRegions.get(0).getRegionInfo().getRegionName());
1160 admin.move(secondTableRegions.get(0).getRegionInfo().getEncodedNameAsBytes(),
1161 Bytes.toBytes(serverName.getServerName()));
1162 Table table1 = null;
1163 Table table2 = null;
1164 try {
1165 table1 = new HTable(TESTING_UTIL.getConfiguration(), firstTable);
1166 table2 = new HTable(TESTING_UTIL.getConfiguration(), firstTable);
1167 insertData(firstTable, admin, table1);
1168 insertData(secondTable, admin, table2);
1169 admin.split(firstTable, "row2".getBytes());
1170 firstTableRegions = cluster.getRegions(firstTable);
1171 while (firstTableRegions.size() != 2) {
1172 Thread.sleep(1000);
1173 firstTableRegions = cluster.getRegions(firstTable);
1174 }
1175 assertEquals("Number of regions after split should be 2.", 2, firstTableRegions.size());
1176 secondTableRegions = cluster.getRegions(secondTable);
1177 assertEquals("Number of regions after split should be 2.", 2, secondTableRegions.size());
1178 } finally {
1179 if (table1 != null) {
1180 table1.close();
1181 }
1182 if (table2 != null) {
1183 table2.close();
1184 }
1185 TESTING_UTIL.deleteTable(firstTable);
1186 TESTING_UTIL.deleteTable(secondTable);
1187 }
1188 }
1189
1190 private void testSplitBeforeSettingSplittingInZKInternals() throws Exception {
1191 final TableName tableName = TableName.valueOf("testSplitBeforeSettingSplittingInZK");
1192 try {
1193
1194 createTableAndWait(tableName, Bytes.toBytes("cf"));
1195
1196 List<HRegion> regions = awaitTableRegions(tableName);
1197 assertTrue("Table not online", cluster.getRegions(tableName).size() != 0);
1198
1199 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionInfo()
1200 .getRegionName());
1201 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
1202 final HRegion region = findSplittableRegion(regions);
1203 assertTrue("not able to find a splittable region", region != null);
1204 SplitTransactionImpl st = new MockedSplitTransaction(region, Bytes.toBytes("row2")) {
1205 @Override
1206 public PairOfSameType<Region> stepsBeforePONR(final Server server,
1207 final RegionServerServices services, boolean testing) throws IOException {
1208 throw new SplittingNodeCreationFailedException ();
1209 }
1210 };
1211 String node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
1212 region.getRegionInfo().getEncodedName());
1213 regionServer.getZooKeeper().sync(node);
1214 for (int i = 0; i < 100; i++) {
1215
1216
1217
1218 if (ZKUtil.checkExists(regionServer.getZooKeeper(), node) != -1) {
1219 Thread.sleep(100);
1220 }
1221 }
1222 try {
1223 st.prepare();
1224 st.execute(regionServer, regionServer);
1225 } catch (IOException e) {
1226
1227
1228
1229 assertTrue("Should be instance of CreateSplittingNodeFailedException",
1230 e instanceof SplittingNodeCreationFailedException );
1231 node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
1232 region.getRegionInfo().getEncodedName());
1233 {
1234 assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
1235 }
1236 assertTrue(st.rollback(regionServer, regionServer));
1237 assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
1238 }
1239 } finally {
1240 TESTING_UTIL.deleteTable(tableName);
1241 }
1242 }
1243
1244 @Test
1245 public void testStoreFileReferenceCreationWhenSplitPolicySaysToSkipRangeCheck()
1246 throws Exception {
1247 final TableName tableName =
1248 TableName.valueOf("testStoreFileReferenceCreationWhenSplitPolicySaysToSkipRangeCheck");
1249 try {
1250 HTableDescriptor htd = new HTableDescriptor(tableName);
1251 htd.addFamily(new HColumnDescriptor("f"));
1252 htd.addFamily(new HColumnDescriptor("i_f"));
1253 htd.setRegionSplitPolicyClassName(CustomSplitPolicy.class.getName());
1254 admin.createTable(htd);
1255 List<HRegion> regions = awaitTableRegions(tableName);
1256 HRegion region = regions.get(0);
1257 for(int i = 3;i<9;i++) {
1258 Put p = new Put(Bytes.toBytes("row"+i));
1259 p.add(Bytes.toBytes("f"), Bytes.toBytes("q"), Bytes.toBytes("value"+i));
1260 p.add(Bytes.toBytes("i_f"), Bytes.toBytes("q"), Bytes.toBytes("value"+i));
1261 region.put(p);
1262 }
1263 region.flush(true);
1264 Store store = region.getStore(Bytes.toBytes("f"));
1265 Collection<StoreFile> storefiles = store.getStorefiles();
1266 assertEquals(storefiles.size(), 1);
1267 assertFalse(region.hasReferences());
1268 Path referencePath =
1269 region.getRegionFileSystem().splitStoreFile(region.getRegionInfo(), "f",
1270 storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy());
1271 assertNull(referencePath);
1272 referencePath =
1273 region.getRegionFileSystem().splitStoreFile(region.getRegionInfo(), "i_f",
1274 storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy());
1275 assertNotNull(referencePath);
1276 } finally {
1277 TESTING_UTIL.deleteTable(tableName);
1278 }
1279 }
1280
1281 @Test(timeout = 120000)
1282 public void testFailedSplit() throws Exception {
1283 TableName tableName = TableName.valueOf("testFailedSplit");
1284 byte[] colFamily = Bytes.toBytes("info");
1285 TESTING_UTIL.createTable(tableName, colFamily);
1286 Connection connection = ConnectionFactory.createConnection(TESTING_UTIL.getConfiguration());
1287 HTable table = (HTable) connection.getTable(tableName);
1288 try {
1289 TESTING_UTIL.loadTable(table, colFamily);
1290 List<HRegionInfo> regions = TESTING_UTIL.getHBaseAdmin().getTableRegions(tableName);
1291 assertTrue(regions.size() == 1);
1292 final HRegion actualRegion = cluster.getRegions(tableName).get(0);
1293 actualRegion.getCoprocessorHost().load(FailingSplitRegionObserver.class,
1294 Coprocessor.PRIORITY_USER, actualRegion.getBaseConf());
1295
1296
1297 admin.split(tableName);
1298 FailingSplitRegionObserver observer = (FailingSplitRegionObserver) actualRegion
1299 .getCoprocessorHost().findCoprocessor(FailingSplitRegionObserver.class.getName());
1300 assertNotNull(observer);
1301 observer.latch.await();
1302 observer.postSplit.await();
1303 LOG.info("Waiting for region to come out of RIT: " + actualRegion);
1304 TESTING_UTIL.waitFor(60000, 1000, new Waiter.Predicate<Exception>() {
1305 @Override
1306 public boolean evaluate() throws Exception {
1307 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
1308 Map<String, RegionState> rit = regionStates.getRegionsInTransition();
1309 return (rit.size() == 0);
1310 }
1311 });
1312 regions = TESTING_UTIL.getHBaseAdmin().getTableRegions(tableName);
1313 assertTrue(regions.size() == 1);
1314 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
1315 Map<String, RegionState> rit = regionStates.getRegionsInTransition();
1316 assertTrue(rit.size() == 0);
1317 } finally {
1318 table.close();
1319 connection.close();
1320 TESTING_UTIL.deleteTable(tableName);
1321 }
1322 }
1323
1324 @Test (timeout=300000)
1325 public void testSSHCleanupDaugtherRegionsOfAbortedSplit() throws Exception {
1326 TableName table = TableName.valueOf("testSSHCleanupDaugtherRegionsOfAbortedSplit");
1327 try {
1328 HTableDescriptor desc = new HTableDescriptor(table);
1329 desc.addFamily(new HColumnDescriptor(Bytes.toBytes("f")));
1330 admin.createTable(desc);
1331 HTable hTable = new HTable(cluster.getConfiguration(), desc.getTableName());
1332 for(int i = 1; i < 5; i++) {
1333 Put p1 = new Put(("r"+i).getBytes());
1334 p1.add(Bytes.toBytes("f"), "q1".getBytes(), "v".getBytes());
1335 hTable.put(p1);
1336 }
1337 admin.flush(desc.getTableName());
1338 List<HRegion> regions = cluster.getRegions(desc.getTableName());
1339 int serverWith = cluster.getServerWith(regions.get(0).getRegionInfo().getRegionName());
1340 HRegionServer regionServer = cluster.getRegionServer(serverWith);
1341 cluster.getServerWith(regions.get(0).getRegionInfo().getRegionName());
1342 SplitTransactionImpl st = new SplitTransactionImpl(regions.get(0), Bytes.toBytes("r3"));
1343 st.prepare();
1344 st.stepsBeforePONR(regionServer, regionServer, false);
1345 Path tableDir =
1346 FSUtils.getTableDir(cluster.getMaster().getMasterFileSystem().getRootDir(),
1347 desc.getTableName());
1348 tableDir.getFileSystem(cluster.getConfiguration());
1349 List<Path> regionDirs =
1350 FSUtils.getRegionDirs(tableDir.getFileSystem(cluster.getConfiguration()), tableDir);
1351 assertEquals(3,regionDirs.size());
1352 cluster.startRegionServer();
1353 regionServer.kill();
1354 cluster.getRegionServerThreads().get(serverWith).join();
1355
1356 while (cluster.getMaster().getServerManager().areDeadServersInProgress()) {
1357 Thread.sleep(10);
1358 }
1359 AssignmentManager am = cluster.getMaster().getAssignmentManager();
1360 while(am.getRegionStates().isRegionsInTransition()) {
1361 Thread.sleep(10);
1362 }
1363 assertEquals(am.getRegionStates().getRegionsInTransition().toString(), 0, am
1364 .getRegionStates().getRegionsInTransition().size());
1365 regionDirs =
1366 FSUtils.getRegionDirs(tableDir.getFileSystem(cluster.getConfiguration()), tableDir);
1367 assertEquals(1,regionDirs.size());
1368 } finally {
1369 TESTING_UTIL.deleteTable(table);
1370 }
1371 }
1372
1373 public static class MockedCoordinatedStateManager extends ZkCoordinatedStateManager {
1374
1375 public void initialize(Server server, HRegion region) {
1376 this.server = server;
1377 this.watcher = server.getZooKeeper();
1378 splitTransactionCoordination = new MockedSplitTransactionCoordination(this, watcher, region);
1379 closeRegionCoordination = new ZkCloseRegionCoordination(this, watcher);
1380 openRegionCoordination = new ZkOpenRegionCoordination(this, watcher);
1381 }
1382 }
1383
1384 public static class MockedSplitTransaction extends SplitTransactionImpl {
1385
1386 private HRegion currentRegion;
1387 public MockedSplitTransaction(HRegion region, byte[] splitrow) {
1388 super(region, splitrow);
1389 this.currentRegion = region;
1390 }
1391 @Override
1392 public boolean rollback(Server server, RegionServerServices services) throws IOException {
1393 if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
1394 .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
1395 if(secondSplit){
1396 super.rollback(server, services);
1397 latch.countDown();
1398 return true;
1399 }
1400 }
1401 return super.rollback(server, services);
1402 }
1403
1404
1405 }
1406
1407 public static class MockedSplitTransactionCoordination extends ZKSplitTransactionCoordination {
1408
1409 private HRegion currentRegion;
1410
1411 public MockedSplitTransactionCoordination(CoordinatedStateManager coordinationProvider,
1412 ZooKeeperWatcher watcher, HRegion region) {
1413 super(coordinationProvider, watcher);
1414 currentRegion = region;
1415 }
1416
1417 @Override
1418 public void completeSplitTransaction(RegionServerServices services, Region a, Region b,
1419 SplitTransactionDetails std, Region parent) throws IOException {
1420 if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
1421 .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
1422 try {
1423 if (!secondSplit){
1424 callRollBack = true;
1425 latch.await();
1426 }
1427 } catch (InterruptedException e) {
1428 }
1429
1430 }
1431 super.completeSplitTransaction(services, a, b, std, parent);
1432 if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
1433 .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
1434 firstSplitCompleted = true;
1435 }
1436 }
1437 }
1438
1439 private HRegion findSplittableRegion(final List<HRegion> regions) throws InterruptedException {
1440 for (int i = 0; i < 5; ++i) {
1441 for (HRegion r: regions) {
1442 if (r.isSplittable() && r.getRegionInfo().getReplicaId() == 0) {
1443 return(r);
1444 }
1445 }
1446 Thread.sleep(100);
1447 }
1448 return(null);
1449 }
1450
1451 private List<HRegion> checkAndGetDaughters(TableName tableName)
1452 throws InterruptedException {
1453 List<HRegion> daughters = null;
1454
1455 for (int i=0; i<100; i++) {
1456 daughters = cluster.getRegions(tableName);
1457 if (daughters.size() >= 2) break;
1458 Thread.sleep(100);
1459 }
1460 assertTrue(daughters.size() >= 2);
1461 return daughters;
1462 }
1463
1464 private MockMasterWithoutCatalogJanitor abortAndWaitForMaster()
1465 throws IOException, InterruptedException {
1466 cluster.abortMaster(0);
1467 cluster.waitOnMaster(0);
1468 cluster.getConfiguration().setClass(HConstants.MASTER_IMPL,
1469 MockMasterWithoutCatalogJanitor.class, HMaster.class);
1470 MockMasterWithoutCatalogJanitor master = null;
1471 master = (MockMasterWithoutCatalogJanitor) cluster.startMaster().getMaster();
1472 cluster.waitForActiveAndReadyMaster();
1473 return master;
1474 }
1475
1476 private void split(final HRegionInfo hri, final HRegionServer server, final int regionCount)
1477 throws IOException, InterruptedException {
1478 this.admin.split(hri.getRegionNameAsString());
1479 try {
1480 for (int i = 0; ProtobufUtil.getOnlineRegions(
1481 server.getRSRpcServices()).size() <= regionCount && i < 300; i++) {
1482 LOG.debug("Waiting on region to split");
1483 Thread.sleep(100);
1484 }
1485
1486 assertFalse("Waited too long for split",
1487 ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size() <= regionCount);
1488 } catch (RegionServerStoppedException e) {
1489 if (useZKForAssignment) {
1490
1491 LOG.error(e);
1492 throw e;
1493 }
1494 }
1495 }
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508 private int ensureTableRegionNotOnSameServerAsMeta(final Admin admin,
1509 final HRegionInfo hri)
1510 throws IOException, MasterNotRunningException,
1511 ZooKeeperConnectionException, InterruptedException {
1512
1513
1514
1515 int metaServerIndex = cluster.getServerWithMeta();
1516 assertTrue(metaServerIndex != -1);
1517 HRegionServer metaRegionServer = cluster.getRegionServer(metaServerIndex);
1518 int tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1519 assertTrue(tableRegionIndex != -1);
1520 HRegionServer tableRegionServer = cluster.getRegionServer(tableRegionIndex);
1521 if (metaRegionServer.getServerName().equals(tableRegionServer.getServerName())) {
1522 HRegionServer hrs = getOtherRegionServer(cluster, metaRegionServer);
1523 assertNotNull(hrs);
1524 assertNotNull(hri);
1525 LOG.info("Moving " + hri.getRegionNameAsString() + " from " +
1526 metaRegionServer.getServerName() + " to " +
1527 hrs.getServerName() + "; metaServerIndex=" + metaServerIndex);
1528 admin.move(hri.getEncodedNameAsBytes(), Bytes.toBytes(hrs.getServerName().toString()));
1529 }
1530
1531 for (int i = 0; i < 20; i++) {
1532 tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1533 if (tableRegionIndex != -1 && tableRegionIndex != metaServerIndex) break;
1534 LOG.debug("Waiting on region move off the hbase:meta server; current index " +
1535 tableRegionIndex + " and metaServerIndex=" + metaServerIndex);
1536 Thread.sleep(1000);
1537 }
1538 assertTrue("Region not moved off hbase:meta server", tableRegionIndex != -1
1539 && tableRegionIndex != metaServerIndex);
1540
1541 tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1542 assertTrue(tableRegionIndex != -1);
1543 assertNotSame(metaServerIndex, tableRegionIndex);
1544 return tableRegionIndex;
1545 }
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556 private HRegionServer getOtherRegionServer(final MiniHBaseCluster cluster,
1557 final HRegionServer notThisOne) {
1558 for (RegionServerThread rst: cluster.getRegionServerThreads()) {
1559 HRegionServer hrs = rst.getRegionServer();
1560 if (hrs.getServerName().equals(notThisOne.getServerName())) continue;
1561 if (hrs.isStopping() || hrs.isStopped()) continue;
1562 return hrs;
1563 }
1564 return null;
1565 }
1566
1567 private void printOutRegions(final HRegionServer hrs, final String prefix)
1568 throws IOException {
1569 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
1570 for (HRegionInfo region: regions) {
1571 LOG.info(prefix + region.getRegionNameAsString());
1572 }
1573 }
1574
1575 private void waitUntilRegionServerDead() throws InterruptedException, InterruptedIOException {
1576
1577 for (int i=0; cluster.getMaster().getClusterStatus().
1578 getServers().size() > NB_SERVERS && i<100; i++) {
1579 LOG.info("Waiting on server to go down");
1580 Thread.sleep(100);
1581 }
1582 assertFalse("Waited too long for RS to die", cluster.getMaster().getClusterStatus().
1583 getServers().size() > NB_SERVERS);
1584 }
1585
1586 private void awaitDaughters(TableName tableName, int numDaughters) throws InterruptedException {
1587
1588 for (int i=0; cluster.getRegions(tableName).size() < numDaughters && i<60; i++) {
1589 LOG.info("Waiting for repair to happen");
1590 Thread.sleep(1000);
1591 }
1592 if (cluster.getRegions(tableName).size() < numDaughters) {
1593 fail("Waiting too long for daughter regions");
1594 }
1595 }
1596
1597 private List<HRegion> awaitTableRegions(final TableName tableName) throws InterruptedException {
1598 List<HRegion> regions = null;
1599 for (int i = 0; i < 100; i++) {
1600 regions = cluster.getRegions(tableName);
1601 if (regions.size() > 0) break;
1602 Thread.sleep(100);
1603 }
1604 return regions;
1605 }
1606
1607 private HTable createTableAndWait(TableName tableName, byte[] cf) throws IOException,
1608 InterruptedException {
1609 HTable t = TESTING_UTIL.createTable(tableName, cf);
1610 awaitTableRegions(tableName);
1611 assertTrue("Table not online: " + tableName,
1612 cluster.getRegions(tableName).size() != 0);
1613 return t;
1614 }
1615
1616 public static class MockMasterWithoutCatalogJanitor extends HMaster {
1617
1618 public MockMasterWithoutCatalogJanitor(Configuration conf, CoordinatedStateManager cp)
1619 throws IOException, KeeperException,
1620 InterruptedException {
1621 super(conf, cp);
1622 }
1623 }
1624
1625 private static class SplittingNodeCreationFailedException extends IOException {
1626 private static final long serialVersionUID = 1652404976265623004L;
1627
1628 public SplittingNodeCreationFailedException () {
1629 super();
1630 }
1631 }
1632
1633 public static class MockedRegionObserver extends BaseRegionObserver {
1634 private SplitTransactionImpl st = null;
1635 private PairOfSameType<Region> daughterRegions = null;
1636
1637 @Override
1638 public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
1639 byte[] splitKey, List<Mutation> metaEntries) throws IOException {
1640 RegionCoprocessorEnvironment environment = ctx.getEnvironment();
1641 HRegionServer rs = (HRegionServer) environment.getRegionServerServices();
1642 List<Region> onlineRegions =
1643 rs.getOnlineRegions(TableName.valueOf("testSplitHooksBeforeAndAfterPONR_2"));
1644 Region region = onlineRegions.get(0);
1645 for (Region r : onlineRegions) {
1646 if (r.getRegionInfo().containsRow(splitKey)) {
1647 region = r;
1648 break;
1649 }
1650 }
1651 st = new SplitTransactionImpl((HRegion) region, splitKey);
1652 if (!st.prepare()) {
1653 LOG.error("Prepare for the table " + region.getTableDesc().getNameAsString()
1654 + " failed. So returning null. ");
1655 ctx.bypass();
1656 return;
1657 }
1658 ((HRegion)region).forceSplit(splitKey);
1659 daughterRegions = st.stepsBeforePONR(rs, rs, false);
1660 HRegionInfo copyOfParent = new HRegionInfo(region.getRegionInfo());
1661 copyOfParent.setOffline(true);
1662 copyOfParent.setSplit(true);
1663
1664 Put putParent = MetaTableAccessor.makePutFromRegionInfo(copyOfParent);
1665 MetaTableAccessor.addDaughtersToPut(putParent, daughterRegions.getFirst().getRegionInfo(),
1666 daughterRegions.getSecond().getRegionInfo());
1667 metaEntries.add(putParent);
1668
1669 Put putA = MetaTableAccessor.makePutFromRegionInfo(
1670 daughterRegions.getFirst().getRegionInfo());
1671 Put putB = MetaTableAccessor.makePutFromRegionInfo(
1672 daughterRegions.getSecond().getRegionInfo());
1673 st.addLocation(putA, rs.getServerName(), 1);
1674 st.addLocation(putB, rs.getServerName(), 1);
1675 metaEntries.add(putA);
1676 metaEntries.add(putB);
1677 }
1678
1679 @Override
1680 public void preSplitAfterPONR(ObserverContext<RegionCoprocessorEnvironment> ctx)
1681 throws IOException {
1682 RegionCoprocessorEnvironment environment = ctx.getEnvironment();
1683 HRegionServer rs = (HRegionServer) environment.getRegionServerServices();
1684 st.stepsAfterPONR(rs, rs, daughterRegions, null);
1685 }
1686
1687 }
1688
1689 static class CustomSplitPolicy extends RegionSplitPolicy {
1690
1691 @Override
1692 protected boolean shouldSplit() {
1693 return true;
1694 }
1695
1696 @Override
1697 public boolean skipStoreFileRangeCheck(String familyName) {
1698 if(familyName.startsWith("i_")) {
1699 return true;
1700 } else {
1701 return false;
1702 }
1703 }
1704 }
1705 }
1706