1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertNotSame;
25 import static org.junit.Assert.assertNull;
26 import static org.junit.Assert.assertTrue;
27 import static org.junit.Assert.fail;
28
29 import java.io.InterruptedIOException;
30 import java.io.IOException;
31 import java.util.Collection;
32 import java.util.List;
33 import java.util.Map;
34 import java.util.concurrent.CountDownLatch;
35 import java.util.concurrent.TimeUnit;
36
37 import org.apache.commons.logging.Log;
38 import org.apache.commons.logging.LogFactory;
39 import org.apache.hadoop.conf.Configuration;
40 import org.apache.hadoop.fs.FileSystem;
41 import org.apache.hadoop.fs.Path;
42 import org.apache.hadoop.hbase.Abortable;
43 import org.apache.hadoop.hbase.CoordinatedStateManager;
44 import org.apache.hadoop.hbase.Coprocessor;
45 import org.apache.hadoop.hbase.CoprocessorEnvironment;
46 import org.apache.hadoop.hbase.HBaseTestingUtility;
47 import org.apache.hadoop.hbase.HColumnDescriptor;
48 import org.apache.hadoop.hbase.HConstants;
49 import org.apache.hadoop.hbase.HRegionInfo;
50 import org.apache.hadoop.hbase.HTableDescriptor;
51 import org.apache.hadoop.hbase.MasterNotRunningException;
52 import org.apache.hadoop.hbase.MetaTableAccessor;
53 import org.apache.hadoop.hbase.MiniHBaseCluster;
54 import org.apache.hadoop.hbase.RegionTransition;
55 import org.apache.hadoop.hbase.Server;
56 import org.apache.hadoop.hbase.ServerName;
57 import org.apache.hadoop.hbase.TableName;
58 import org.apache.hadoop.hbase.UnknownRegionException;
59 import org.apache.hadoop.hbase.Waiter;
60 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
61 import org.apache.hadoop.hbase.client.Admin;
62 import org.apache.hadoop.hbase.client.Connection;
63 import org.apache.hadoop.hbase.client.ConnectionFactory;
64 import org.apache.hadoop.hbase.client.Consistency;
65 import org.apache.hadoop.hbase.client.Delete;
66 import org.apache.hadoop.hbase.client.Get;
67 import org.apache.hadoop.hbase.client.HBaseAdmin;
68 import org.apache.hadoop.hbase.client.HTable;
69 import org.apache.hadoop.hbase.client.Mutation;
70 import org.apache.hadoop.hbase.client.Put;
71 import org.apache.hadoop.hbase.client.Result;
72 import org.apache.hadoop.hbase.client.ResultScanner;
73 import org.apache.hadoop.hbase.client.Scan;
74 import org.apache.hadoop.hbase.client.Table;
75 import org.apache.hadoop.hbase.client.TestReplicasClient.SlowMeCopro;
76 import org.apache.hadoop.hbase.coordination.ZKSplitTransactionCoordination;
77 import org.apache.hadoop.hbase.coordination.ZkCloseRegionCoordination;
78 import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
79 import org.apache.hadoop.hbase.coordination.ZkOpenRegionCoordination;
80 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
81 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
82 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
83 import org.apache.hadoop.hbase.exceptions.DeserializationException;
84 import org.apache.hadoop.hbase.executor.EventType;
85 import org.apache.hadoop.hbase.master.AssignmentManager;
86 import org.apache.hadoop.hbase.master.HMaster;
87 import org.apache.hadoop.hbase.master.RegionState;
88 import org.apache.hadoop.hbase.master.RegionState.State;
89 import org.apache.hadoop.hbase.master.RegionStates;
90 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
91 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
92 import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
93 import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
94 import org.apache.hadoop.hbase.testclassification.LargeTests;
95 import org.apache.hadoop.hbase.util.Bytes;
96 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
97 import org.apache.hadoop.hbase.util.FSUtils;
98 import org.apache.hadoop.hbase.util.HBaseFsck;
99 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
100 import org.apache.hadoop.hbase.util.PairOfSameType;
101 import org.apache.hadoop.hbase.util.RetryCounter;
102 import org.apache.hadoop.hbase.util.Threads;
103 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
104 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
105 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
106 import org.apache.zookeeper.KeeperException;
107 import org.apache.zookeeper.KeeperException.NodeExistsException;
108 import org.apache.zookeeper.data.Stat;
109 import org.junit.After;
110 import org.junit.AfterClass;
111 import org.junit.Assert;
112 import org.junit.Before;
113 import org.junit.BeforeClass;
114 import org.junit.Test;
115 import org.junit.experimental.categories.Category;
116
117 import com.google.protobuf.ServiceException;
118
119
120
121
122
123
124 @Category(LargeTests.class)
125 @SuppressWarnings("deprecation")
126 public class TestSplitTransactionOnCluster {
127 private static final Log LOG =
128 LogFactory.getLog(TestSplitTransactionOnCluster.class);
129 private HBaseAdmin admin = null;
130 private MiniHBaseCluster cluster = null;
131 private static Configuration conf;
132 private static final int NB_SERVERS = 3;
133 private static CountDownLatch latch = new CountDownLatch(1);
134 private static volatile boolean secondSplit = false;
135 private static volatile boolean callRollBack = false;
136 private static volatile boolean firstSplitCompleted = false;
137 private static boolean useZKForAssignment;
138
139 static final HBaseTestingUtility TESTING_UTIL =
140 new HBaseTestingUtility();
141
142 static void setupOnce() throws Exception {
143 TESTING_UTIL.getConfiguration().setInt("hbase.balancer.period", 60000);
144 useZKForAssignment = TESTING_UTIL.getConfiguration().getBoolean(
145 "hbase.assignment.usezk", true);
146 TESTING_UTIL.startMiniCluster(NB_SERVERS);
147 }
148
149 @BeforeClass
150 public static void before() throws Exception {
151 conf = TESTING_UTIL.getConfiguration();
152
153 conf.setBoolean("hbase.assignment.usezk", true);
154 setupOnce();
155 }
156
157 @AfterClass public static void after() throws Exception {
158 TESTING_UTIL.shutdownMiniCluster();
159 }
160
161 @Before public void setup() throws IOException {
162 TESTING_UTIL.ensureSomeNonStoppedRegionServersAvailable(NB_SERVERS);
163 this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
164 this.cluster = TESTING_UTIL.getMiniHBaseCluster();
165 }
166
167 @After
168 public void tearDown() throws Exception {
169 this.admin.close();
170 }
171
172 private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions)
173 throws IOException, InterruptedException {
174 assertEquals(1, regions.size());
175 HRegionInfo hri = regions.get(0).getRegionInfo();
176 TESTING_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
177 .waitOnRegionToClearRegionsInTransition(hri, 600000);
178 return hri;
179 }
180
181 @Test(timeout = 60000)
182 public void testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack() throws Exception {
183 final TableName tableName =
184 TableName.valueOf("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack");
185
186 if (!useZKForAssignment) {
187
188 return;
189 }
190
191 try {
192
193 HTable t = createTableAndWait(tableName, Bytes.toBytes("cf"));
194 final List<HRegion> regions = cluster.getRegions(tableName);
195 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
196 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionInfo()
197 .getRegionName());
198 final HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
199 insertData(tableName, admin, t);
200 t.close();
201
202
203 this.admin.setBalancerRunning(false, true);
204
205 cluster.getMaster().setCatalogJanitorEnabled(false);
206
207
208 final HRegion region = findSplittableRegion(regions);
209 assertTrue("not able to find a splittable region", region != null);
210 MockedCoordinatedStateManager cp = new MockedCoordinatedStateManager();
211 cp.initialize(regionServer, region);
212 cp.start();
213 regionServer.csm = cp;
214
215 new Thread() {
216 @Override
217 public void run() {
218 SplitTransaction st = null;
219 st = new MockedSplitTransaction(region, Bytes.toBytes("row2"));
220 try {
221 st.prepare();
222 st.execute(regionServer, regionServer);
223 } catch (IOException e) {
224
225 }
226 }
227 }.start();
228 for (int i = 0; !callRollBack && i < 100; i++) {
229 Thread.sleep(100);
230 }
231 assertTrue("Waited too long for rollback", callRollBack);
232 SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row3"));
233 try {
234 secondSplit = true;
235
236 region.initialize();
237 st.prepare();
238 st.execute(regionServer, regionServer);
239 } catch (IOException e) {
240 LOG.debug("Rollback started :"+ e.getMessage());
241 st.rollback(regionServer, regionServer);
242 }
243 for (int i=0; !firstSplitCompleted && i<100; i++) {
244 Thread.sleep(100);
245 }
246 assertTrue("fist split did not complete", firstSplitCompleted);
247
248 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
249 Map<String, RegionState> rit = regionStates.getRegionsInTransition();
250
251 for (int i=0; rit.containsKey(hri.getTable()) && i<100; i++) {
252 Thread.sleep(100);
253 }
254 assertFalse("region still in transition", rit.containsKey(
255 rit.containsKey(hri.getTable())));
256
257 List<Region> onlineRegions = regionServer.getOnlineRegions(tableName);
258
259 assertEquals("The parent region should be splitted", 2, onlineRegions.size());
260
261 List<HRegionInfo> regionsOfTable = cluster.getMaster().getAssignmentManager()
262 .getRegionStates().getRegionsOfTable(tableName);
263
264 assertEquals("No of regions in master", 2, regionsOfTable.size());
265 } finally {
266 admin.setBalancerRunning(true, false);
267 secondSplit = false;
268 firstSplitCompleted = false;
269 callRollBack = false;
270 cluster.getMaster().setCatalogJanitorEnabled(true);
271 TESTING_UTIL.deleteTable(tableName);
272 }
273 }
274
275 @Test(timeout = 60000)
276 public void testRITStateForRollback() throws Exception {
277 final TableName tableName =
278 TableName.valueOf("testRITStateForRollback");
279 try {
280
281 Table t = createTableAndWait(tableName, Bytes.toBytes("cf"));
282 final List<HRegion> regions = cluster.getRegions(tableName);
283 final HRegionInfo hri = getAndCheckSingleTableRegion(regions);
284 insertData(tableName, admin, t);
285 t.close();
286
287
288 this.admin.setBalancerRunning(false, true);
289
290 cluster.getMaster().setCatalogJanitorEnabled(false);
291
292
293 final HRegion region = findSplittableRegion(regions);
294 assertTrue("not able to find a splittable region", region != null);
295
296
297 region.getCoprocessorHost().load(FailingSplitRegionObserver.class,
298 Coprocessor.PRIORITY_USER, region.getBaseConf());
299
300
301 this.admin.split(region.getRegionInfo().getRegionName(), new byte[] {42});
302
303
304 FailingSplitRegionObserver observer = (FailingSplitRegionObserver) region
305 .getCoprocessorHost().findCoprocessor(FailingSplitRegionObserver.class.getName());
306 assertNotNull(observer);
307 observer.latch.await();
308
309 LOG.info("Waiting for region to come out of RIT");
310 TESTING_UTIL.waitUntilNoRegionsInTransition(60000);
311 } finally {
312 admin.setBalancerRunning(true, false);
313 cluster.getMaster().setCatalogJanitorEnabled(true);
314 TESTING_UTIL.deleteTable(tableName);
315 }
316 }
317 @Test(timeout = 60000)
318 public void testSplitFailedCompactionAndSplit() throws Exception {
319 final TableName tableName = TableName.valueOf("testSplitFailedCompactionAndSplit");
320 Configuration conf = TESTING_UTIL.getConfiguration();
321 try {
322 HBaseAdmin admin = new HBaseAdmin(conf);
323
324 HTableDescriptor htd = new HTableDescriptor(tableName);
325 byte[] cf = Bytes.toBytes("cf");
326 htd.addFamily(new HColumnDescriptor(cf));
327 admin.createTable(htd);
328
329 for (int i = 0; cluster.getRegions(tableName).size() == 0 && i < 100; i++) {
330 Thread.sleep(100);
331 }
332 assertEquals(1, cluster.getRegions(tableName).size());
333
334 HRegion region = cluster.getRegions(tableName).get(0);
335 Store store = region.getStore(cf);
336 int regionServerIndex = cluster.getServerWith(region.getRegionInfo().getRegionName());
337 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
338
339 Table t = new HTable(conf, tableName);
340
341 insertData(tableName, admin, t);
342 insertData(tableName, admin, t);
343
344 int fileNum = store.getStorefiles().size();
345
346 store.triggerMajorCompaction();
347 CompactionContext cc = store.requestCompaction();
348 assertNotNull(cc);
349
350
351 assertEquals(2, region.close(false).get(cf).size());
352
353 region.initialize();
354
355
356 assertFalse(region.compact(cc, store, NoLimitThroughputController.INSTANCE));
357 assertTrue(fileNum > store.getStorefiles().size());
358
359
360 SplitTransaction st = new SplitTransactionImpl(region, Bytes.toBytes("row3"));
361 assertTrue(st.prepare());
362 st.execute(regionServer, regionServer);
363 LOG.info("Waiting for region to come out of RIT");
364 TESTING_UTIL.waitFor(60000, 1000, new Waiter.Predicate<Exception>() {
365 @Override
366 public boolean evaluate() throws Exception {
367 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
368 Map<String, RegionState> rit = regionStates.getRegionsInTransition();
369 return (rit.size() == 0);
370 }
371 });
372 assertEquals(2, cluster.getRegions(tableName).size());
373 } finally {
374 TESTING_UTIL.deleteTable(tableName);
375 }
376 }
377
378 public static class FailingSplitRegionObserver extends BaseRegionObserver {
379 volatile CountDownLatch latch;
380 volatile CountDownLatch postSplit;
381 @Override
382 public void start(CoprocessorEnvironment e) throws IOException {
383 latch = new CountDownLatch(1);
384 postSplit = new CountDownLatch(1);
385 }
386 @Override
387 public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
388 byte[] splitKey, List<Mutation> metaEntries) throws IOException {
389 latch.countDown();
390 LOG.info("Causing rollback of region split");
391 throw new IOException("Causing rollback of region split");
392 }
393 @Override
394 public void postCompleteSplit(ObserverContext<RegionCoprocessorEnvironment> ctx)
395 throws IOException {
396 postSplit.countDown();
397 LOG.info("postCompleteSplit called");
398 }
399 }
400
401
402
403
404
405
406
407 @Test (timeout = 60000)
408 public void testSplitIsRolledBackOnSplitFailure() throws Exception {
409 final TableName tableName = TableName.valueOf("testSplitIsRolledBackOnSplitFailure");
410
411
412 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
413 List<HRegion> regions = cluster.getRegions(tableName);
414 final HRegionInfo hri = getAndCheckSingleTableRegion(regions);
415
416
417 this.admin.setBalancerRunning(false, true);
418
419 cluster.getMaster().setCatalogJanitorEnabled(false);
420
421 int serverIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
422
423 try {
424
425 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
426
427 HRegionServer server = cluster.getRegionServer(serverIndex);
428 printOutRegions(server, "Initial regions: ");
429 int regionCount = ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size();
430
431
432 AssignmentManager.setTestSkipSplitHandling(true);
433
434 split(hri, server, regionCount);
435
436 ZooKeeperWatcher zkw = TESTING_UTIL.getZooKeeperWatcher();
437 String path = ZKAssign.getNodeName(zkw, hri.getEncodedName());
438 RegionTransition rt = null;
439 Stat stats = null;
440
441
442 for (int i = 0; i < 100; i++) {
443 stats = zkw.getRecoverableZooKeeper().exists(path, false);
444 rt = RegionTransition.parseFrom(ZKAssign.getData(zkw, hri.getEncodedName()));
445 if (rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)) {
446 break;
447 }
448 Thread.sleep(100);
449 }
450 LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats=" + stats);
451 assertTrue(rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT));
452
453 abortServerAndWaitForProcessingToComplete(serverIndex);
454 waitUntilRegionServerDead();
455
456 TESTING_UTIL.waitUntilNoRegionsInTransition();
457
458
459 TESTING_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
460 @Override
461 public boolean evaluate() {
462 for (HRegion region : cluster.getRegions(tableName)) {
463 if (Bytes.equals(region.getRegionInfo().getRegionName(), hri.getRegionName())) {
464 return true;
465 } else {
466 LOG.debug("Wait for some more time, online region: " + region);
467 }
468 }
469 return false;
470 }
471 });
472
473 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
474 assertTrue("Parent region should be online", regionStates.isRegionOnline(hri));
475
476 List<HRegionInfo> tableRegions = MetaTableAccessor.getTableRegions(zkw,
477 cluster.getMaster().getConnection(), tableName);
478 assertEquals("Only parent region should be present, but we have: " + tableRegions,
479 1, tableRegions.size());
480 Path tableDir = FSUtils.getTableDir(FSUtils.getRootDir(conf), tableName);
481 List<Path> regionDirs =
482 FSUtils.getRegionDirs(cluster.getMaster().getFileSystem(), tableDir);
483 assertEquals("Only one region dir should be present, we have, dirs: " + regionDirs,
484 1, regionDirs.size());
485 assertTrue("Region dir doesn't belong to region: " + hri + " dir: " + regionDirs,
486 regionDirs.get(0).getName().endsWith(hri.getEncodedName()));
487 } finally {
488
489 AssignmentManager.setTestSkipSplitHandling(false);
490 admin.setBalancerRunning(true, false);
491 cluster.getMaster().setCatalogJanitorEnabled(true);
492 cluster.startRegionServer();
493 t.close();
494 TESTING_UTIL.deleteTable(tableName);
495 }
496 }
497
498 private void abortServerAndWaitForProcessingToComplete(int serverIndex) throws Exception {
499
500 final HMaster master = TESTING_UTIL.getMiniHBaseCluster().getMaster();
501 cluster.abortRegionServer(serverIndex);
502 TESTING_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
503 @Override
504 public boolean evaluate() throws Exception {
505 return master.getServerManager().areDeadServersInProgress();
506 }
507 });
508 }
509
510 @Test (timeout = 300000) public void testExistingZnodeBlocksSplitAndWeRollback()
511 throws IOException, InterruptedException, NodeExistsException, KeeperException, ServiceException {
512 final TableName tableName =
513 TableName.valueOf("testExistingZnodeBlocksSplitAndWeRollback");
514
515
516 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
517 List<HRegion> regions = cluster.getRegions(tableName);
518 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
519
520 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
521
522 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
523
524
525 this.admin.setBalancerRunning(false, true);
526
527 cluster.getMaster().setCatalogJanitorEnabled(false);
528 try {
529
530 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
531
532 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
533 printOutRegions(server, "Initial regions: ");
534 int regionCount = ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size();
535
536
537 ServerName fakedServer = ServerName.valueOf("any.old.server", 1234, -1);
538 if (useZKForAssignment) {
539 ZKAssign.createNodeClosing(TESTING_UTIL.getZooKeeperWatcher(),
540 hri, fakedServer);
541 } else {
542 regionStates.updateRegionState(hri, RegionState.State.CLOSING);
543 }
544
545
546 this.admin.split(hri.getRegionNameAsString());
547 this.admin.split(hri.getRegionNameAsString());
548 this.admin.split(hri.getRegionNameAsString());
549
550 for (int i = 0; i < 10; i++) {
551 Thread.sleep(100);
552 assertEquals(regionCount, ProtobufUtil.getOnlineRegions(
553 server.getRSRpcServices()).size());
554 }
555 if (useZKForAssignment) {
556
557 ZKAssign.deleteClosingNode(TESTING_UTIL.getZooKeeperWatcher(),
558 hri, fakedServer);
559 } else {
560 regionStates.regionOnline(hri, server.getServerName());
561 }
562
563 split(hri, server, regionCount);
564
565 checkAndGetDaughters(tableName);
566
567 } finally {
568 admin.setBalancerRunning(true, false);
569 cluster.getMaster().setCatalogJanitorEnabled(true);
570 t.close();
571 }
572 }
573
574
575
576
577
578
579
580 @Test (timeout=300000) public void testShutdownFixupWhenDaughterHasSplit()
581 throws IOException, InterruptedException, ServiceException {
582 final TableName tableName =
583 TableName.valueOf("testShutdownFixupWhenDaughterHasSplit");
584
585
586 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
587 List<HRegion> regions = cluster.getRegions(tableName);
588 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
589
590 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
591
592
593 this.admin.setBalancerRunning(false, true);
594
595 cluster.getMaster().setCatalogJanitorEnabled(false);
596 try {
597
598 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
599
600 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
601 printOutRegions(server, "Initial regions: ");
602 int regionCount = ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size();
603
604 split(hri, server, regionCount);
605
606 List<HRegion> daughters = checkAndGetDaughters(tableName);
607
608 regionCount = ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size();
609 HRegionInfo daughter = daughters.get(0).getRegionInfo();
610 LOG.info("Daughter we are going to split: " + daughter);
611
612
613 this.admin.compact(daughter.getRegionName());
614 RetryCounter retrier = new RetryCounter(30, 1, TimeUnit.SECONDS);
615 while (CompactionState.NONE != admin.getCompactionStateForRegion(daughter.getRegionName())
616 && retrier.shouldRetry()) {
617 retrier.sleepUntilNextRetry();
618 }
619
620 daughters = cluster.getRegions(tableName);
621 HRegion daughterRegion = null;
622 for (HRegion r : daughters) {
623 if (r.getRegionInfo().equals(daughter)) {
624 daughterRegion = r;
625
626 r.getStores().get(0).closeAndArchiveCompactedFiles();
627
628 LOG.info("Found matching HRI: " + daughterRegion);
629 break;
630 }
631 }
632 assertTrue(daughterRegion != null);
633 for (int i=0; i<100; i++) {
634 if (!daughterRegion.hasReferences()) break;
635 Threads.sleep(100);
636 }
637 assertFalse("Waiting for reference to be compacted", daughterRegion.hasReferences());
638 LOG.info("Daughter hri before split (has been compacted): " + daughter);
639 split(daughter, server, regionCount);
640
641 daughters = cluster.getRegions(tableName);
642 for (HRegion d: daughters) {
643 LOG.info("Regions before crash: " + d);
644 }
645
646 cluster.abortRegionServer(tableRegionIndex);
647 waitUntilRegionServerDead();
648 awaitDaughters(tableName, daughters.size());
649
650
651 regions = cluster.getRegions(tableName);
652 for (HRegion d: daughters) {
653 LOG.info("Regions after crash: " + d);
654 }
655 assertEquals(daughters.size(), regions.size());
656 for (HRegion r: regions) {
657 LOG.info("Regions post crash " + r);
658 assertTrue("Missing region post crash " + r, daughters.contains(r));
659 }
660 } finally {
661 admin.setBalancerRunning(true, false);
662 cluster.getMaster().setCatalogJanitorEnabled(true);
663 t.close();
664 }
665 }
666
667 @Test(timeout = 180000)
668 public void testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles() throws Exception {
669 TableName userTableName =
670 TableName.valueOf("testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles");
671 HTableDescriptor htd = new HTableDescriptor(userTableName);
672 HColumnDescriptor hcd = new HColumnDescriptor("col");
673 htd.addFamily(hcd);
674 admin.createTable(htd);
675 Table table = new HTable(TESTING_UTIL.getConfiguration(), userTableName);
676 try {
677 for (int i = 0; i <= 5; i++) {
678 String row = "row" + i;
679 Put p = new Put(row.getBytes());
680 String val = "Val" + i;
681 p.add("col".getBytes(), "ql".getBytes(), val.getBytes());
682 table.put(p);
683 admin.flush(userTableName.getName());
684 Delete d = new Delete(row.getBytes());
685
686 table.delete(d);
687 admin.flush(userTableName.getName());
688 }
689 admin.majorCompact(userTableName.getName());
690 List<HRegionInfo> regionsOfTable = TESTING_UTIL.getMiniHBaseCluster()
691 .getMaster().getAssignmentManager().getRegionStates()
692 .getRegionsOfTable(userTableName);
693 HRegionInfo hRegionInfo = regionsOfTable.get(0);
694 Put p = new Put("row6".getBytes());
695 p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
696 table.put(p);
697 p = new Put("row7".getBytes());
698 p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
699 table.put(p);
700 p = new Put("row8".getBytes());
701 p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
702 table.put(p);
703 admin.flush(userTableName.getName());
704 admin.split(hRegionInfo.getRegionName(), "row7".getBytes());
705 regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
706 .getAssignmentManager().getRegionStates()
707 .getRegionsOfTable(userTableName);
708
709 while (regionsOfTable.size() != 2) {
710 Thread.sleep(2000);
711 regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
712 .getAssignmentManager().getRegionStates()
713 .getRegionsOfTable(userTableName);
714 }
715 Assert.assertEquals(2, regionsOfTable.size());
716 Scan s = new Scan();
717 ResultScanner scanner = table.getScanner(s);
718 int mainTableCount = 0;
719 for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
720 mainTableCount++;
721 }
722 Assert.assertEquals(3, mainTableCount);
723 } finally {
724 table.close();
725 }
726 }
727
728
729
730
731 static class UselessTestAbortable implements Abortable {
732 boolean aborted = false;
733 @Override
734 public void abort(String why, Throwable e) {
735 LOG.warn("ABORTED (But nothing to abort): why=" + why, e);
736 aborted = true;
737 }
738
739 @Override
740 public boolean isAborted() {
741 return this.aborted;
742 }
743 }
744
745
746
747
748
749
750
751
752
753
754
755 @Test(timeout = 400000)
756 public void testMasterRestartWhenSplittingIsPartial()
757 throws IOException, InterruptedException, NodeExistsException,
758 KeeperException, DeserializationException, ServiceException {
759 final TableName tableName = TableName.valueOf("testMasterRestartWhenSplittingIsPartial");
760
761 if (!useZKForAssignment) {
762
763 return;
764 }
765
766
767 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
768 List<HRegion> regions = cluster.getRegions(tableName);
769 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
770
771 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
772
773
774 this.admin.setBalancerRunning(false, true);
775
776 cluster.getMaster().setCatalogJanitorEnabled(false);
777 ZooKeeperWatcher zkw = new ZooKeeperWatcher(t.getConfiguration(),
778 "testMasterRestartWhenSplittingIsPartial", new UselessTestAbortable());
779 try {
780
781 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
782
783 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
784 printOutRegions(server, "Initial regions: ");
785
786
787 AssignmentManager.setTestSkipSplitHandling(true);
788
789
790 this.admin.split(hri.getRegionNameAsString());
791 checkAndGetDaughters(tableName);
792
793 String path = ZKAssign.getNodeName(zkw, hri.getEncodedName());
794 Stat stats = zkw.getRecoverableZooKeeper().exists(path, false);
795 LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
796 + stats);
797 byte[] bytes = ZKAssign.getData(zkw, hri.getEncodedName());
798 RegionTransition rtd = RegionTransition.parseFrom(bytes);
799
800 assertTrue(rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)
801 || rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLITTING));
802
803
804 MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
805
806 this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
807
808
809
810 hri.setOffline(true);
811 hri.setSplit(true);
812 ServerName regionServerOfRegion = master.getAssignmentManager()
813 .getRegionStates().getRegionServerOfRegion(hri);
814 assertTrue(regionServerOfRegion != null);
815
816
817 AssignmentManager.setTestSkipSplitHandling(false);
818 String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
819 Stat stat = new Stat();
820 byte[] data = ZKUtil.getDataNoWatch(zkw, node, stat);
821
822 for (int i=0; data != null && i<60; i++) {
823 Thread.sleep(1000);
824 data = ZKUtil.getDataNoWatch(zkw, node, stat);
825 }
826 assertNull("Waited too long for ZK node to be removed: "+node, data);
827 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
828 assertTrue("Split parent should be in SPLIT state",
829 regionStates.isRegionInState(hri, State.SPLIT));
830 regionServerOfRegion = regionStates.getRegionServerOfRegion(hri);
831 assertTrue(regionServerOfRegion == null);
832 } finally {
833
834 AssignmentManager.setTestSkipSplitHandling(false);
835 admin.setBalancerRunning(true, false);
836 cluster.getMaster().setCatalogJanitorEnabled(true);
837 t.close();
838 zkw.close();
839 }
840 }
841
842
843
844
845
846
847
848
849
850 @Test (timeout = 300000)
851 public void testMasterRestartAtRegionSplitPendingCatalogJanitor()
852 throws IOException, InterruptedException, NodeExistsException,
853 KeeperException, ServiceException {
854 final TableName tableName = TableName
855 .valueOf("testMasterRestartAtRegionSplitPendingCatalogJanitor");
856
857
858 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
859 List<HRegion> regions = cluster.getRegions(tableName);
860 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
861
862 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
863
864
865 this.admin.setBalancerRunning(false, true);
866
867 cluster.getMaster().setCatalogJanitorEnabled(false);
868 ZooKeeperWatcher zkw = new ZooKeeperWatcher(t.getConfiguration(),
869 "testMasterRestartAtRegionSplitPendingCatalogJanitor", new UselessTestAbortable());
870 try {
871
872 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
873
874 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
875 printOutRegions(server, "Initial regions: ");
876
877 this.admin.split(hri.getRegionNameAsString());
878 checkAndGetDaughters(tableName);
879
880 String path = ZKAssign.getNodeName(zkw, hri.getEncodedName());
881 Stat stats = zkw.getRecoverableZooKeeper().exists(path, false);
882 LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
883 + stats);
884 String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
885 Stat stat = new Stat();
886 byte[] data = ZKUtil.getDataNoWatch(zkw, node, stat);
887
888 for (int i=0; data != null && i<60; i++) {
889 Thread.sleep(1000);
890 data = ZKUtil.getDataNoWatch(zkw, node, stat);
891 }
892 assertNull("Waited too long for ZK node to be removed: "+node, data);
893
894 MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
895
896 this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
897
898
899
900 hri.setOffline(true);
901 hri.setSplit(true);
902 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
903 assertTrue("Split parent should be in SPLIT state",
904 regionStates.isRegionInState(hri, State.SPLIT));
905 ServerName regionServerOfRegion = regionStates.getRegionServerOfRegion(hri);
906 assertTrue(regionServerOfRegion == null);
907 } finally {
908 this.admin.setBalancerRunning(true, false);
909 cluster.getMaster().setCatalogJanitorEnabled(true);
910 t.close();
911 zkw.close();
912 }
913 }
914
915
916
917
918
919
920
921
922
923
924
925
926 @Test(timeout = 60000)
927 public void testSplitBeforeSettingSplittingInZK() throws Exception,
928 InterruptedException, KeeperException {
929 testSplitBeforeSettingSplittingInZKInternals();
930 }
931
932 @Test(timeout = 60000)
933 public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Exception {
934 ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
935 final TableName tableName =
936 TableName.valueOf("testTableExistsIfTheSpecifiedTableRegionIsSplitParent");
937
938 Table t = createTableAndWait(tableName, Bytes.toBytes("cf"));
939 List<HRegion> regions = null;
940 try {
941 regions = cluster.getRegions(tableName);
942 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionInfo()
943 .getRegionName());
944 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
945 insertData(tableName, admin, t);
946
947 admin.setBalancerRunning(false, true);
948
949 cluster.getMaster().setCatalogJanitorEnabled(false);
950 boolean tableExists = MetaTableAccessor.tableExists(regionServer.getConnection(),
951 tableName);
952 assertEquals("The specified table should present.", true, tableExists);
953 final HRegion region = findSplittableRegion(regions);
954 assertTrue("not able to find a splittable region", region != null);
955 SplitTransactionImpl st = new SplitTransactionImpl(region, Bytes.toBytes("row2"));
956 try {
957 st.prepare();
958 st.createDaughters(regionServer, regionServer, null);
959 } catch (IOException e) {
960
961 }
962 tableExists = MetaTableAccessor.tableExists(regionServer.getConnection(),
963 tableName);
964 assertEquals("The specified table should present.", true, tableExists);
965 Map<String, RegionState> rit = cluster.getMaster().getAssignmentManager().getRegionStates()
966 .getRegionsInTransition();
967 assertTrue(rit.size() == 3);
968 cluster.getMaster().getAssignmentManager().regionOffline(st.getFirstDaughter());
969 cluster.getMaster().getAssignmentManager().regionOffline(st.getSecondDaughter());
970 cluster.getMaster().getAssignmentManager().regionOffline(region.getRegionInfo());
971 rit = cluster.getMaster().getAssignmentManager().getRegionStates().getRegionsInTransition();
972 assertTrue(rit.size() == 0);
973 }
974 finally {
975 admin.setBalancerRunning(true, false);
976 cluster.getMaster().setCatalogJanitorEnabled(true);
977 t.close();
978 TESTING_UTIL.deleteTable(tableName);
979 }
980 }
981
982 @Test
983 public void testSplitWithRegionReplicas() throws Exception {
984 ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
985 final TableName tableName =
986 TableName.valueOf("foobar");
987 HTableDescriptor htd = TESTING_UTIL.createTableDescriptor("foobar");
988 htd.setRegionReplication(2);
989 htd.addCoprocessor(SlowMeCopro.class.getName());
990
991 HTable t = TESTING_UTIL.createTable(htd, new byte[][]{Bytes.toBytes("cf")},
992 TESTING_UTIL.getConfiguration());
993 int count;
994 List<HRegion> oldRegions;
995 do {
996 oldRegions = cluster.getRegions(tableName);
997 Thread.sleep(10);
998 } while (oldRegions.size() != 2);
999 for (HRegion h : oldRegions) LOG.debug("OLDREGION " + h.getRegionInfo());
1000 try {
1001 int regionServerIndex = cluster.getServerWith(oldRegions.get(0).getRegionInfo()
1002 .getRegionName());
1003 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
1004 insertData(tableName, admin, t);
1005
1006 admin.setBalancerRunning(false, true);
1007
1008 cluster.getMaster().setCatalogJanitorEnabled(false);
1009 boolean tableExists = MetaTableAccessor.tableExists(regionServer.getConnection(),
1010 tableName);
1011 assertEquals("The specified table should be present.", true, tableExists);
1012 final HRegion region = findSplittableRegion(oldRegions);
1013 regionServerIndex = cluster.getServerWith(region.getRegionInfo().getRegionName());
1014 regionServer = cluster.getRegionServer(regionServerIndex);
1015 assertTrue("not able to find a splittable region", region != null);
1016 String node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
1017 region.getRegionInfo().getEncodedName());
1018 regionServer.getZooKeeper().sync(node);
1019 SplitTransactionImpl st = new SplitTransactionImpl(region, Bytes.toBytes("row2"));
1020 try {
1021 st.prepare();
1022 st.execute(regionServer, regionServer);
1023 } catch (IOException e) {
1024 e.printStackTrace();
1025 fail("Split execution should have succeeded with no exceptions thrown " + e);
1026 }
1027
1028 List<HRegion> newRegions;
1029 do {
1030 newRegions = cluster.getRegions(tableName);
1031 for (HRegion h : newRegions) LOG.debug("NEWREGION " + h.getRegionInfo());
1032 Thread.sleep(1000);
1033 } while ((newRegions.contains(oldRegions.get(0)) || newRegions.contains(oldRegions.get(1)))
1034 || newRegions.size() != 4);
1035 tableExists = MetaTableAccessor.tableExists(regionServer.getConnection(),
1036 tableName);
1037 assertEquals("The specified table should be present.", true, tableExists);
1038
1039 byte[] b1 = "row1".getBytes();
1040 Get g = new Get(b1);
1041 g.setConsistency(Consistency.STRONG);
1042
1043
1044
1045 Result r = t.get(g);
1046 Assert.assertFalse(r.isStale());
1047 LOG.info("exists stale after flush done");
1048
1049 SlowMeCopro.getCdl().set(new CountDownLatch(1));
1050 g = new Get(b1);
1051 g.setConsistency(Consistency.TIMELINE);
1052
1053 r = t.get(g);
1054 Assert.assertTrue(r.isStale());
1055 SlowMeCopro.getCdl().get().countDown();
1056 } finally {
1057 SlowMeCopro.getCdl().get().countDown();
1058 admin.setBalancerRunning(true, false);
1059 cluster.getMaster().setCatalogJanitorEnabled(true);
1060 t.close();
1061 }
1062 }
1063
1064 private void insertData(final TableName tableName, HBaseAdmin admin, Table t) throws IOException,
1065 InterruptedException {
1066 Put p = new Put(Bytes.toBytes("row1"));
1067 p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("1"));
1068 t.put(p);
1069 p = new Put(Bytes.toBytes("row2"));
1070 p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("2"));
1071 t.put(p);
1072 p = new Put(Bytes.toBytes("row3"));
1073 p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("3"));
1074 t.put(p);
1075 p = new Put(Bytes.toBytes("row4"));
1076 p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("4"));
1077 t.put(p);
1078 admin.flush(tableName);
1079 }
1080
1081
1082
1083
1084
1085 @Test(timeout = 60000)
1086 public void testSplitRegionWithNoStoreFiles()
1087 throws Exception {
1088 final TableName tableName =
1089 TableName.valueOf("testSplitRegionWithNoStoreFiles");
1090
1091 createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
1092 List<HRegion> regions = cluster.getRegions(tableName);
1093 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
1094 ensureTableRegionNotOnSameServerAsMeta(admin, hri);
1095 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionInfo()
1096 .getRegionName());
1097 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
1098
1099 this.admin.setBalancerRunning(false, true);
1100
1101 cluster.getMaster().setCatalogJanitorEnabled(false);
1102 try {
1103
1104 printOutRegions(regionServer, "Initial regions: ");
1105 Configuration conf = cluster.getConfiguration();
1106 HBaseFsck.debugLsr(conf, new Path("/"));
1107 Path rootDir = FSUtils.getRootDir(conf);
1108 FileSystem fs = TESTING_UTIL.getDFSCluster().getFileSystem();
1109 Map<String, Path> storefiles =
1110 FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
1111 assertEquals("Expected nothing but found " + storefiles.toString(), storefiles.size(), 0);
1112
1113
1114 regions = cluster.getRegions(tableName);
1115 final HRegion region = findSplittableRegion(regions);
1116 assertTrue("not able to find a splittable region", region != null);
1117
1118
1119 SplitTransactionImpl st = new MockedSplitTransaction(region, Bytes.toBytes("row2"));
1120 try {
1121 st.prepare();
1122 st.execute(regionServer, regionServer);
1123 } catch (IOException e) {
1124 fail("Split execution should have succeeded with no exceptions thrown");
1125 }
1126
1127
1128
1129 List<HRegion> daughters = cluster.getRegions(tableName);
1130 assertTrue(daughters.size() == 2);
1131
1132
1133 HBaseFsck.debugLsr(conf, new Path("/"));
1134 Map<String, Path> storefilesAfter =
1135 FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
1136 assertEquals("Expected nothing but found " + storefilesAfter.toString(),
1137 storefilesAfter.size(), 0);
1138
1139 hri = region.getRegionInfo();
1140 AssignmentManager am = cluster.getMaster().getAssignmentManager();
1141 RegionStates regionStates = am.getRegionStates();
1142 long start = EnvironmentEdgeManager.currentTime();
1143 while (!regionStates.isRegionInState(hri, State.SPLIT)) {
1144 assertFalse("Timed out in waiting split parent to be in state SPLIT",
1145 EnvironmentEdgeManager.currentTime() - start > 60000);
1146 Thread.sleep(500);
1147 }
1148
1149
1150 am.assign(hri, true, true);
1151 assertFalse("Split region can't be assigned",
1152 regionStates.isRegionInTransition(hri));
1153 assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
1154
1155
1156 am.unassign(hri, true, null);
1157 assertFalse("Split region can't be unassigned",
1158 regionStates.isRegionInTransition(hri));
1159 assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
1160 } finally {
1161 admin.setBalancerRunning(true, false);
1162 cluster.getMaster().setCatalogJanitorEnabled(true);
1163 }
1164 }
1165
1166 @Test(timeout = 180000)
1167 public void testSplitHooksBeforeAndAfterPONR() throws Exception {
1168 TableName firstTable = TableName.valueOf("testSplitHooksBeforeAndAfterPONR_1");
1169 TableName secondTable = TableName.valueOf("testSplitHooksBeforeAndAfterPONR_2");
1170 HColumnDescriptor hcd = new HColumnDescriptor("cf");
1171
1172 HTableDescriptor desc = new HTableDescriptor(firstTable);
1173 desc.addCoprocessor(MockedRegionObserver.class.getName());
1174 desc.addFamily(hcd);
1175 admin.createTable(desc);
1176 TESTING_UTIL.waitUntilAllRegionsAssigned(firstTable);
1177
1178 desc = new HTableDescriptor(secondTable);
1179 desc.addFamily(hcd);
1180 admin.createTable(desc);
1181 TESTING_UTIL.waitUntilAllRegionsAssigned(secondTable);
1182
1183 List<HRegion> firstTableRegions = cluster.getRegions(firstTable);
1184 List<HRegion> secondTableRegions = cluster.getRegions(secondTable);
1185
1186
1187 if (firstTableRegions.size() == 0 || secondTableRegions.size() == 0) {
1188 fail("Each table should have at least one region.");
1189 }
1190 ServerName serverName = cluster.getServerHoldingRegion(firstTable,
1191 firstTableRegions.get(0).getRegionInfo().getRegionName());
1192 admin.move(secondTableRegions.get(0).getRegionInfo().getEncodedNameAsBytes(),
1193 Bytes.toBytes(serverName.getServerName()));
1194 Table table1 = null;
1195 Table table2 = null;
1196 try {
1197 table1 = new HTable(TESTING_UTIL.getConfiguration(), firstTable);
1198 table2 = new HTable(TESTING_UTIL.getConfiguration(), firstTable);
1199 insertData(firstTable, admin, table1);
1200 insertData(secondTable, admin, table2);
1201 admin.split(firstTable, "row2".getBytes());
1202 firstTableRegions = cluster.getRegions(firstTable);
1203 while (firstTableRegions.size() != 2) {
1204 Thread.sleep(1000);
1205 firstTableRegions = cluster.getRegions(firstTable);
1206 }
1207 assertEquals("Number of regions after split should be 2.", 2, firstTableRegions.size());
1208 secondTableRegions = cluster.getRegions(secondTable);
1209 assertEquals("Number of regions after split should be 2.", 2, secondTableRegions.size());
1210 } finally {
1211 if (table1 != null) {
1212 table1.close();
1213 }
1214 if (table2 != null) {
1215 table2.close();
1216 }
1217 TESTING_UTIL.deleteTable(firstTable);
1218 TESTING_UTIL.deleteTable(secondTable);
1219 }
1220 }
1221
1222 private void testSplitBeforeSettingSplittingInZKInternals() throws Exception {
1223 final TableName tableName = TableName.valueOf("testSplitBeforeSettingSplittingInZK");
1224 try {
1225
1226 createTableAndWait(tableName, Bytes.toBytes("cf"));
1227
1228 List<HRegion> regions = awaitTableRegions(tableName);
1229 assertTrue("Table not online", cluster.getRegions(tableName).size() != 0);
1230
1231 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionInfo()
1232 .getRegionName());
1233 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
1234 final HRegion region = findSplittableRegion(regions);
1235 assertTrue("not able to find a splittable region", region != null);
1236 SplitTransactionImpl st = new MockedSplitTransaction(region, Bytes.toBytes("row2")) {
1237 @Override
1238 public PairOfSameType<Region> stepsBeforePONR(final Server server,
1239 final RegionServerServices services, boolean testing) throws IOException {
1240 throw new SplittingNodeCreationFailedException ();
1241 }
1242 };
1243 String node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
1244 region.getRegionInfo().getEncodedName());
1245 regionServer.getZooKeeper().sync(node);
1246 for (int i = 0; i < 100; i++) {
1247
1248
1249
1250 if (ZKUtil.checkExists(regionServer.getZooKeeper(), node) != -1) {
1251 Thread.sleep(100);
1252 }
1253 }
1254 try {
1255 st.prepare();
1256 st.execute(regionServer, regionServer);
1257 } catch (IOException e) {
1258
1259
1260
1261 assertTrue("Should be instance of CreateSplittingNodeFailedException",
1262 e instanceof SplittingNodeCreationFailedException );
1263 node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
1264 region.getRegionInfo().getEncodedName());
1265 {
1266 assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
1267 }
1268 assertTrue(st.rollback(regionServer, regionServer));
1269 assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
1270 }
1271 } finally {
1272 TESTING_UTIL.deleteTable(tableName);
1273 }
1274 }
1275
1276 @Test
1277 public void testStoreFileReferenceCreationWhenSplitPolicySaysToSkipRangeCheck()
1278 throws Exception {
1279 final TableName tableName =
1280 TableName.valueOf("testStoreFileReferenceCreationWhenSplitPolicySaysToSkipRangeCheck");
1281 try {
1282 HTableDescriptor htd = new HTableDescriptor(tableName);
1283 htd.addFamily(new HColumnDescriptor("f"));
1284 htd.addFamily(new HColumnDescriptor("i_f"));
1285 htd.setRegionSplitPolicyClassName(CustomSplitPolicy.class.getName());
1286 admin.createTable(htd);
1287 List<HRegion> regions = awaitTableRegions(tableName);
1288 HRegion region = regions.get(0);
1289 for(int i = 3;i<9;i++) {
1290 Put p = new Put(Bytes.toBytes("row"+i));
1291 p.add(Bytes.toBytes("f"), Bytes.toBytes("q"), Bytes.toBytes("value"+i));
1292 p.add(Bytes.toBytes("i_f"), Bytes.toBytes("q"), Bytes.toBytes("value"+i));
1293 region.put(p);
1294 }
1295 region.flush(true);
1296 Store store = region.getStore(Bytes.toBytes("f"));
1297 Collection<StoreFile> storefiles = store.getStorefiles();
1298 assertEquals(storefiles.size(), 1);
1299 assertFalse(region.hasReferences());
1300 Path referencePath =
1301 region.getRegionFileSystem().splitStoreFile(region.getRegionInfo(), "f",
1302 storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy());
1303 assertNull(referencePath);
1304 referencePath =
1305 region.getRegionFileSystem().splitStoreFile(region.getRegionInfo(), "i_f",
1306 storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy());
1307 assertNotNull(referencePath);
1308 } finally {
1309 TESTING_UTIL.deleteTable(tableName);
1310 }
1311 }
1312
1313 @Test(timeout = 120000)
1314 public void testFailedSplit() throws Exception {
1315 TableName tableName = TableName.valueOf("testFailedSplit");
1316 byte[] colFamily = Bytes.toBytes("info");
1317 TESTING_UTIL.createTable(tableName, colFamily);
1318 Connection connection = ConnectionFactory.createConnection(TESTING_UTIL.getConfiguration());
1319 HTable table = (HTable) connection.getTable(tableName);
1320 try {
1321 TESTING_UTIL.loadTable(table, colFamily);
1322 List<HRegionInfo> regions = TESTING_UTIL.getHBaseAdmin().getTableRegions(tableName);
1323 assertTrue(regions.size() == 1);
1324 final HRegion actualRegion = cluster.getRegions(tableName).get(0);
1325 actualRegion.getCoprocessorHost().load(FailingSplitRegionObserver.class,
1326 Coprocessor.PRIORITY_USER, actualRegion.getBaseConf());
1327
1328
1329 admin.split(tableName);
1330 FailingSplitRegionObserver observer = (FailingSplitRegionObserver) actualRegion
1331 .getCoprocessorHost().findCoprocessor(FailingSplitRegionObserver.class.getName());
1332 assertNotNull(observer);
1333 observer.latch.await();
1334 observer.postSplit.await();
1335 LOG.info("Waiting for region to come out of RIT: " + actualRegion);
1336 TESTING_UTIL.waitFor(60000, 1000, new Waiter.Predicate<Exception>() {
1337 @Override
1338 public boolean evaluate() throws Exception {
1339 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
1340 Map<String, RegionState> rit = regionStates.getRegionsInTransition();
1341 return (rit.size() == 0);
1342 }
1343 });
1344 regions = TESTING_UTIL.getHBaseAdmin().getTableRegions(tableName);
1345 assertTrue(regions.size() == 1);
1346 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
1347 Map<String, RegionState> rit = regionStates.getRegionsInTransition();
1348 assertTrue(rit.size() == 0);
1349 } finally {
1350 table.close();
1351 connection.close();
1352 TESTING_UTIL.deleteTable(tableName);
1353 }
1354 }
1355
1356 @Test (timeout=300000)
1357 public void testSSHCleanupDaugtherRegionsOfAbortedSplit() throws Exception {
1358 TableName table = TableName.valueOf("testSSHCleanupDaugtherRegionsOfAbortedSplit");
1359 try {
1360 HTableDescriptor desc = new HTableDescriptor(table);
1361 desc.addFamily(new HColumnDescriptor(Bytes.toBytes("f")));
1362 admin.createTable(desc);
1363 HTable hTable = new HTable(cluster.getConfiguration(), desc.getTableName());
1364 for(int i = 1; i < 5; i++) {
1365 Put p1 = new Put(("r"+i).getBytes());
1366 p1.add(Bytes.toBytes("f"), "q1".getBytes(), "v".getBytes());
1367 hTable.put(p1);
1368 }
1369 admin.flush(desc.getTableName());
1370 List<HRegion> regions = cluster.getRegions(desc.getTableName());
1371 int serverWith = cluster.getServerWith(regions.get(0).getRegionInfo().getRegionName());
1372 HRegionServer regionServer = cluster.getRegionServer(serverWith);
1373 cluster.getServerWith(regions.get(0).getRegionInfo().getRegionName());
1374 SplitTransactionImpl st = new SplitTransactionImpl(regions.get(0), Bytes.toBytes("r3"));
1375 st.prepare();
1376 st.stepsBeforePONR(regionServer, regionServer, false);
1377 Path tableDir =
1378 FSUtils.getTableDir(cluster.getMaster().getMasterFileSystem().getRootDir(),
1379 desc.getTableName());
1380 tableDir.getFileSystem(cluster.getConfiguration());
1381 List<Path> regionDirs =
1382 FSUtils.getRegionDirs(tableDir.getFileSystem(cluster.getConfiguration()), tableDir);
1383 assertEquals(3,regionDirs.size());
1384 cluster.startRegionServer();
1385 regionServer.kill();
1386 cluster.getRegionServerThreads().get(serverWith).join();
1387
1388 TESTING_UTIL.waitFor(60000, 1000, new Waiter.Predicate<Exception>() {
1389 @Override
1390 public boolean evaluate() throws Exception {
1391 return !cluster.getMaster().getServerManager().areDeadServersInProgress();
1392 }
1393 });
1394
1395 TESTING_UTIL.waitFor(60000, 1000, new Waiter.Predicate<Exception>() {
1396 @Override
1397 public boolean evaluate() throws Exception {
1398 return !cluster.getMaster().getAssignmentManager().
1399 getRegionStates().isRegionsInTransition();
1400 }
1401 });
1402 regionDirs =
1403 FSUtils.getRegionDirs(tableDir.getFileSystem(cluster.getConfiguration()), tableDir);
1404 assertEquals(1,regionDirs.size());
1405 } finally {
1406 TESTING_UTIL.deleteTable(table);
1407 }
1408 }
1409
1410 public static class MockedCoordinatedStateManager extends ZkCoordinatedStateManager {
1411
1412 public void initialize(Server server, HRegion region) {
1413 this.server = server;
1414 this.watcher = server.getZooKeeper();
1415 splitTransactionCoordination = new MockedSplitTransactionCoordination(this, watcher, region);
1416 closeRegionCoordination = new ZkCloseRegionCoordination(this, watcher);
1417 openRegionCoordination = new ZkOpenRegionCoordination(this, watcher);
1418 }
1419 }
1420
1421 public static class MockedSplitTransaction extends SplitTransactionImpl {
1422
1423 private HRegion currentRegion;
1424 public MockedSplitTransaction(HRegion region, byte[] splitrow) {
1425 super(region, splitrow);
1426 this.currentRegion = region;
1427 }
1428 @Override
1429 public boolean rollback(Server server, RegionServerServices services) throws IOException {
1430 if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
1431 .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
1432 if(secondSplit){
1433 super.rollback(server, services);
1434 latch.countDown();
1435 return true;
1436 }
1437 }
1438 return super.rollback(server, services);
1439 }
1440
1441
1442 }
1443
1444 public static class MockedSplitTransactionCoordination extends ZKSplitTransactionCoordination {
1445
1446 private HRegion currentRegion;
1447
1448 public MockedSplitTransactionCoordination(CoordinatedStateManager coordinationProvider,
1449 ZooKeeperWatcher watcher, HRegion region) {
1450 super(coordinationProvider, watcher);
1451 currentRegion = region;
1452 }
1453
1454 @Override
1455 public void completeSplitTransaction(RegionServerServices services, Region a, Region b,
1456 SplitTransactionDetails std, Region parent) throws IOException {
1457 if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
1458 .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
1459 try {
1460 if (!secondSplit){
1461 callRollBack = true;
1462 latch.await();
1463 }
1464 } catch (InterruptedException e) {
1465 }
1466
1467 }
1468 super.completeSplitTransaction(services, a, b, std, parent);
1469 if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
1470 .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
1471 firstSplitCompleted = true;
1472 }
1473 }
1474 }
1475
1476 private HRegion findSplittableRegion(final List<HRegion> regions) throws InterruptedException {
1477 for (int i = 0; i < 5; ++i) {
1478 for (HRegion r: regions) {
1479 if (r.isSplittable() && r.getRegionInfo().getReplicaId() == 0) {
1480 return(r);
1481 }
1482 }
1483 Thread.sleep(100);
1484 }
1485 return(null);
1486 }
1487
1488 private List<HRegion> checkAndGetDaughters(TableName tableName)
1489 throws InterruptedException {
1490 List<HRegion> daughters = null;
1491
1492 for (int i=0; i<100; i++) {
1493 daughters = cluster.getRegions(tableName);
1494 if (daughters.size() >= 2) break;
1495 Thread.sleep(100);
1496 }
1497 assertTrue(daughters.size() >= 2);
1498 return daughters;
1499 }
1500
1501 private MockMasterWithoutCatalogJanitor abortAndWaitForMaster()
1502 throws IOException, InterruptedException {
1503 cluster.abortMaster(0);
1504 cluster.waitOnMaster(0);
1505 cluster.getConfiguration().setClass(HConstants.MASTER_IMPL,
1506 MockMasterWithoutCatalogJanitor.class, HMaster.class);
1507 MockMasterWithoutCatalogJanitor master = null;
1508 master = (MockMasterWithoutCatalogJanitor) cluster.startMaster().getMaster();
1509 cluster.waitForActiveAndReadyMaster();
1510 return master;
1511 }
1512
1513 private void split(final HRegionInfo hri, final HRegionServer server, final int regionCount)
1514 throws IOException, InterruptedException {
1515 this.admin.split(hri.getRegionNameAsString());
1516 try {
1517 for (int i = 0; ProtobufUtil.getOnlineRegions(
1518 server.getRSRpcServices()).size() <= regionCount && i < 300; i++) {
1519 LOG.debug("Waiting on region to split");
1520 Thread.sleep(100);
1521 }
1522
1523 assertFalse("Waited too long for split",
1524 ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size() <= regionCount);
1525 } catch (RegionServerStoppedException e) {
1526 if (useZKForAssignment) {
1527
1528 LOG.error(e);
1529 throw e;
1530 }
1531 }
1532 }
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545 private int ensureTableRegionNotOnSameServerAsMeta(final Admin admin,
1546 final HRegionInfo hri)
1547 throws IOException, MasterNotRunningException,
1548 ZooKeeperConnectionException, InterruptedException {
1549
1550
1551
1552 int metaServerIndex = cluster.getServerWithMeta();
1553 assertTrue(metaServerIndex != -1);
1554 HRegionServer metaRegionServer = cluster.getRegionServer(metaServerIndex);
1555 int tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1556 assertTrue(tableRegionIndex != -1);
1557 HRegionServer tableRegionServer = cluster.getRegionServer(tableRegionIndex);
1558 if (metaRegionServer.getServerName().equals(tableRegionServer.getServerName())) {
1559 HRegionServer hrs = getOtherRegionServer(cluster, metaRegionServer);
1560 assertNotNull(hrs);
1561 assertNotNull(hri);
1562 LOG.info("Moving " + hri.getRegionNameAsString() + " from " +
1563 metaRegionServer.getServerName() + " to " +
1564 hrs.getServerName() + "; metaServerIndex=" + metaServerIndex);
1565 admin.move(hri.getEncodedNameAsBytes(), Bytes.toBytes(hrs.getServerName().toString()));
1566 }
1567
1568 for (int i = 0; i < 20; i++) {
1569 tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1570 if (tableRegionIndex != -1 && tableRegionIndex != metaServerIndex) break;
1571 LOG.debug("Waiting on region move off the hbase:meta server; current index " +
1572 tableRegionIndex + " and metaServerIndex=" + metaServerIndex);
1573 Thread.sleep(1000);
1574 }
1575 assertTrue("Region not moved off hbase:meta server", tableRegionIndex != -1
1576 && tableRegionIndex != metaServerIndex);
1577
1578 tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1579 assertTrue(tableRegionIndex != -1);
1580 assertNotSame(metaServerIndex, tableRegionIndex);
1581 return tableRegionIndex;
1582 }
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593 private HRegionServer getOtherRegionServer(final MiniHBaseCluster cluster,
1594 final HRegionServer notThisOne) {
1595 for (RegionServerThread rst: cluster.getRegionServerThreads()) {
1596 HRegionServer hrs = rst.getRegionServer();
1597 if (hrs.getServerName().equals(notThisOne.getServerName())) continue;
1598 if (hrs.isStopping() || hrs.isStopped()) continue;
1599 return hrs;
1600 }
1601 return null;
1602 }
1603
1604 private void printOutRegions(final HRegionServer hrs, final String prefix)
1605 throws IOException {
1606 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
1607 for (HRegionInfo region: regions) {
1608 LOG.info(prefix + region.getRegionNameAsString());
1609 }
1610 }
1611
1612 private void waitUntilRegionServerDead() throws InterruptedException, InterruptedIOException {
1613
1614 for (int i=0; cluster.getMaster().getClusterStatus().
1615 getServers().size() > NB_SERVERS && i<100; i++) {
1616 LOG.info("Waiting on server to go down");
1617 Thread.sleep(100);
1618 }
1619 assertFalse("Waited too long for RS to die", cluster.getMaster().getClusterStatus().
1620 getServers().size() > NB_SERVERS);
1621 }
1622
1623 private void awaitDaughters(TableName tableName, int numDaughters) throws InterruptedException {
1624
1625 for (int i=0; cluster.getRegions(tableName).size() < numDaughters && i<60; i++) {
1626 LOG.info("Waiting for repair to happen");
1627 Thread.sleep(1000);
1628 }
1629 if (cluster.getRegions(tableName).size() < numDaughters) {
1630 fail("Waiting too long for daughter regions");
1631 }
1632 }
1633
1634 private List<HRegion> awaitTableRegions(final TableName tableName) throws InterruptedException {
1635 List<HRegion> regions = null;
1636 for (int i = 0; i < 100; i++) {
1637 regions = cluster.getRegions(tableName);
1638 if (regions.size() > 0) break;
1639 Thread.sleep(100);
1640 }
1641 return regions;
1642 }
1643
1644 private HTable createTableAndWait(TableName tableName, byte[] cf) throws IOException,
1645 InterruptedException {
1646 HTable t = TESTING_UTIL.createTable(tableName, cf);
1647 awaitTableRegions(tableName);
1648 assertTrue("Table not online: " + tableName,
1649 cluster.getRegions(tableName).size() != 0);
1650 return t;
1651 }
1652
1653 public static class MockMasterWithoutCatalogJanitor extends HMaster {
1654
1655 public MockMasterWithoutCatalogJanitor(Configuration conf, CoordinatedStateManager cp)
1656 throws IOException, KeeperException,
1657 InterruptedException {
1658 super(conf, cp);
1659 }
1660 }
1661
1662 private static class SplittingNodeCreationFailedException extends IOException {
1663 private static final long serialVersionUID = 1652404976265623004L;
1664
1665 public SplittingNodeCreationFailedException () {
1666 super();
1667 }
1668 }
1669
1670 public static class MockedRegionObserver extends BaseRegionObserver {
1671 private SplitTransactionImpl st = null;
1672 private PairOfSameType<Region> daughterRegions = null;
1673
1674 @Override
1675 public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
1676 byte[] splitKey, List<Mutation> metaEntries) throws IOException {
1677 RegionCoprocessorEnvironment environment = ctx.getEnvironment();
1678 HRegionServer rs = (HRegionServer) environment.getRegionServerServices();
1679 List<Region> onlineRegions =
1680 rs.getOnlineRegions(TableName.valueOf("testSplitHooksBeforeAndAfterPONR_2"));
1681 Region region = onlineRegions.get(0);
1682 for (Region r : onlineRegions) {
1683 if (r.getRegionInfo().containsRow(splitKey)) {
1684 region = r;
1685 break;
1686 }
1687 }
1688 st = new SplitTransactionImpl((HRegion) region, splitKey);
1689 if (!st.prepare()) {
1690 LOG.error("Prepare for the table " + region.getTableDesc().getNameAsString()
1691 + " failed. So returning null. ");
1692 ctx.bypass();
1693 return;
1694 }
1695 ((HRegion)region).forceSplit(splitKey);
1696 daughterRegions = st.stepsBeforePONR(rs, rs, false);
1697 HRegionInfo copyOfParent = new HRegionInfo(region.getRegionInfo());
1698 copyOfParent.setOffline(true);
1699 copyOfParent.setSplit(true);
1700
1701 Put putParent = MetaTableAccessor.makePutFromRegionInfo(copyOfParent);
1702 MetaTableAccessor.addDaughtersToPut(putParent, daughterRegions.getFirst().getRegionInfo(),
1703 daughterRegions.getSecond().getRegionInfo());
1704 metaEntries.add(putParent);
1705
1706 Put putA = MetaTableAccessor.makePutFromRegionInfo(
1707 daughterRegions.getFirst().getRegionInfo());
1708 Put putB = MetaTableAccessor.makePutFromRegionInfo(
1709 daughterRegions.getSecond().getRegionInfo());
1710 st.addLocation(putA, rs.getServerName(), 1);
1711 st.addLocation(putB, rs.getServerName(), 1);
1712 metaEntries.add(putA);
1713 metaEntries.add(putB);
1714 }
1715
1716 @Override
1717 public void preSplitAfterPONR(ObserverContext<RegionCoprocessorEnvironment> ctx)
1718 throws IOException {
1719 RegionCoprocessorEnvironment environment = ctx.getEnvironment();
1720 HRegionServer rs = (HRegionServer) environment.getRegionServerServices();
1721 st.stepsAfterPONR(rs, rs, daughterRegions, null);
1722 }
1723
1724 }
1725
1726 static class CustomSplitPolicy extends RegionSplitPolicy {
1727
1728 @Override
1729 protected boolean shouldSplit() {
1730 return true;
1731 }
1732
1733 @Override
1734 public boolean skipStoreFileRangeCheck(String familyName) {
1735 if(familyName.startsWith("i_")) {
1736 return true;
1737 } else {
1738 return false;
1739 }
1740 }
1741 }
1742 }
1743