1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertNotSame;
25 import static org.junit.Assert.assertNull;
26 import static org.junit.Assert.assertTrue;
27 import static org.junit.Assert.fail;
28
29 import java.io.IOException;
30 import java.util.Collection;
31 import java.util.List;
32 import java.util.Map;
33 import java.util.Set;
34 import java.util.concurrent.CountDownLatch;
35 import java.util.concurrent.TimeUnit;
36
37 import org.apache.commons.logging.Log;
38 import org.apache.commons.logging.LogFactory;
39 import org.apache.hadoop.conf.Configuration;
40 import org.apache.hadoop.fs.FileSystem;
41 import org.apache.hadoop.fs.Path;
42 import org.apache.hadoop.hbase.Abortable;
43 import org.apache.hadoop.hbase.CoordinatedStateManager;
44 import org.apache.hadoop.hbase.Coprocessor;
45 import org.apache.hadoop.hbase.CoprocessorEnvironment;
46 import org.apache.hadoop.hbase.HBaseTestingUtility;
47 import org.apache.hadoop.hbase.HColumnDescriptor;
48 import org.apache.hadoop.hbase.HConstants;
49 import org.apache.hadoop.hbase.HRegionInfo;
50 import org.apache.hadoop.hbase.HTableDescriptor;
51 import org.apache.hadoop.hbase.MasterNotRunningException;
52 import org.apache.hadoop.hbase.MetaTableAccessor;
53 import org.apache.hadoop.hbase.MiniHBaseCluster;
54 import org.apache.hadoop.hbase.RegionTransition;
55 import org.apache.hadoop.hbase.Server;
56 import org.apache.hadoop.hbase.ServerName;
57 import org.apache.hadoop.hbase.TableName;
58 import org.apache.hadoop.hbase.UnknownRegionException;
59 import org.apache.hadoop.hbase.Waiter;
60 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
61 import org.apache.hadoop.hbase.client.Admin;
62 import org.apache.hadoop.hbase.client.Connection;
63 import org.apache.hadoop.hbase.client.ConnectionFactory;
64 import org.apache.hadoop.hbase.client.Consistency;
65 import org.apache.hadoop.hbase.client.Delete;
66 import org.apache.hadoop.hbase.client.Get;
67 import org.apache.hadoop.hbase.client.HBaseAdmin;
68 import org.apache.hadoop.hbase.client.HTable;
69 import org.apache.hadoop.hbase.client.Mutation;
70 import org.apache.hadoop.hbase.client.Put;
71 import org.apache.hadoop.hbase.client.Result;
72 import org.apache.hadoop.hbase.client.ResultScanner;
73 import org.apache.hadoop.hbase.client.Scan;
74 import org.apache.hadoop.hbase.client.Table;
75 import org.apache.hadoop.hbase.client.TestReplicasClient.SlowMeCopro;
76 import org.apache.hadoop.hbase.coordination.ZKSplitTransactionCoordination;
77 import org.apache.hadoop.hbase.coordination.ZkCloseRegionCoordination;
78 import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
79 import org.apache.hadoop.hbase.coordination.ZkOpenRegionCoordination;
80 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
81 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
82 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
83 import org.apache.hadoop.hbase.exceptions.DeserializationException;
84 import org.apache.hadoop.hbase.executor.EventType;
85 import org.apache.hadoop.hbase.master.AssignmentManager;
86 import org.apache.hadoop.hbase.master.HMaster;
87 import org.apache.hadoop.hbase.master.RegionState;
88 import org.apache.hadoop.hbase.master.RegionState.State;
89 import org.apache.hadoop.hbase.master.RegionStates;
90 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
91 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
92 import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
93 import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
94 import org.apache.hadoop.hbase.testclassification.LargeTests;
95 import org.apache.hadoop.hbase.util.Bytes;
96 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
97 import org.apache.hadoop.hbase.util.FSUtils;
98 import org.apache.hadoop.hbase.util.HBaseFsck;
99 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
100 import org.apache.hadoop.hbase.util.PairOfSameType;
101 import org.apache.hadoop.hbase.util.RetryCounter;
102 import org.apache.hadoop.hbase.util.Threads;
103 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
104 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
105 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
106 import org.apache.zookeeper.KeeperException;
107 import org.apache.zookeeper.KeeperException.NodeExistsException;
108 import org.apache.zookeeper.data.Stat;
109 import org.junit.After;
110 import org.junit.AfterClass;
111 import org.junit.Assert;
112 import org.junit.Before;
113 import org.junit.BeforeClass;
114 import org.junit.Test;
115 import org.junit.experimental.categories.Category;
116
117 import com.google.protobuf.ServiceException;
118
119
120
121
122
123
124 @Category(LargeTests.class)
125 @SuppressWarnings("deprecation")
126 public class TestSplitTransactionOnCluster {
127 private static final Log LOG =
128 LogFactory.getLog(TestSplitTransactionOnCluster.class);
129 private HBaseAdmin admin = null;
130 private MiniHBaseCluster cluster = null;
131 private static Configuration conf;
132 private static final int NB_SERVERS = 3;
133 private static CountDownLatch latch = new CountDownLatch(1);
134 private static volatile boolean secondSplit = false;
135 private static volatile boolean callRollBack = false;
136 private static volatile boolean firstSplitCompleted = false;
137 private static boolean useZKForAssignment;
138
139 static final HBaseTestingUtility TESTING_UTIL =
140 new HBaseTestingUtility();
141
142 static void setupOnce() throws Exception {
143 TESTING_UTIL.getConfiguration().setInt(HConstants.HBASE_BALANCER_PERIOD, 60000);
144 useZKForAssignment = TESTING_UTIL.getConfiguration().getBoolean(
145 "hbase.assignment.usezk", true);
146 TESTING_UTIL.startMiniCluster(NB_SERVERS);
147 }
148
149 @BeforeClass
150 public static void before() throws Exception {
151 conf = TESTING_UTIL.getConfiguration();
152
153 conf.setBoolean("hbase.assignment.usezk", true);
154 setupOnce();
155 }
156
157 @AfterClass public static void after() throws Exception {
158 TESTING_UTIL.shutdownMiniCluster();
159 }
160
161 @Before public void setup() throws IOException {
162 TESTING_UTIL.ensureSomeNonStoppedRegionServersAvailable(NB_SERVERS);
163 this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
164 this.cluster = TESTING_UTIL.getMiniHBaseCluster();
165 }
166
167 @After
168 public void tearDown() throws Exception {
169 this.admin.close();
170 }
171
172 private HRegionInfo getAndCheckSingleTableRegion(final List<HRegion> regions)
173 throws IOException, InterruptedException {
174 assertEquals(1, regions.size());
175 HRegionInfo hri = regions.get(0).getRegionInfo();
176 TESTING_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
177 .waitOnRegionToClearRegionsInTransition(hri, 600000);
178 return hri;
179 }
180
181 @Test(timeout = 60000)
182 public void testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack() throws Exception {
183 final TableName tableName =
184 TableName.valueOf("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack");
185
186 if (!useZKForAssignment) {
187
188 return;
189 }
190
191 try {
192
193 HTable t = createTableAndWait(tableName, Bytes.toBytes("cf"));
194 final List<HRegion> regions = cluster.getRegions(tableName);
195 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
196 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionInfo()
197 .getRegionName());
198 final HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
199 insertData(tableName, admin, t);
200 t.close();
201
202
203 this.admin.setBalancerRunning(false, true);
204
205 cluster.getMaster().setCatalogJanitorEnabled(false);
206
207
208 final HRegion region = findSplittableRegion(regions);
209 assertTrue("not able to find a splittable region", region != null);
210 MockedCoordinatedStateManager cp = new MockedCoordinatedStateManager();
211 cp.initialize(regionServer, region);
212 cp.start();
213 regionServer.csm = cp;
214
215 new Thread() {
216 @Override
217 public void run() {
218 SplitTransaction st = null;
219 st = new MockedSplitTransaction(region, Bytes.toBytes("row2"));
220 try {
221 st.prepare();
222 st.execute(regionServer, regionServer);
223 } catch (IOException e) {
224
225 }
226 }
227 }.start();
228 for (int i = 0; !callRollBack && i < 100; i++) {
229 Thread.sleep(100);
230 }
231 assertTrue("Waited too long for rollback", callRollBack);
232 SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row3"));
233 try {
234 secondSplit = true;
235
236 region.initialize();
237 st.prepare();
238 st.execute(regionServer, regionServer);
239 } catch (IOException e) {
240 LOG.debug("Rollback started :"+ e.getMessage());
241 st.rollback(regionServer, regionServer);
242 }
243 for (int i=0; !firstSplitCompleted && i<100; i++) {
244 Thread.sleep(100);
245 }
246 assertTrue("fist split did not complete", firstSplitCompleted);
247
248 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
249 for (int i = 0; regionStates.isRegionInTransition(hri) && i < 100; i++) {
250 Thread.sleep(100);
251 }
252 assertFalse("region still in transition", regionStates.isRegionInTransition(hri));
253
254 List<Region> onlineRegions = regionServer.getOnlineRegions(tableName);
255
256 assertEquals("The parent region should be splitted", 2, onlineRegions.size());
257
258 List<HRegionInfo> regionsOfTable = cluster.getMaster().getAssignmentManager()
259 .getRegionStates().getRegionsOfTable(tableName);
260
261 assertEquals("No of regions in master", 2, regionsOfTable.size());
262 } finally {
263 admin.setBalancerRunning(true, false);
264 secondSplit = false;
265 firstSplitCompleted = false;
266 callRollBack = false;
267 cluster.getMaster().setCatalogJanitorEnabled(true);
268 TESTING_UTIL.deleteTable(tableName);
269 }
270 }
271
272 @Test(timeout = 60000)
273 public void testRITStateForRollback() throws Exception {
274 final TableName tableName =
275 TableName.valueOf("testRITStateForRollback");
276 try {
277
278 Table t = createTableAndWait(tableName, Bytes.toBytes("cf"));
279 final List<HRegion> regions = cluster.getRegions(tableName);
280 final HRegionInfo hri = getAndCheckSingleTableRegion(regions);
281 insertData(tableName, admin, t);
282 t.close();
283
284
285 this.admin.setBalancerRunning(false, true);
286
287 cluster.getMaster().setCatalogJanitorEnabled(false);
288
289
290 final HRegion region = findSplittableRegion(regions);
291 assertTrue("not able to find a splittable region", region != null);
292
293
294 region.getCoprocessorHost().load(FailingSplitRegionObserver.class,
295 Coprocessor.PRIORITY_USER, region.getBaseConf());
296
297
298 this.admin.split(region.getRegionInfo().getRegionName(), new byte[] {42});
299
300
301 FailingSplitRegionObserver observer = (FailingSplitRegionObserver) region
302 .getCoprocessorHost().findCoprocessor(FailingSplitRegionObserver.class.getName());
303 assertNotNull(observer);
304 observer.latch.await();
305
306 LOG.info("Waiting for region to come out of RIT");
307 TESTING_UTIL.waitUntilNoRegionsInTransition(60000);
308 } finally {
309 admin.setBalancerRunning(true, false);
310 cluster.getMaster().setCatalogJanitorEnabled(true);
311 TESTING_UTIL.deleteTable(tableName);
312 }
313 }
314 @Test(timeout = 60000)
315 public void testSplitFailedCompactionAndSplit() throws Exception {
316 final TableName tableName = TableName.valueOf("testSplitFailedCompactionAndSplit");
317 Configuration conf = TESTING_UTIL.getConfiguration();
318 try {
319 HBaseAdmin admin = new HBaseAdmin(conf);
320
321 HTableDescriptor htd = new HTableDescriptor(tableName);
322 byte[] cf = Bytes.toBytes("cf");
323 htd.addFamily(new HColumnDescriptor(cf));
324 admin.createTable(htd);
325
326 for (int i = 0; cluster.getRegions(tableName).size() == 0 && i < 100; i++) {
327 Thread.sleep(100);
328 }
329 assertEquals(1, cluster.getRegions(tableName).size());
330
331 HRegion region = cluster.getRegions(tableName).get(0);
332 Store store = region.getStore(cf);
333 int regionServerIndex = cluster.getServerWith(region.getRegionInfo().getRegionName());
334 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
335
336 Table t = new HTable(conf, tableName);
337
338 insertData(tableName, admin, t);
339 insertData(tableName, admin, t);
340
341 int fileNum = store.getStorefiles().size();
342
343 store.triggerMajorCompaction();
344 CompactionContext cc = store.requestCompaction();
345 assertNotNull(cc);
346
347
348 assertEquals(2, region.close(false).get(cf).size());
349
350 region.initialize();
351
352
353 assertFalse(region.compact(cc, store, NoLimitThroughputController.INSTANCE));
354 assertTrue(fileNum > store.getStorefiles().size());
355
356
357 SplitTransaction st = new SplitTransactionImpl(region, Bytes.toBytes("row3"));
358 assertTrue(st.prepare());
359 st.execute(regionServer, regionServer);
360 LOG.info("Waiting for region to come out of RIT");
361 TESTING_UTIL.waitFor(60000, 1000, new Waiter.Predicate<Exception>() {
362 @Override
363 public boolean evaluate() throws Exception {
364 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
365 return !regionStates.isRegionsInTransition();
366 }
367 });
368 assertEquals(2, cluster.getRegions(tableName).size());
369 } finally {
370 TESTING_UTIL.deleteTable(tableName);
371 }
372 }
373
374 public static class FailingSplitRegionObserver extends BaseRegionObserver {
375 volatile CountDownLatch latch;
376 volatile CountDownLatch postSplit;
377 @Override
378 public void start(CoprocessorEnvironment e) throws IOException {
379 latch = new CountDownLatch(1);
380 postSplit = new CountDownLatch(1);
381 }
382 @Override
383 public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
384 byte[] splitKey, List<Mutation> metaEntries) throws IOException {
385 latch.countDown();
386 LOG.info("Causing rollback of region split");
387 throw new IOException("Causing rollback of region split");
388 }
389 @Override
390 public void postCompleteSplit(ObserverContext<RegionCoprocessorEnvironment> ctx)
391 throws IOException {
392 postSplit.countDown();
393 LOG.info("postCompleteSplit called");
394 }
395 }
396
397
398
399
400
401
402
403 @Test (timeout = 60000)
404 public void testSplitIsRolledBackOnSplitFailure() throws Exception {
405 final TableName tableName = TableName.valueOf("testSplitIsRolledBackOnSplitFailure");
406
407
408 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
409 List<HRegion> regions = cluster.getRegions(tableName);
410 final HRegionInfo hri = getAndCheckSingleTableRegion(regions);
411
412
413 this.admin.setBalancerRunning(false, true);
414
415 cluster.getMaster().setCatalogJanitorEnabled(false);
416
417 int serverIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
418
419 try {
420
421 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
422
423 HRegionServer server = cluster.getRegionServer(serverIndex);
424 printOutRegions(server, "Initial regions: ");
425 int regionCount = ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size();
426
427
428 AssignmentManager.setTestSkipSplitHandling(true);
429
430 split(hri, server, regionCount);
431
432 ZooKeeperWatcher zkw = TESTING_UTIL.getZooKeeperWatcher();
433 String path = ZKAssign.getNodeName(zkw, hri.getEncodedName());
434 RegionTransition rt = null;
435 Stat stats = null;
436
437
438 for (int i = 0; i < 100; i++) {
439 stats = zkw.getRecoverableZooKeeper().exists(path, false);
440 rt = RegionTransition.parseFrom(ZKAssign.getData(zkw, hri.getEncodedName()));
441 if (rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)) {
442 break;
443 }
444 Thread.sleep(100);
445 }
446 LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats=" + stats);
447 assertTrue(rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT));
448
449 abortServerAndWaitForProcessingToComplete(serverIndex);
450 waitUntilRegionServerDead();
451
452 TESTING_UTIL.waitUntilNoRegionsInTransition();
453
454
455 TESTING_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
456 @Override
457 public boolean evaluate() {
458 for (HRegion region : cluster.getRegions(tableName)) {
459 if (Bytes.equals(region.getRegionInfo().getRegionName(), hri.getRegionName())) {
460 return true;
461 } else {
462 LOG.debug("Wait for some more time, online region: " + region);
463 }
464 }
465 return false;
466 }
467 });
468
469 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
470 assertTrue("Parent region should be online", regionStates.isRegionOnline(hri));
471
472 List<HRegionInfo> tableRegions = MetaTableAccessor.getTableRegions(zkw,
473 cluster.getMaster().getConnection(), tableName);
474 assertEquals("Only parent region should be present, but we have: " + tableRegions,
475 1, tableRegions.size());
476 Path tableDir = FSUtils.getTableDir(FSUtils.getRootDir(conf), tableName);
477 List<Path> regionDirs =
478 FSUtils.getRegionDirs(cluster.getMaster().getFileSystem(), tableDir);
479 assertEquals("Only one region dir should be present, we have, dirs: " + regionDirs,
480 1, regionDirs.size());
481 assertTrue("Region dir doesn't belong to region: " + hri + " dir: " + regionDirs,
482 regionDirs.get(0).getName().endsWith(hri.getEncodedName()));
483 } finally {
484
485 AssignmentManager.setTestSkipSplitHandling(false);
486 admin.setBalancerRunning(true, false);
487 cluster.getMaster().setCatalogJanitorEnabled(true);
488 cluster.startRegionServer();
489 t.close();
490 TESTING_UTIL.deleteTable(tableName);
491 }
492 }
493
494 private void abortServerAndWaitForProcessingToComplete(int serverIndex) throws Exception {
495
496 final HMaster master = TESTING_UTIL.getMiniHBaseCluster().getMaster();
497 cluster.abortRegionServer(serverIndex);
498 TESTING_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
499 @Override
500 public boolean evaluate() throws Exception {
501 return master.getServerManager().areDeadServersInProgress();
502 }
503 });
504 }
505
506 @Test (timeout = 300000) public void testExistingZnodeBlocksSplitAndWeRollback()
507 throws IOException, InterruptedException, NodeExistsException, KeeperException, ServiceException {
508 final TableName tableName =
509 TableName.valueOf("testExistingZnodeBlocksSplitAndWeRollback");
510
511
512 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
513 List<HRegion> regions = cluster.getRegions(tableName);
514 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
515
516 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
517
518 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
519
520
521 this.admin.setBalancerRunning(false, true);
522
523 cluster.getMaster().setCatalogJanitorEnabled(false);
524 try {
525
526 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
527
528 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
529 printOutRegions(server, "Initial regions: ");
530 int regionCount = ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size();
531
532
533 ServerName fakedServer = ServerName.valueOf("any.old.server", 1234, -1);
534 if (useZKForAssignment) {
535 ZKAssign.createNodeClosing(TESTING_UTIL.getZooKeeperWatcher(),
536 hri, fakedServer);
537 } else {
538 regionStates.updateRegionState(hri, RegionState.State.CLOSING);
539 }
540
541
542 this.admin.split(hri.getRegionNameAsString());
543 this.admin.split(hri.getRegionNameAsString());
544 this.admin.split(hri.getRegionNameAsString());
545
546 for (int i = 0; i < 10; i++) {
547 Thread.sleep(100);
548 assertEquals(regionCount, ProtobufUtil.getOnlineRegions(
549 server.getRSRpcServices()).size());
550 }
551 if (useZKForAssignment) {
552
553 ZKAssign.deleteClosingNode(TESTING_UTIL.getZooKeeperWatcher(),
554 hri, fakedServer);
555 } else {
556 regionStates.regionOnline(hri, server.getServerName());
557 }
558
559 split(hri, server, regionCount);
560
561 checkAndGetDaughters(tableName);
562
563 } finally {
564 admin.setBalancerRunning(true, false);
565 cluster.getMaster().setCatalogJanitorEnabled(true);
566 t.close();
567 }
568 }
569
570
571
572
573
574
575
576 @Test (timeout=300000) public void testShutdownFixupWhenDaughterHasSplit()
577 throws IOException, InterruptedException, ServiceException {
578 final TableName tableName =
579 TableName.valueOf("testShutdownFixupWhenDaughterHasSplit");
580
581
582 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
583 List<HRegion> regions = cluster.getRegions(tableName);
584 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
585
586 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
587
588
589 this.admin.setBalancerRunning(false, true);
590
591 cluster.getMaster().setCatalogJanitorEnabled(false);
592 try {
593
594 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
595
596 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
597 printOutRegions(server, "Initial regions: ");
598 int regionCount = ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size();
599
600 split(hri, server, regionCount);
601
602 List<HRegion> daughters = checkAndGetDaughters(tableName);
603
604 regionCount = ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size();
605 HRegionInfo daughter = daughters.get(0).getRegionInfo();
606 LOG.info("Daughter we are going to split: " + daughter);
607
608
609 this.admin.compact(daughter.getRegionName());
610 RetryCounter retrier = new RetryCounter(30, 1, TimeUnit.SECONDS);
611 while (CompactionState.NONE != admin.getCompactionStateForRegion(daughter.getRegionName())
612 && retrier.shouldRetry()) {
613 retrier.sleepUntilNextRetry();
614 }
615
616 daughters = cluster.getRegions(tableName);
617 HRegion daughterRegion = null;
618 for (HRegion r : daughters) {
619 if (r.getRegionInfo().equals(daughter)) {
620 daughterRegion = r;
621
622 r.getStores().get(0).closeAndArchiveCompactedFiles();
623
624 LOG.info("Found matching HRI: " + daughterRegion);
625 break;
626 }
627 }
628 assertTrue(daughterRegion != null);
629 for (int i=0; i<100; i++) {
630 if (!daughterRegion.hasReferences()) break;
631 Threads.sleep(100);
632 }
633 assertFalse("Waiting for reference to be compacted", daughterRegion.hasReferences());
634 LOG.info("Daughter hri before split (has been compacted): " + daughter);
635 split(daughter, server, regionCount);
636
637 daughters = cluster.getRegions(tableName);
638 for (HRegion d: daughters) {
639 LOG.info("Regions before crash: " + d);
640 }
641
642 cluster.abortRegionServer(tableRegionIndex);
643 waitUntilRegionServerDead();
644 awaitDaughters(tableName, daughters.size());
645
646
647 regions = cluster.getRegions(tableName);
648 for (HRegion d: daughters) {
649 LOG.info("Regions after crash: " + d);
650 }
651 assertEquals(daughters.size(), regions.size());
652 for (HRegion r: regions) {
653 LOG.info("Regions post crash " + r);
654 assertTrue("Missing region post crash " + r, daughters.contains(r));
655 }
656 } finally {
657 admin.setBalancerRunning(true, false);
658 cluster.getMaster().setCatalogJanitorEnabled(true);
659 t.close();
660 }
661 }
662
663 @Test(timeout = 180000)
664 public void testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles() throws Exception {
665 TableName userTableName =
666 TableName.valueOf("testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles");
667 HTableDescriptor htd = new HTableDescriptor(userTableName);
668 HColumnDescriptor hcd = new HColumnDescriptor("col");
669 htd.addFamily(hcd);
670 admin.createTable(htd);
671 Table table = new HTable(TESTING_UTIL.getConfiguration(), userTableName);
672 try {
673 for (int i = 0; i <= 5; i++) {
674 String row = "row" + i;
675 Put p = new Put(row.getBytes());
676 String val = "Val" + i;
677 p.add("col".getBytes(), "ql".getBytes(), val.getBytes());
678 table.put(p);
679 admin.flush(userTableName.getName());
680 Delete d = new Delete(row.getBytes());
681
682 table.delete(d);
683 admin.flush(userTableName.getName());
684 }
685 admin.majorCompact(userTableName.getName());
686 List<HRegionInfo> regionsOfTable = TESTING_UTIL.getMiniHBaseCluster()
687 .getMaster().getAssignmentManager().getRegionStates()
688 .getRegionsOfTable(userTableName);
689 HRegionInfo hRegionInfo = regionsOfTable.get(0);
690 Put p = new Put("row6".getBytes());
691 p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
692 table.put(p);
693 p = new Put("row7".getBytes());
694 p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
695 table.put(p);
696 p = new Put("row8".getBytes());
697 p.add("col".getBytes(), "ql".getBytes(), "val".getBytes());
698 table.put(p);
699 admin.flush(userTableName.getName());
700 admin.split(hRegionInfo.getRegionName(), "row7".getBytes());
701 regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
702 .getAssignmentManager().getRegionStates()
703 .getRegionsOfTable(userTableName);
704
705 while (regionsOfTable.size() != 2) {
706 Thread.sleep(2000);
707 regionsOfTable = TESTING_UTIL.getMiniHBaseCluster().getMaster()
708 .getAssignmentManager().getRegionStates()
709 .getRegionsOfTable(userTableName);
710 }
711 Assert.assertEquals(2, regionsOfTable.size());
712 Scan s = new Scan();
713 ResultScanner scanner = table.getScanner(s);
714 int mainTableCount = 0;
715 for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
716 mainTableCount++;
717 }
718 Assert.assertEquals(3, mainTableCount);
719 } finally {
720 table.close();
721 }
722 }
723
724
725
726
727 static class UselessTestAbortable implements Abortable {
728 boolean aborted = false;
729 @Override
730 public void abort(String why, Throwable e) {
731 LOG.warn("ABORTED (But nothing to abort): why=" + why, e);
732 aborted = true;
733 }
734
735 @Override
736 public boolean isAborted() {
737 return this.aborted;
738 }
739 }
740
741
742
743
744
745
746
747
748
749
750
751 @Test(timeout = 400000)
752 public void testMasterRestartWhenSplittingIsPartial()
753 throws IOException, InterruptedException, NodeExistsException,
754 KeeperException, DeserializationException, ServiceException {
755 final TableName tableName = TableName.valueOf("testMasterRestartWhenSplittingIsPartial");
756
757 if (!useZKForAssignment) {
758
759 return;
760 }
761
762
763 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
764 List<HRegion> regions = cluster.getRegions(tableName);
765 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
766
767 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
768
769
770 this.admin.setBalancerRunning(false, true);
771
772 cluster.getMaster().setCatalogJanitorEnabled(false);
773 ZooKeeperWatcher zkw = new ZooKeeperWatcher(t.getConfiguration(),
774 "testMasterRestartWhenSplittingIsPartial", new UselessTestAbortable());
775 try {
776
777 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
778
779 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
780 printOutRegions(server, "Initial regions: ");
781
782
783 AssignmentManager.setTestSkipSplitHandling(true);
784
785
786 this.admin.split(hri.getRegionNameAsString());
787 checkAndGetDaughters(tableName);
788
789 String path = ZKAssign.getNodeName(zkw, hri.getEncodedName());
790 Stat stats = zkw.getRecoverableZooKeeper().exists(path, false);
791 LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
792 + stats);
793 byte[] bytes = ZKAssign.getData(zkw, hri.getEncodedName());
794 RegionTransition rtd = RegionTransition.parseFrom(bytes);
795
796 assertTrue(rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)
797 || rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLITTING));
798
799
800 MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
801
802 this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
803
804
805
806 hri.setOffline(true);
807 hri.setSplit(true);
808 ServerName regionServerOfRegion = master.getAssignmentManager()
809 .getRegionStates().getRegionServerOfRegion(hri);
810 assertTrue(regionServerOfRegion != null);
811
812
813 AssignmentManager.setTestSkipSplitHandling(false);
814 String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
815 Stat stat = new Stat();
816 byte[] data = ZKUtil.getDataNoWatch(zkw, node, stat);
817
818 for (int i=0; data != null && i<60; i++) {
819 Thread.sleep(1000);
820 data = ZKUtil.getDataNoWatch(zkw, node, stat);
821 }
822 assertNull("Waited too long for ZK node to be removed: "+node, data);
823 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
824 assertTrue("Split parent should be in SPLIT state",
825 regionStates.isRegionInState(hri, State.SPLIT));
826 regionServerOfRegion = regionStates.getRegionServerOfRegion(hri);
827 assertTrue(regionServerOfRegion == null);
828 } finally {
829
830 AssignmentManager.setTestSkipSplitHandling(false);
831 admin.setBalancerRunning(true, false);
832 cluster.getMaster().setCatalogJanitorEnabled(true);
833 t.close();
834 zkw.close();
835 }
836 }
837
838
839
840
841
842
843
844
845
846 @Test (timeout = 300000)
847 public void testMasterRestartAtRegionSplitPendingCatalogJanitor()
848 throws IOException, InterruptedException, NodeExistsException,
849 KeeperException, ServiceException {
850 final TableName tableName = TableName
851 .valueOf("testMasterRestartAtRegionSplitPendingCatalogJanitor");
852
853
854 HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
855 List<HRegion> regions = cluster.getRegions(tableName);
856 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
857
858 int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
859
860
861 this.admin.setBalancerRunning(false, true);
862
863 cluster.getMaster().setCatalogJanitorEnabled(false);
864 ZooKeeperWatcher zkw = new ZooKeeperWatcher(t.getConfiguration(),
865 "testMasterRestartAtRegionSplitPendingCatalogJanitor", new UselessTestAbortable());
866 try {
867
868 TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
869
870 HRegionServer server = cluster.getRegionServer(tableRegionIndex);
871 printOutRegions(server, "Initial regions: ");
872
873 this.admin.split(hri.getRegionNameAsString());
874 checkAndGetDaughters(tableName);
875
876 String path = ZKAssign.getNodeName(zkw, hri.getEncodedName());
877 Stat stats = zkw.getRecoverableZooKeeper().exists(path, false);
878 LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
879 + stats);
880 String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
881 Stat stat = new Stat();
882 byte[] data = ZKUtil.getDataNoWatch(zkw, node, stat);
883
884 for (int i=0; data != null && i<60; i++) {
885 Thread.sleep(1000);
886 data = ZKUtil.getDataNoWatch(zkw, node, stat);
887 }
888 assertNull("Waited too long for ZK node to be removed: "+node, data);
889
890 MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
891
892 this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
893
894
895
896 hri.setOffline(true);
897 hri.setSplit(true);
898 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
899 assertTrue("Split parent should be in SPLIT state",
900 regionStates.isRegionInState(hri, State.SPLIT));
901 ServerName regionServerOfRegion = regionStates.getRegionServerOfRegion(hri);
902 assertTrue(regionServerOfRegion == null);
903 } finally {
904 this.admin.setBalancerRunning(true, false);
905 cluster.getMaster().setCatalogJanitorEnabled(true);
906 t.close();
907 zkw.close();
908 }
909 }
910
911
912
913
914
915
916
917
918
919
920
921
922 @Test(timeout = 60000)
923 public void testSplitBeforeSettingSplittingInZK() throws Exception,
924 InterruptedException, KeeperException {
925 testSplitBeforeSettingSplittingInZKInternals();
926 }
927
928 @Test(timeout = 60000)
929 public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Exception {
930 ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
931 final TableName tableName =
932 TableName.valueOf("testTableExistsIfTheSpecifiedTableRegionIsSplitParent");
933
934 Table t = createTableAndWait(tableName, Bytes.toBytes("cf"));
935 List<HRegion> regions = null;
936 try {
937 regions = cluster.getRegions(tableName);
938 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionInfo()
939 .getRegionName());
940 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
941 insertData(tableName, admin, t);
942
943 admin.setBalancerRunning(false, true);
944
945 cluster.getMaster().setCatalogJanitorEnabled(false);
946 boolean tableExists = MetaTableAccessor.tableExists(regionServer.getConnection(),
947 tableName);
948 assertEquals("The specified table should present.", true, tableExists);
949 final HRegion region = findSplittableRegion(regions);
950 assertTrue("not able to find a splittable region", region != null);
951 SplitTransactionImpl st = new SplitTransactionImpl(region, Bytes.toBytes("row2"));
952 try {
953 st.prepare();
954 st.createDaughters(regionServer, regionServer, null);
955 } catch (IOException e) {
956
957 }
958 tableExists = MetaTableAccessor.tableExists(regionServer.getConnection(),
959 tableName);
960 assertEquals("The specified table should present.", true, tableExists);
961 Set<RegionState> rit = cluster.getMaster().getAssignmentManager().getRegionStates()
962 .getRegionsInTransition();
963 assertTrue(rit.size() == 3);
964 cluster.getMaster().getAssignmentManager().regionOffline(st.getFirstDaughter());
965 cluster.getMaster().getAssignmentManager().regionOffline(st.getSecondDaughter());
966 cluster.getMaster().getAssignmentManager().regionOffline(region.getRegionInfo());
967 rit = cluster.getMaster().getAssignmentManager().getRegionStates().getRegionsInTransition();
968 assertTrue(rit.size() == 0);
969 }
970 finally {
971 admin.setBalancerRunning(true, false);
972 cluster.getMaster().setCatalogJanitorEnabled(true);
973 t.close();
974 TESTING_UTIL.deleteTable(tableName);
975 }
976 }
977
978 @Test
979 public void testSplitWithRegionReplicas() throws Exception {
980 ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
981 final TableName tableName =
982 TableName.valueOf("foobar");
983 HTableDescriptor htd = TESTING_UTIL.createTableDescriptor("foobar");
984 htd.setRegionReplication(2);
985 htd.addCoprocessor(SlowMeCopro.class.getName());
986
987 HTable t = TESTING_UTIL.createTable(htd, new byte[][]{Bytes.toBytes("cf")},
988 TESTING_UTIL.getConfiguration());
989 int count;
990 List<HRegion> oldRegions;
991 do {
992 oldRegions = cluster.getRegions(tableName);
993 Thread.sleep(10);
994 } while (oldRegions.size() != 2);
995 for (HRegion h : oldRegions) LOG.debug("OLDREGION " + h.getRegionInfo());
996 try {
997 int regionServerIndex = cluster.getServerWith(oldRegions.get(0).getRegionInfo()
998 .getRegionName());
999 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
1000 insertData(tableName, admin, t);
1001
1002 admin.setBalancerRunning(false, true);
1003
1004 cluster.getMaster().setCatalogJanitorEnabled(false);
1005 boolean tableExists = MetaTableAccessor.tableExists(regionServer.getConnection(),
1006 tableName);
1007 assertEquals("The specified table should be present.", true, tableExists);
1008 final HRegion region = findSplittableRegion(oldRegions);
1009 regionServerIndex = cluster.getServerWith(region.getRegionInfo().getRegionName());
1010 regionServer = cluster.getRegionServer(regionServerIndex);
1011 assertTrue("not able to find a splittable region", region != null);
1012 String node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
1013 region.getRegionInfo().getEncodedName());
1014 regionServer.getZooKeeper().sync(node);
1015 SplitTransactionImpl st = new SplitTransactionImpl(region, Bytes.toBytes("row2"));
1016 try {
1017 st.prepare();
1018 st.execute(regionServer, regionServer);
1019 } catch (IOException e) {
1020 e.printStackTrace();
1021 fail("Split execution should have succeeded with no exceptions thrown " + e);
1022 }
1023
1024 List<HRegion> newRegions;
1025 do {
1026 newRegions = cluster.getRegions(tableName);
1027 for (HRegion h : newRegions) LOG.debug("NEWREGION " + h.getRegionInfo());
1028 Thread.sleep(1000);
1029 } while ((newRegions.contains(oldRegions.get(0)) || newRegions.contains(oldRegions.get(1)))
1030 || newRegions.size() != 4);
1031 tableExists = MetaTableAccessor.tableExists(regionServer.getConnection(),
1032 tableName);
1033 assertEquals("The specified table should be present.", true, tableExists);
1034
1035 byte[] b1 = "row1".getBytes();
1036 Get g = new Get(b1);
1037 g.setConsistency(Consistency.STRONG);
1038
1039
1040
1041 Result r = t.get(g);
1042 Assert.assertFalse(r.isStale());
1043 LOG.info("exists stale after flush done");
1044
1045 SlowMeCopro.getPrimaryCdl().set(new CountDownLatch(1));
1046 g = new Get(b1);
1047 g.setConsistency(Consistency.TIMELINE);
1048
1049 r = t.get(g);
1050 Assert.assertTrue(r.isStale());
1051 SlowMeCopro.getPrimaryCdl().get().countDown();
1052 } finally {
1053 SlowMeCopro.getPrimaryCdl().get().countDown();
1054 admin.setBalancerRunning(true, false);
1055 cluster.getMaster().setCatalogJanitorEnabled(true);
1056 t.close();
1057 }
1058 }
1059
1060 private void insertData(final TableName tableName, HBaseAdmin admin, Table t) throws IOException,
1061 InterruptedException {
1062 Put p = new Put(Bytes.toBytes("row1"));
1063 p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("1"));
1064 t.put(p);
1065 p = new Put(Bytes.toBytes("row2"));
1066 p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("2"));
1067 t.put(p);
1068 p = new Put(Bytes.toBytes("row3"));
1069 p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("3"));
1070 t.put(p);
1071 p = new Put(Bytes.toBytes("row4"));
1072 p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("4"));
1073 t.put(p);
1074 admin.flush(tableName);
1075 }
1076
1077
1078
1079
1080
1081 @Test(timeout = 60000)
1082 public void testSplitRegionWithNoStoreFiles()
1083 throws Exception {
1084 final TableName tableName =
1085 TableName.valueOf("testSplitRegionWithNoStoreFiles");
1086
1087 createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
1088 List<HRegion> regions = cluster.getRegions(tableName);
1089 HRegionInfo hri = getAndCheckSingleTableRegion(regions);
1090 ensureTableRegionNotOnSameServerAsMeta(admin, hri);
1091 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionInfo()
1092 .getRegionName());
1093 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
1094
1095 this.admin.setBalancerRunning(false, true);
1096
1097 cluster.getMaster().setCatalogJanitorEnabled(false);
1098 try {
1099
1100 printOutRegions(regionServer, "Initial regions: ");
1101 Configuration conf = cluster.getConfiguration();
1102 HBaseFsck.debugLsr(conf, new Path("/"));
1103 Path rootDir = FSUtils.getRootDir(conf);
1104 FileSystem fs = TESTING_UTIL.getDFSCluster().getFileSystem();
1105 Map<String, Path> storefiles =
1106 FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
1107 assertEquals("Expected nothing but found " + storefiles.toString(), storefiles.size(), 0);
1108
1109
1110 regions = cluster.getRegions(tableName);
1111 final HRegion region = findSplittableRegion(regions);
1112 assertTrue("not able to find a splittable region", region != null);
1113
1114
1115 SplitTransactionImpl st = new MockedSplitTransaction(region, Bytes.toBytes("row2"));
1116 try {
1117 st.prepare();
1118 st.execute(regionServer, regionServer);
1119 } catch (IOException e) {
1120 fail("Split execution should have succeeded with no exceptions thrown");
1121 }
1122
1123
1124
1125 List<HRegion> daughters = cluster.getRegions(tableName);
1126 assertTrue(daughters.size() == 2);
1127
1128
1129 HBaseFsck.debugLsr(conf, new Path("/"));
1130 Map<String, Path> storefilesAfter =
1131 FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
1132 assertEquals("Expected nothing but found " + storefilesAfter.toString(),
1133 storefilesAfter.size(), 0);
1134
1135 hri = region.getRegionInfo();
1136 AssignmentManager am = cluster.getMaster().getAssignmentManager();
1137 RegionStates regionStates = am.getRegionStates();
1138 long start = EnvironmentEdgeManager.currentTime();
1139 while (!regionStates.isRegionInState(hri, State.SPLIT)) {
1140 assertFalse("Timed out in waiting split parent to be in state SPLIT",
1141 EnvironmentEdgeManager.currentTime() - start > 60000);
1142 Thread.sleep(500);
1143 }
1144
1145
1146 am.assign(hri, true, true);
1147 assertFalse("Split region can't be assigned",
1148 regionStates.isRegionInTransition(hri));
1149 assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
1150
1151
1152 am.unassign(hri, true, null);
1153 assertFalse("Split region can't be unassigned",
1154 regionStates.isRegionInTransition(hri));
1155 assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
1156 } finally {
1157 admin.setBalancerRunning(true, false);
1158 cluster.getMaster().setCatalogJanitorEnabled(true);
1159 }
1160 }
1161
1162 @Test(timeout = 180000)
1163 public void testSplitHooksBeforeAndAfterPONR() throws Exception {
1164 TableName firstTable = TableName.valueOf("testSplitHooksBeforeAndAfterPONR_1");
1165 TableName secondTable = TableName.valueOf("testSplitHooksBeforeAndAfterPONR_2");
1166 HColumnDescriptor hcd = new HColumnDescriptor("cf");
1167
1168 HTableDescriptor desc = new HTableDescriptor(firstTable);
1169 desc.addCoprocessor(MockedRegionObserver.class.getName());
1170 desc.addFamily(hcd);
1171 admin.createTable(desc);
1172 TESTING_UTIL.waitUntilAllRegionsAssigned(firstTable);
1173
1174 desc = new HTableDescriptor(secondTable);
1175 desc.addFamily(hcd);
1176 admin.createTable(desc);
1177 TESTING_UTIL.waitUntilAllRegionsAssigned(secondTable);
1178
1179 List<HRegion> firstTableRegions = cluster.getRegions(firstTable);
1180 List<HRegion> secondTableRegions = cluster.getRegions(secondTable);
1181
1182
1183 if (firstTableRegions.size() == 0 || secondTableRegions.size() == 0) {
1184 fail("Each table should have at least one region.");
1185 }
1186 ServerName serverName = cluster.getServerHoldingRegion(firstTable,
1187 firstTableRegions.get(0).getRegionInfo().getRegionName());
1188 admin.move(secondTableRegions.get(0).getRegionInfo().getEncodedNameAsBytes(),
1189 Bytes.toBytes(serverName.getServerName()));
1190 Table table1 = null;
1191 Table table2 = null;
1192 try {
1193 table1 = new HTable(TESTING_UTIL.getConfiguration(), firstTable);
1194 table2 = new HTable(TESTING_UTIL.getConfiguration(), firstTable);
1195 insertData(firstTable, admin, table1);
1196 insertData(secondTable, admin, table2);
1197 admin.split(firstTable, "row2".getBytes());
1198 firstTableRegions = cluster.getRegions(firstTable);
1199 while (firstTableRegions.size() != 2) {
1200 Thread.sleep(1000);
1201 firstTableRegions = cluster.getRegions(firstTable);
1202 }
1203 assertEquals("Number of regions after split should be 2.", 2, firstTableRegions.size());
1204 secondTableRegions = cluster.getRegions(secondTable);
1205 assertEquals("Number of regions after split should be 2.", 2, secondTableRegions.size());
1206 } finally {
1207 if (table1 != null) {
1208 table1.close();
1209 }
1210 if (table2 != null) {
1211 table2.close();
1212 }
1213 TESTING_UTIL.deleteTable(firstTable);
1214 TESTING_UTIL.deleteTable(secondTable);
1215 }
1216 }
1217
1218 private void testSplitBeforeSettingSplittingInZKInternals() throws Exception {
1219 final TableName tableName = TableName.valueOf("testSplitBeforeSettingSplittingInZK");
1220 try {
1221
1222 createTableAndWait(tableName, Bytes.toBytes("cf"));
1223
1224 List<HRegion> regions = awaitTableRegions(tableName);
1225 assertTrue("Table not online", cluster.getRegions(tableName).size() != 0);
1226
1227 int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionInfo()
1228 .getRegionName());
1229 HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
1230 final HRegion region = findSplittableRegion(regions);
1231 assertTrue("not able to find a splittable region", region != null);
1232 SplitTransactionImpl st = new MockedSplitTransaction(region, Bytes.toBytes("row2")) {
1233 @Override
1234 public PairOfSameType<Region> stepsBeforePONR(final Server server,
1235 final RegionServerServices services, boolean testing) throws IOException {
1236 throw new SplittingNodeCreationFailedException ();
1237 }
1238 };
1239 String node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
1240 region.getRegionInfo().getEncodedName());
1241 regionServer.getZooKeeper().sync(node);
1242 for (int i = 0; i < 100; i++) {
1243
1244
1245
1246 if (ZKUtil.checkExists(regionServer.getZooKeeper(), node) != -1) {
1247 Thread.sleep(100);
1248 }
1249 }
1250 try {
1251 st.prepare();
1252 st.execute(regionServer, regionServer);
1253 } catch (IOException e) {
1254
1255
1256
1257 assertTrue("Should be instance of CreateSplittingNodeFailedException",
1258 e instanceof SplittingNodeCreationFailedException );
1259 node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
1260 region.getRegionInfo().getEncodedName());
1261 {
1262 assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
1263 }
1264 assertTrue(st.rollback(regionServer, regionServer));
1265 assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
1266 }
1267 } finally {
1268 TESTING_UTIL.deleteTable(tableName);
1269 }
1270 }
1271
1272 @Test
1273 public void testStoreFileReferenceCreationWhenSplitPolicySaysToSkipRangeCheck()
1274 throws Exception {
1275 final TableName tableName =
1276 TableName.valueOf("testStoreFileReferenceCreationWhenSplitPolicySaysToSkipRangeCheck");
1277 try {
1278 HTableDescriptor htd = new HTableDescriptor(tableName);
1279 htd.addFamily(new HColumnDescriptor("f"));
1280 htd.addFamily(new HColumnDescriptor("i_f"));
1281 htd.setRegionSplitPolicyClassName(CustomSplitPolicy.class.getName());
1282 admin.createTable(htd);
1283 List<HRegion> regions = awaitTableRegions(tableName);
1284 HRegion region = regions.get(0);
1285 for(int i = 3;i<9;i++) {
1286 Put p = new Put(Bytes.toBytes("row"+i));
1287 p.add(Bytes.toBytes("f"), Bytes.toBytes("q"), Bytes.toBytes("value"+i));
1288 p.add(Bytes.toBytes("i_f"), Bytes.toBytes("q"), Bytes.toBytes("value"+i));
1289 region.put(p);
1290 }
1291 region.flush(true);
1292 Store store = region.getStore(Bytes.toBytes("f"));
1293 Collection<StoreFile> storefiles = store.getStorefiles();
1294 assertEquals(storefiles.size(), 1);
1295 assertFalse(region.hasReferences());
1296 Path referencePath =
1297 region.getRegionFileSystem().splitStoreFile(region.getRegionInfo(), "f",
1298 storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy());
1299 assertNull(referencePath);
1300 referencePath =
1301 region.getRegionFileSystem().splitStoreFile(region.getRegionInfo(), "i_f",
1302 storefiles.iterator().next(), Bytes.toBytes("row1"), false, region.getSplitPolicy());
1303 assertNotNull(referencePath);
1304 } finally {
1305 TESTING_UTIL.deleteTable(tableName);
1306 }
1307 }
1308
1309 @Test(timeout = 120000)
1310 public void testFailedSplit() throws Exception {
1311 TableName tableName = TableName.valueOf("testFailedSplit");
1312 byte[] colFamily = Bytes.toBytes("info");
1313 TESTING_UTIL.createTable(tableName, colFamily);
1314 Connection connection = ConnectionFactory.createConnection(TESTING_UTIL.getConfiguration());
1315 HTable table = (HTable) connection.getTable(tableName);
1316 try {
1317 TESTING_UTIL.loadTable(table, colFamily);
1318 List<HRegionInfo> regions = TESTING_UTIL.getHBaseAdmin().getTableRegions(tableName);
1319 assertTrue(regions.size() == 1);
1320 final HRegion actualRegion = cluster.getRegions(tableName).get(0);
1321 actualRegion.getCoprocessorHost().load(FailingSplitRegionObserver.class,
1322 Coprocessor.PRIORITY_USER, actualRegion.getBaseConf());
1323
1324
1325 admin.split(tableName);
1326 FailingSplitRegionObserver observer = (FailingSplitRegionObserver) actualRegion
1327 .getCoprocessorHost().findCoprocessor(FailingSplitRegionObserver.class.getName());
1328 assertNotNull(observer);
1329 observer.latch.await();
1330 observer.postSplit.await();
1331 LOG.info("Waiting for region to come out of RIT: " + actualRegion);
1332 TESTING_UTIL.waitFor(60000, 1000, new Waiter.Predicate<Exception>() {
1333 @Override
1334 public boolean evaluate() throws Exception {
1335 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
1336 return !regionStates.isRegionsInTransition();
1337 }
1338 });
1339 regions = TESTING_UTIL.getHBaseAdmin().getTableRegions(tableName);
1340 assertTrue(regions.size() == 1);
1341 RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
1342 Set<RegionState> rit = regionStates.getRegionsInTransition();
1343 assertTrue(rit.size() == 0);
1344 } finally {
1345 table.close();
1346 connection.close();
1347 TESTING_UTIL.deleteTable(tableName);
1348 }
1349 }
1350
1351 @Test (timeout=300000)
1352 public void testSSHCleanupDaugtherRegionsOfAbortedSplit() throws Exception {
1353 TableName table = TableName.valueOf("testSSHCleanupDaugtherRegionsOfAbortedSplit");
1354 try {
1355 HTableDescriptor desc = new HTableDescriptor(table);
1356 desc.addFamily(new HColumnDescriptor(Bytes.toBytes("f")));
1357 admin.createTable(desc);
1358 HTable hTable = new HTable(cluster.getConfiguration(), desc.getTableName());
1359 for(int i = 1; i < 5; i++) {
1360 Put p1 = new Put(("r"+i).getBytes());
1361 p1.add(Bytes.toBytes("f"), "q1".getBytes(), "v".getBytes());
1362 hTable.put(p1);
1363 }
1364 admin.flush(desc.getTableName());
1365 List<HRegion> regions = cluster.getRegions(desc.getTableName());
1366 int serverWith = cluster.getServerWith(regions.get(0).getRegionInfo().getRegionName());
1367 HRegionServer regionServer = cluster.getRegionServer(serverWith);
1368 cluster.getServerWith(regions.get(0).getRegionInfo().getRegionName());
1369 SplitTransactionImpl st = new SplitTransactionImpl(regions.get(0), Bytes.toBytes("r3"));
1370 st.prepare();
1371 st.stepsBeforePONR(regionServer, regionServer, false);
1372 Path tableDir =
1373 FSUtils.getTableDir(cluster.getMaster().getMasterFileSystem().getRootDir(),
1374 desc.getTableName());
1375 tableDir.getFileSystem(cluster.getConfiguration());
1376 List<Path> regionDirs =
1377 FSUtils.getRegionDirs(tableDir.getFileSystem(cluster.getConfiguration()), tableDir);
1378 assertEquals(3,regionDirs.size());
1379 cluster.startRegionServer();
1380 regionServer.kill();
1381 cluster.getRegionServerThreads().get(serverWith).join();
1382
1383 TESTING_UTIL.waitFor(60000, 1000, new Waiter.Predicate<Exception>() {
1384 @Override
1385 public boolean evaluate() throws Exception {
1386 return !cluster.getMaster().getServerManager().areDeadServersInProgress();
1387 }
1388 });
1389
1390 TESTING_UTIL.waitFor(60000, 1000, new Waiter.Predicate<Exception>() {
1391 @Override
1392 public boolean evaluate() throws Exception {
1393 return !cluster.getMaster().getAssignmentManager().
1394 getRegionStates().isRegionsInTransition();
1395 }
1396 });
1397 regionDirs =
1398 FSUtils.getRegionDirs(tableDir.getFileSystem(cluster.getConfiguration()), tableDir);
1399 assertEquals(1,regionDirs.size());
1400 } finally {
1401 TESTING_UTIL.deleteTable(table);
1402 }
1403 }
1404
1405 public static class MockedCoordinatedStateManager extends ZkCoordinatedStateManager {
1406
1407 public void initialize(Server server, HRegion region) {
1408 this.server = server;
1409 this.watcher = server.getZooKeeper();
1410 splitTransactionCoordination = new MockedSplitTransactionCoordination(this, watcher, region);
1411 closeRegionCoordination = new ZkCloseRegionCoordination(this, watcher);
1412 openRegionCoordination = new ZkOpenRegionCoordination(this, watcher);
1413 }
1414 }
1415
1416 public static class MockedSplitTransaction extends SplitTransactionImpl {
1417
1418 private HRegion currentRegion;
1419 public MockedSplitTransaction(HRegion region, byte[] splitrow) {
1420 super(region, splitrow);
1421 this.currentRegion = region;
1422 }
1423 @Override
1424 public boolean rollback(Server server, RegionServerServices services) throws IOException {
1425 if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
1426 .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
1427 if(secondSplit){
1428 super.rollback(server, services);
1429 latch.countDown();
1430 return true;
1431 }
1432 }
1433 return super.rollback(server, services);
1434 }
1435
1436
1437 }
1438
1439 public static class MockedSplitTransactionCoordination extends ZKSplitTransactionCoordination {
1440
1441 private HRegion currentRegion;
1442
1443 public MockedSplitTransactionCoordination(CoordinatedStateManager coordinationProvider,
1444 ZooKeeperWatcher watcher, HRegion region) {
1445 super(coordinationProvider, watcher);
1446 currentRegion = region;
1447 }
1448
1449 @Override
1450 public void completeSplitTransaction(RegionServerServices services, Region a, Region b,
1451 SplitTransactionDetails std, Region parent) throws IOException {
1452 if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
1453 .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
1454 try {
1455 if (!secondSplit){
1456 callRollBack = true;
1457 latch.await();
1458 }
1459 } catch (InterruptedException e) {
1460 }
1461
1462 }
1463 super.completeSplitTransaction(services, a, b, std, parent);
1464 if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
1465 .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
1466 firstSplitCompleted = true;
1467 }
1468 }
1469 }
1470
1471 private HRegion findSplittableRegion(final List<HRegion> regions) throws InterruptedException {
1472 for (int i = 0; i < 5; ++i) {
1473 for (HRegion r: regions) {
1474 if (r.isSplittable() && r.getRegionInfo().getReplicaId() == 0) {
1475 return(r);
1476 }
1477 }
1478 Thread.sleep(100);
1479 }
1480 return(null);
1481 }
1482
1483 private List<HRegion> checkAndGetDaughters(TableName tableName)
1484 throws InterruptedException {
1485 List<HRegion> daughters = null;
1486
1487 for (int i=0; i<100; i++) {
1488 daughters = cluster.getRegions(tableName);
1489 if (daughters.size() >= 2) break;
1490 Thread.sleep(100);
1491 }
1492 assertTrue(daughters.size() >= 2);
1493 return daughters;
1494 }
1495
1496 private MockMasterWithoutCatalogJanitor abortAndWaitForMaster()
1497 throws IOException, InterruptedException {
1498 cluster.abortMaster(0);
1499 cluster.waitOnMaster(0);
1500 cluster.getConfiguration().setClass(HConstants.MASTER_IMPL,
1501 MockMasterWithoutCatalogJanitor.class, HMaster.class);
1502 MockMasterWithoutCatalogJanitor master = null;
1503 master = (MockMasterWithoutCatalogJanitor) cluster.startMaster().getMaster();
1504 cluster.waitForActiveAndReadyMaster();
1505 return master;
1506 }
1507
1508 private void split(final HRegionInfo hri, final HRegionServer server, final int regionCount)
1509 throws IOException, InterruptedException {
1510 this.admin.split(hri.getRegionNameAsString());
1511 try {
1512 for (int i = 0; ProtobufUtil.getOnlineRegions(
1513 server.getRSRpcServices()).size() <= regionCount && i < 300; i++) {
1514 LOG.debug("Waiting on region to split");
1515 Thread.sleep(100);
1516 }
1517
1518 assertFalse("Waited too long for split",
1519 ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size() <= regionCount);
1520 } catch (RegionServerStoppedException e) {
1521 if (useZKForAssignment) {
1522
1523 LOG.error(e);
1524 throw e;
1525 }
1526 }
1527 }
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540 private int ensureTableRegionNotOnSameServerAsMeta(final Admin admin,
1541 final HRegionInfo hri)
1542 throws IOException, MasterNotRunningException,
1543 ZooKeeperConnectionException, InterruptedException {
1544
1545
1546
1547 int metaServerIndex = cluster.getServerWithMeta();
1548 assertTrue(metaServerIndex != -1);
1549 HRegionServer metaRegionServer = cluster.getRegionServer(metaServerIndex);
1550 int tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1551 assertTrue(tableRegionIndex != -1);
1552 HRegionServer tableRegionServer = cluster.getRegionServer(tableRegionIndex);
1553 if (metaRegionServer.getServerName().equals(tableRegionServer.getServerName())) {
1554 HRegionServer hrs = getOtherRegionServer(cluster, metaRegionServer);
1555 assertNotNull(hrs);
1556 assertNotNull(hri);
1557 LOG.info("Moving " + hri.getRegionNameAsString() + " from " +
1558 metaRegionServer.getServerName() + " to " +
1559 hrs.getServerName() + "; metaServerIndex=" + metaServerIndex);
1560 admin.move(hri.getEncodedNameAsBytes(), Bytes.toBytes(hrs.getServerName().toString()));
1561 }
1562
1563 for (int i = 0; i < 20; i++) {
1564 tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1565 if (tableRegionIndex != -1 && tableRegionIndex != metaServerIndex) break;
1566 LOG.debug("Waiting on region move off the hbase:meta server; current index " +
1567 tableRegionIndex + " and metaServerIndex=" + metaServerIndex);
1568 Thread.sleep(1000);
1569 }
1570 assertTrue("Region not moved off hbase:meta server", tableRegionIndex != -1
1571 && tableRegionIndex != metaServerIndex);
1572
1573 tableRegionIndex = cluster.getServerWith(hri.getRegionName());
1574 assertTrue(tableRegionIndex != -1);
1575 assertNotSame(metaServerIndex, tableRegionIndex);
1576 return tableRegionIndex;
1577 }
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588 private HRegionServer getOtherRegionServer(final MiniHBaseCluster cluster,
1589 final HRegionServer notThisOne) {
1590 for (RegionServerThread rst: cluster.getRegionServerThreads()) {
1591 HRegionServer hrs = rst.getRegionServer();
1592 if (hrs.getServerName().equals(notThisOne.getServerName())) continue;
1593 if (hrs.isStopping() || hrs.isStopped()) continue;
1594 return hrs;
1595 }
1596 return null;
1597 }
1598
1599 private void printOutRegions(final HRegionServer hrs, final String prefix)
1600 throws IOException {
1601 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
1602 for (HRegionInfo region: regions) {
1603 LOG.info(prefix + region.getRegionNameAsString());
1604 }
1605 }
1606
1607 private void waitUntilRegionServerDead() throws InterruptedException, IOException {
1608
1609 for (int i=0; cluster.getMaster().getClusterStatus().
1610 getServers().size() > NB_SERVERS && i<100; i++) {
1611 LOG.info("Waiting on server to go down");
1612 Thread.sleep(100);
1613 }
1614 assertFalse("Waited too long for RS to die", cluster.getMaster().getClusterStatus().
1615 getServers().size() > NB_SERVERS);
1616 }
1617
1618 private void awaitDaughters(TableName tableName, int numDaughters) throws InterruptedException {
1619
1620 for (int i=0; cluster.getRegions(tableName).size() < numDaughters && i<60; i++) {
1621 LOG.info("Waiting for repair to happen");
1622 Thread.sleep(1000);
1623 }
1624 if (cluster.getRegions(tableName).size() < numDaughters) {
1625 fail("Waiting too long for daughter regions");
1626 }
1627 }
1628
1629 private List<HRegion> awaitTableRegions(final TableName tableName) throws InterruptedException {
1630 List<HRegion> regions = null;
1631 for (int i = 0; i < 100; i++) {
1632 regions = cluster.getRegions(tableName);
1633 if (regions.size() > 0) break;
1634 Thread.sleep(100);
1635 }
1636 return regions;
1637 }
1638
1639 private HTable createTableAndWait(TableName tableName, byte[] cf) throws IOException,
1640 InterruptedException {
1641 HTable t = TESTING_UTIL.createTable(tableName, cf);
1642 awaitTableRegions(tableName);
1643 assertTrue("Table not online: " + tableName,
1644 cluster.getRegions(tableName).size() != 0);
1645 return t;
1646 }
1647
1648 public static class MockMasterWithoutCatalogJanitor extends HMaster {
1649
1650 public MockMasterWithoutCatalogJanitor(Configuration conf, CoordinatedStateManager cp)
1651 throws IOException, KeeperException,
1652 InterruptedException {
1653 super(conf, cp);
1654 }
1655 }
1656
1657 private static class SplittingNodeCreationFailedException extends IOException {
1658 private static final long serialVersionUID = 1652404976265623004L;
1659
1660 public SplittingNodeCreationFailedException () {
1661 super();
1662 }
1663 }
1664
1665 public static class MockedRegionObserver extends BaseRegionObserver {
1666 private SplitTransactionImpl st = null;
1667 private PairOfSameType<Region> daughterRegions = null;
1668
1669 @Override
1670 public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
1671 byte[] splitKey, List<Mutation> metaEntries) throws IOException {
1672 RegionCoprocessorEnvironment environment = ctx.getEnvironment();
1673 HRegionServer rs = (HRegionServer) environment.getRegionServerServices();
1674 List<Region> onlineRegions =
1675 rs.getOnlineRegions(TableName.valueOf("testSplitHooksBeforeAndAfterPONR_2"));
1676 Region region = onlineRegions.get(0);
1677 for (Region r : onlineRegions) {
1678 if (r.getRegionInfo().containsRow(splitKey)) {
1679 region = r;
1680 break;
1681 }
1682 }
1683 st = new SplitTransactionImpl((HRegion) region, splitKey);
1684 if (!st.prepare()) {
1685 LOG.error("Prepare for the table " + region.getTableDesc().getNameAsString()
1686 + " failed. So returning null. ");
1687 ctx.bypass();
1688 return;
1689 }
1690 ((HRegion)region).forceSplit(splitKey);
1691 daughterRegions = st.stepsBeforePONR(rs, rs, false);
1692 HRegionInfo copyOfParent = new HRegionInfo(region.getRegionInfo());
1693 copyOfParent.setOffline(true);
1694 copyOfParent.setSplit(true);
1695
1696 Put putParent = MetaTableAccessor.makePutFromRegionInfo(copyOfParent);
1697 MetaTableAccessor.addDaughtersToPut(putParent, daughterRegions.getFirst().getRegionInfo(),
1698 daughterRegions.getSecond().getRegionInfo());
1699 metaEntries.add(putParent);
1700
1701 Put putA = MetaTableAccessor.makePutFromRegionInfo(
1702 daughterRegions.getFirst().getRegionInfo());
1703 Put putB = MetaTableAccessor.makePutFromRegionInfo(
1704 daughterRegions.getSecond().getRegionInfo());
1705 st.addLocation(putA, rs.getServerName(), 1);
1706 st.addLocation(putB, rs.getServerName(), 1);
1707 metaEntries.add(putA);
1708 metaEntries.add(putB);
1709 }
1710
1711 @Override
1712 public void preSplitAfterPONR(ObserverContext<RegionCoprocessorEnvironment> ctx)
1713 throws IOException {
1714 RegionCoprocessorEnvironment environment = ctx.getEnvironment();
1715 HRegionServer rs = (HRegionServer) environment.getRegionServerServices();
1716 st.stepsAfterPONR(rs, rs, daughterRegions, null);
1717 }
1718
1719 }
1720
1721 static class CustomSplitPolicy extends RegionSplitPolicy {
1722
1723 @Override
1724 protected boolean shouldSplit() {
1725 return true;
1726 }
1727
1728 @Override
1729 public boolean skipStoreFileRangeCheck(String familyName) {
1730 if(familyName.startsWith("i_")) {
1731 return true;
1732 } else {
1733 return false;
1734 }
1735 }
1736 }
1737 }
1738