1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertNotSame;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.util.ArrayList;
28 import java.util.HashMap;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.concurrent.atomic.AtomicBoolean;
32
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.hbase.CellScannable;
35 import org.apache.hadoop.hbase.CellUtil;
36 import org.apache.hadoop.hbase.CoordinatedStateException;
37 import org.apache.hadoop.hbase.CoordinatedStateManager;
38 import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
39 import org.apache.hadoop.hbase.DoNotRetryIOException;
40 import org.apache.hadoop.hbase.HBaseConfiguration;
41 import org.apache.hadoop.hbase.HBaseTestingUtility;
42 import org.apache.hadoop.hbase.HConstants;
43 import org.apache.hadoop.hbase.HRegionInfo;
44 import org.apache.hadoop.hbase.MetaMockingUtil;
45 import org.apache.hadoop.hbase.RegionException;
46 import org.apache.hadoop.hbase.RegionTransition;
47 import org.apache.hadoop.hbase.Server;
48 import org.apache.hadoop.hbase.ServerLoad;
49 import org.apache.hadoop.hbase.ServerName;
50 import org.apache.hadoop.hbase.TableName;
51 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
52 import org.apache.hadoop.hbase.client.ClusterConnection;
53 import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
54 import org.apache.hadoop.hbase.client.Result;
55 import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
56 import org.apache.hadoop.hbase.coordination.OpenRegionCoordination;
57 import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
58 import org.apache.hadoop.hbase.coordination.ZkOpenRegionCoordination;
59 import org.apache.hadoop.hbase.exceptions.DeserializationException;
60 import org.apache.hadoop.hbase.executor.EventType;
61 import org.apache.hadoop.hbase.executor.ExecutorService;
62 import org.apache.hadoop.hbase.executor.ExecutorType;
63 import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
64 import org.apache.hadoop.hbase.master.RegionState.State;
65 import org.apache.hadoop.hbase.master.TableLockManager.NullTableLockManager;
66 import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
67 import org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer;
68 import org.apache.hadoop.hbase.master.handler.EnableTableHandler;
69 import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
70 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
71 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
72 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest;
73 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetResponse;
74 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
75 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
76 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
77 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.Table;
78 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
79 import org.apache.hadoop.hbase.regionserver.RegionServerAbortedException;
80 import org.apache.hadoop.hbase.testclassification.MediumTests;
81 import org.apache.hadoop.hbase.util.Bytes;
82 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
83 import org.apache.hadoop.hbase.util.Threads;
84 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
85 import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper;
86 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
87 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
88 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
89 import org.apache.zookeeper.KeeperException;
90 import org.apache.zookeeper.KeeperException.NodeExistsException;
91 import org.apache.zookeeper.Watcher;
92 import org.junit.After;
93 import org.junit.AfterClass;
94 import org.junit.Before;
95 import org.junit.BeforeClass;
96 import org.junit.Test;
97 import org.junit.experimental.categories.Category;
98 import org.mockito.Mockito;
99 import org.mockito.internal.util.reflection.Whitebox;
100 import org.mockito.invocation.InvocationOnMock;
101 import org.mockito.stubbing.Answer;
102
103 import com.google.protobuf.RpcController;
104 import com.google.protobuf.ServiceException;
105
106
107
108
109
110 @Category(MediumTests.class)
111 public class TestAssignmentManager {
112 private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
113 private static final ServerName SERVERNAME_A =
114 ServerName.valueOf("example.org", 1234, 5678);
115 private static final ServerName SERVERNAME_AA =
116 ServerName.valueOf("example.org", 1234, 9999);
117 private static final ServerName SERVERNAME_B =
118 ServerName.valueOf("example.org", 0, 5678);
119 private static final HRegionInfo REGIONINFO =
120 new HRegionInfo(TableName.valueOf("t"),
121 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
122 private static int assignmentCount;
123 private static boolean enabling = false;
124
125
126 private Server server;
127 private ServerManager serverManager;
128 private ZooKeeperWatcher watcher;
129 private CoordinatedStateManager cp;
130 private MetaTableLocator mtl;
131 private LoadBalancer balancer;
132 private HMaster master;
133 private ClusterConnection connection;
134
135 @BeforeClass
136 public static void beforeClass() throws Exception {
137 HTU.getConfiguration().setBoolean("hbase.assignment.usezk", true);
138 HTU.startMiniZKCluster();
139 }
140
141 @AfterClass
142 public static void afterClass() throws IOException {
143 HTU.shutdownMiniZKCluster();
144 }
145
146 @Before
147 public void before() throws ZooKeeperConnectionException, IOException {
148
149
150
151
152
153
154 this.server = Mockito.mock(Server.class);
155 Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("master,1,1"));
156 Mockito.when(server.getConfiguration()).thenReturn(HTU.getConfiguration());
157 this.watcher =
158 new ZooKeeperWatcher(HTU.getConfiguration(), "mockedServer", this.server, true);
159 Mockito.when(server.getZooKeeper()).thenReturn(this.watcher);
160 Mockito.doThrow(new RuntimeException("Aborted")).
161 when(server).abort(Mockito.anyString(), (Throwable)Mockito.anyObject());
162
163 cp = new ZkCoordinatedStateManager();
164 cp.initialize(this.server);
165 cp.start();
166
167 mtl = Mockito.mock(MetaTableLocator.class);
168
169 Mockito.when(server.getCoordinatedStateManager()).thenReturn(cp);
170 Mockito.when(server.getMetaTableLocator()).thenReturn(mtl);
171
172
173 this.connection =
174 (ClusterConnection)HConnectionTestingUtility.getMockedConnection(HTU.getConfiguration());
175
176
177
178 Mockito.when(server.getConnection()).thenReturn(connection);
179 Mockito.when(connection.isManaged()).thenReturn(true);
180
181
182
183 this.serverManager = Mockito.mock(ServerManager.class);
184 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
185 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(true);
186 Mockito.when(this.serverManager.getDeadServers()).thenReturn(new DeadServer());
187 final Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
188 onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
189 onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
190 Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(
191 new ArrayList<ServerName>(onlineServers.keySet()));
192 Mockito.when(this.serverManager.getOnlineServers()).thenReturn(onlineServers);
193
194 List<ServerName> avServers = new ArrayList<ServerName>();
195 avServers.addAll(onlineServers.keySet());
196 Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(avServers);
197 Mockito.when(this.serverManager.createDestinationServersList(null)).thenReturn(avServers);
198
199 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, REGIONINFO, -1)).
200 thenReturn(true);
201 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_B, REGIONINFO, -1)).
202 thenReturn(true);
203
204 Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_A, REGIONINFO, -1, null)).
205 thenReturn(RegionOpeningState.OPENED);
206 Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_B, REGIONINFO, -1, null)).
207 thenReturn(RegionOpeningState.OPENED);
208 this.master = Mockito.mock(HMaster.class);
209
210 Mockito.when(this.master.getServerManager()).thenReturn(serverManager);
211 }
212
213 @After public void after() throws KeeperException, IOException {
214 if (this.watcher != null) {
215
216 ZKAssign.deleteAllNodes(this.watcher);
217 this.watcher.close();
218 this.cp.stop();
219 }
220 if (this.connection != null) this.connection.close();
221 }
222
223
224
225
226
227
228
229
230
231 @Test(timeout = 60000)
232 public void testBalanceOnMasterFailoverScenarioWithOpenedNode()
233 throws IOException, KeeperException, InterruptedException, ServiceException,
234 DeserializationException, CoordinatedStateException {
235 AssignmentManagerWithExtrasForTesting am =
236 setUpMockedAssignmentManager(this.server, this.serverManager);
237 try {
238 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
239 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
240 while (!am.processRITInvoked) Thread.sleep(1);
241
242
243
244 am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_B));
245
246 Mocking.waitForRegionFailedToCloseAndSetToPendingClose(am, REGIONINFO);
247
248
249
250
251
252
253 int versionid =
254 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
255 assertNotSame(versionid, -1);
256 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
257
258
259
260 versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
261 assertNotSame(-1, versionid);
262
263 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
264 SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
265 EventType.RS_ZK_REGION_OPENING, versionid);
266 assertNotSame(-1, versionid);
267
268 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
269 SERVERNAME_B, versionid);
270 assertNotSame(-1, versionid);
271 am.gate.set(false);
272
273 ZKAssign.blockUntilNoRIT(watcher);
274 } finally {
275 am.getExecutorService().shutdown();
276 am.shutdown();
277 }
278 }
279
280 @Test(timeout = 60000)
281 public void testBalanceOnMasterFailoverScenarioWithClosedNode()
282 throws IOException, KeeperException, InterruptedException, ServiceException,
283 DeserializationException, CoordinatedStateException {
284 AssignmentManagerWithExtrasForTesting am =
285 setUpMockedAssignmentManager(this.server, this.serverManager);
286 try {
287 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
288 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
289 while (!am.processRITInvoked) Thread.sleep(1);
290
291
292
293 am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_B));
294
295 Mocking.waitForRegionFailedToCloseAndSetToPendingClose(am, REGIONINFO);
296
297
298
299
300
301
302 int versionid =
303 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
304 assertNotSame(versionid, -1);
305 am.gate.set(false);
306 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
307
308
309
310 versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
311 assertNotSame(-1, versionid);
312
313 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
314 SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
315 EventType.RS_ZK_REGION_OPENING, versionid);
316 assertNotSame(-1, versionid);
317
318 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
319 SERVERNAME_B, versionid);
320 assertNotSame(-1, versionid);
321
322
323 ZKAssign.blockUntilNoRIT(watcher);
324 } finally {
325 am.getExecutorService().shutdown();
326 am.shutdown();
327 }
328 }
329
330 @Test(timeout = 60000)
331 public void testBalanceOnMasterFailoverScenarioWithOfflineNode()
332 throws IOException, KeeperException, InterruptedException, ServiceException,
333 DeserializationException, CoordinatedStateException {
334 AssignmentManagerWithExtrasForTesting am =
335 setUpMockedAssignmentManager(this.server, this.serverManager);
336 try {
337 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
338 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
339 while (!am.processRITInvoked) Thread.sleep(1);
340
341
342
343 am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_B));
344
345 Mocking.waitForRegionFailedToCloseAndSetToPendingClose(am, REGIONINFO);
346
347
348
349
350
351
352 int versionid =
353 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
354 assertNotSame(versionid, -1);
355 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
356
357 am.gate.set(false);
358
359
360 versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
361 assertNotSame(-1, versionid);
362
363 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
364 SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
365 EventType.RS_ZK_REGION_OPENING, versionid);
366 assertNotSame(-1, versionid);
367
368 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
369 SERVERNAME_B, versionid);
370 assertNotSame(-1, versionid);
371
372 ZKAssign.blockUntilNoRIT(watcher);
373 } finally {
374 am.getExecutorService().shutdown();
375 am.shutdown();
376 }
377 }
378
379 private void createRegionPlanAndBalance(
380 final AssignmentManager am, final ServerName from,
381 final ServerName to, final HRegionInfo hri) throws RegionException {
382
383
384 am.regionOnline(hri, from);
385
386
387 am.balance(new RegionPlan(hri, from, to));
388 }
389
390
391
392
393
394
395
396
397 @Test (timeout=180000)
398 public void testBalance() throws IOException, KeeperException, DeserializationException,
399 InterruptedException, CoordinatedStateException {
400
401
402 ExecutorService executor = startupMasterExecutor("testBalanceExecutor");
403
404
405 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
406 .getConfiguration());
407
408 AssignmentManager am = new AssignmentManager(this.server,
409 this.serverManager, balancer, executor, null, master.getTableLockManager());
410 am.failoverCleanupDone.set(true);
411 try {
412
413
414 this.watcher.registerListenerFirst(am);
415
416
417 am.regionOnline(REGIONINFO, SERVERNAME_A);
418
419 RegionPlan plan = new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B);
420 am.balance(plan);
421
422 RegionStates regionStates = am.getRegionStates();
423
424 assertTrue(regionStates.isRegionInTransition(REGIONINFO)
425 && regionStates.isRegionInState(REGIONINFO, State.FAILED_CLOSE));
426
427 regionStates.updateRegionState(REGIONINFO, State.PENDING_CLOSE);
428
429
430
431
432
433
434 int versionid =
435 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
436 assertNotSame(versionid, -1);
437
438
439
440
441
442 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
443
444
445 versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
446 assertNotSame(-1, versionid);
447
448 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
449 SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
450 EventType.RS_ZK_REGION_OPENING, versionid);
451 assertNotSame(-1, versionid);
452
453 versionid =
454 ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO, SERVERNAME_B, versionid);
455 assertNotSame(-1, versionid);
456
457 while(regionStates.isRegionInTransition(REGIONINFO)) Threads.sleep(1);
458 } finally {
459 executor.shutdown();
460 am.shutdown();
461
462 ZKAssign.deleteAllNodes(this.watcher);
463 }
464 }
465
466
467
468
469
470
471 @Test (timeout=180000)
472 public void testShutdownHandler()
473 throws KeeperException, IOException, CoordinatedStateException, ServiceException {
474
475
476 ExecutorService executor = startupMasterExecutor("testShutdownHandler");
477
478
479 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
480 this.server, this.serverManager);
481 try {
482 processServerShutdownHandler(am, false);
483 } finally {
484 executor.shutdown();
485 am.shutdown();
486
487 ZKAssign.deleteAllNodes(this.watcher);
488 }
489 }
490
491
492
493
494
495
496 @Test (timeout=180000)
497 public void testShutdownHandlerWithRestartedServer()
498 throws KeeperException, IOException, CoordinatedStateException, ServiceException {
499
500
501 ExecutorService executor = startupMasterExecutor("testShutdownHandlerWithRestartedServer");
502
503
504 AssignmentManagerWithExtrasForTesting am =
505 setUpMockedAssignmentManager(this.server, this.serverManager);
506 am.getRegionStates().regionOnline(REGIONINFO, SERVERNAME_A);
507 am.getTableStateManager().setTableState(REGIONINFO.getTable(), Table.State.ENABLED);
508 try {
509 processServerShutdownHandler(am, false, true);
510 } finally {
511 executor.shutdown();
512 am.shutdown();
513
514 ZKAssign.deleteAllNodes(this.watcher);
515 }
516 }
517
518
519
520
521
522
523
524
525
526
527 @Test (timeout=180000)
528 public void testSSHWhenDisableTableInProgress() throws KeeperException, IOException,
529 CoordinatedStateException, ServiceException {
530 testCaseWithPartiallyDisabledState(Table.State.DISABLING);
531 testCaseWithPartiallyDisabledState(Table.State.DISABLED);
532 }
533
534
535
536
537
538
539
540
541
542 @Test (timeout=180000)
543 public void testSSHWhenSplitRegionInProgress() throws KeeperException, IOException, Exception {
544
545 testCaseWithSplitRegionPartial(true);
546
547 testCaseWithSplitRegionPartial(false);
548 }
549
550 private void testCaseWithSplitRegionPartial(boolean regionSplitDone) throws KeeperException,
551 IOException, InterruptedException,
552 CoordinatedStateException, ServiceException {
553
554
555 ExecutorService executor = startupMasterExecutor("testSSHWhenSplitRegionInProgress");
556
557 ZKAssign.deleteAllNodes(this.watcher);
558
559
560 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
561 this.server, this.serverManager);
562
563 am.regionOnline(REGIONINFO, SERVERNAME_A);
564
565 am.getRegionStates().updateRegionState(
566 REGIONINFO, State.SPLITTING, SERVERNAME_A);
567 am.getTableStateManager().setTableState(REGIONINFO.getTable(),
568 Table.State.ENABLED);
569 RegionTransition data = RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_SPLITTING,
570 REGIONINFO.getRegionName(), SERVERNAME_A);
571 String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
572
573 ZKUtil.createAndWatch(this.watcher, node, data.toByteArray());
574
575 try {
576 processServerShutdownHandler(am, regionSplitDone);
577
578
579
580 if (regionSplitDone) {
581 assertFalse("Region state of region in SPLITTING should be removed from rit.",
582 am.getRegionStates().isRegionsInTransition());
583 } else {
584 while (!am.assignInvoked) {
585 Thread.sleep(1);
586 }
587 assertTrue("Assign should be invoked.", am.assignInvoked);
588 }
589 } finally {
590 REGIONINFO.setOffline(false);
591 REGIONINFO.setSplit(false);
592 executor.shutdown();
593 am.shutdown();
594
595 ZKAssign.deleteAllNodes(this.watcher);
596 }
597 }
598
599 private void testCaseWithPartiallyDisabledState(Table.State state) throws KeeperException,
600 IOException, CoordinatedStateException, ServiceException {
601
602
603 ExecutorService executor = startupMasterExecutor("testSSHWhenDisableTableInProgress");
604 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
605 ZKAssign.deleteAllNodes(this.watcher);
606
607
608 AssignmentManager am = new AssignmentManager(this.server,
609 this.serverManager, balancer, executor, null, master.getTableLockManager());
610
611 am.regionOnline(REGIONINFO, SERVERNAME_A);
612
613 am.getRegionStates().updateRegionState(REGIONINFO, State.PENDING_CLOSE);
614 if (state == Table.State.DISABLING) {
615 am.getTableStateManager().setTableState(REGIONINFO.getTable(),
616 Table.State.DISABLING);
617 } else {
618 am.getTableStateManager().setTableState(REGIONINFO.getTable(),
619 Table.State.DISABLED);
620 }
621 RegionTransition data = RegionTransition.createRegionTransition(EventType.M_ZK_REGION_CLOSING,
622 REGIONINFO.getRegionName(), SERVERNAME_A);
623
624
625
626 String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
627
628 ZKUtil.createAndWatch(this.watcher, node, data.toByteArray());
629
630 try {
631 processServerShutdownHandler(am, false);
632
633
634 assertTrue("The znode should be deleted.", ZKUtil.checkExists(this.watcher, node) == -1);
635
636
637
638 if (state == Table.State.DISABLED) {
639 assertFalse("Region state of region in pending close should be removed from rit.",
640 am.getRegionStates().isRegionsInTransition());
641 }
642 } finally {
643 am.setEnabledTable(REGIONINFO.getTable());
644 executor.shutdown();
645 am.shutdown();
646
647 ZKAssign.deleteAllNodes(this.watcher);
648 }
649 }
650
651 private void processServerShutdownHandler(AssignmentManager am, boolean splitRegion)
652 throws IOException, ServiceException {
653 processServerShutdownHandler(am, splitRegion, false);
654 }
655
656 private void processServerShutdownHandler(
657 AssignmentManager am, boolean splitRegion, boolean deadserverRestarted)
658 throws IOException, ServiceException {
659
660
661 this.watcher.registerListenerFirst(am);
662
663
664
665 ClientProtos.ClientService.BlockingInterface implementation =
666 Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
667
668
669 Result r;
670 if (splitRegion) {
671 r = MetaMockingUtil.getMetaTableRowResultAsSplitRegion(REGIONINFO, SERVERNAME_A);
672 } else {
673 r = MetaMockingUtil.getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
674 }
675
676 final ScanResponse.Builder builder = ScanResponse.newBuilder();
677 builder.setMoreResults(true);
678 builder.addCellsPerResult(r.size());
679 final List<CellScannable> cellScannables = new ArrayList<CellScannable>(1);
680 cellScannables.add(r);
681 Mockito.when(implementation.scan(
682 (RpcController)Mockito.any(), (ScanRequest)Mockito.any())).
683 thenAnswer(new Answer<ScanResponse>() {
684 @Override
685 public ScanResponse answer(InvocationOnMock invocation) throws Throwable {
686 PayloadCarryingRpcController controller = (PayloadCarryingRpcController) invocation
687 .getArguments()[0];
688 if (controller != null) {
689 controller.setCellScanner(CellUtil.createCellScanner(cellScannables));
690 }
691 return builder.build();
692 }
693 });
694
695
696 ClusterConnection connection =
697 HConnectionTestingUtility.getMockedConnectionAndDecorate(HTU.getConfiguration(),
698 null, implementation, SERVERNAME_B, REGIONINFO);
699
700
701
702
703
704 Mockito.when(connection.isManaged()).thenReturn(true);
705 try {
706
707
708 Mockito.when(this.server.getConnection()).thenReturn(connection);
709
710
711
712 DeadServer deadServers = new DeadServer();
713 deadServers.add(SERVERNAME_A);
714 Mockito.when(this.serverManager.isServerReachable(SERVERNAME_B)).thenReturn(true);
715 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(false);
716 final Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
717 onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
718 if (deadserverRestarted) {
719
720
721 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_AA)).thenReturn(true);
722 Mockito.when(this.serverManager.isServerReachable(SERVERNAME_AA)).thenReturn(true);
723 Mockito.when(
724 this.serverManager.isServerWithSameHostnamePortOnline(SERVERNAME_A)).thenReturn(true);
725 onlineServers.put(SERVERNAME_AA, ServerLoad.EMPTY_SERVERLOAD);
726 }
727 Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(
728 new ArrayList<ServerName>(onlineServers.keySet()));
729 Mockito.when(this.serverManager.getOnlineServers()).thenReturn(onlineServers);
730 List<ServerName> avServers = new ArrayList<ServerName>();
731 avServers.addAll(onlineServers.keySet());
732 Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(avServers);
733
734 MasterFileSystem fs = Mockito.mock(MasterFileSystem.class);
735 Mockito.doNothing().when(fs).setLogRecoveryMode();
736 Mockito.when(fs.getLogRecoveryMode()).thenReturn(RecoveryMode.LOG_REPLAY);
737 MasterServices services = Mockito.mock(MasterServices.class);
738 Mockito.when(services.getAssignmentManager()).thenReturn(am);
739 Mockito.when(services.getServerManager()).thenReturn(this.serverManager);
740 Mockito.when(services.getZooKeeper()).thenReturn(this.watcher);
741 Mockito.when(services.getMasterFileSystem()).thenReturn(fs);
742 Mockito.when(services.getConnection()).thenReturn(connection);
743 Configuration conf = server.getConfiguration();
744 Mockito.when(services.getConfiguration()).thenReturn(conf);
745 ServerShutdownHandler handler = new ServerShutdownHandler(this.server,
746 services, deadServers, SERVERNAME_A, false);
747 am.failoverCleanupDone.set(true);
748 handler.process();
749
750 } finally {
751 if (connection != null) connection.close();
752 }
753 }
754
755
756
757
758
759
760
761 private ExecutorService startupMasterExecutor(final String name) {
762
763 ExecutorService executor = new ExecutorService(name);
764 executor.startExecutorService(ExecutorType.MASTER_OPEN_REGION, 3);
765 executor.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, 3);
766 executor.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS, 3);
767 executor.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS, 3);
768 return executor;
769 }
770
771 @Test (timeout=180000)
772 public void testUnassignWithSplitAtSameTime() throws KeeperException,
773 IOException, CoordinatedStateException {
774
775 final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
776
777
778
779 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, hri, -1)).thenReturn(true);
780
781 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
782 .getConfiguration());
783
784 AssignmentManager am = new AssignmentManager(this.server,
785 this.serverManager, balancer, null, null, master.getTableLockManager());
786 try {
787
788 unassign(am, SERVERNAME_A, hri);
789
790 ZKAssign.deleteClosingNode(this.watcher, hri, SERVERNAME_A);
791
792
793
794 int version = createNodeSplitting(this.watcher, hri, SERVERNAME_A);
795
796
797
798
799 unassign(am, SERVERNAME_A, hri);
800
801 ZKAssign.transitionNode(this.watcher, hri, SERVERNAME_A,
802 EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
803 assertFalse(am.getRegionStates().isRegionInTransition(hri));
804 } finally {
805 am.shutdown();
806 }
807 }
808
809
810
811
812
813
814
815 @Test(timeout = 60000)
816 public void testProcessDeadServersAndRegionsInTransitionShouldNotFailWithNPE()
817 throws IOException, KeeperException, CoordinatedStateException,
818 InterruptedException, ServiceException {
819 final RecoverableZooKeeper recoverableZk = Mockito
820 .mock(RecoverableZooKeeper.class);
821 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
822 this.server, this.serverManager);
823 Watcher zkw = new ZooKeeperWatcher(HBaseConfiguration.create(), "unittest",
824 null) {
825 @Override
826 public RecoverableZooKeeper getRecoverableZooKeeper() {
827 return recoverableZk;
828 }
829 };
830 ((ZooKeeperWatcher) zkw).registerListener(am);
831 Mockito.doThrow(new InterruptedException()).when(recoverableZk)
832 .getChildren("/hbase/region-in-transition", null);
833 am.setWatcher((ZooKeeperWatcher) zkw);
834 try {
835 am.processDeadServersAndRegionsInTransition(null);
836 fail("Expected to abort");
837 } catch (NullPointerException e) {
838 fail("Should not throw NPE");
839 } catch (RuntimeException e) {
840 assertEquals("Aborted", e.getLocalizedMessage());
841 } finally {
842 am.shutdown();
843 }
844 }
845
846
847
848
849 @Test(timeout = 60000)
850 public void testRegionPlanIsUpdatedWhenRegionFailsToOpen() throws IOException, KeeperException,
851 ServiceException, InterruptedException, CoordinatedStateException {
852 this.server.getConfiguration().setClass(
853 HConstants.HBASE_MASTER_LOADBALANCER_CLASS, MockedLoadBalancer.class,
854 LoadBalancer.class);
855 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
856 this.server, this.serverManager);
857 try {
858
859
860
861 AtomicBoolean gate = new AtomicBoolean(false);
862 if (balancer instanceof MockedLoadBalancer) {
863 ((MockedLoadBalancer) balancer).setGateVariable(gate);
864 }
865 ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A);
866 int v = ZKAssign.getVersion(this.watcher, REGIONINFO);
867 ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A,
868 EventType.M_ZK_REGION_OFFLINE, EventType.RS_ZK_REGION_FAILED_OPEN, v);
869 String path = ZKAssign.getNodeName(this.watcher, REGIONINFO
870 .getEncodedName());
871 am.getRegionStates().updateRegionState(
872 REGIONINFO, State.OPENING, SERVERNAME_A);
873
874
875 am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(
876 REGIONINFO, null, SERVERNAME_A));
877 RegionPlan regionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
878 List<ServerName> serverList = new ArrayList<ServerName>(2);
879 serverList.add(SERVERNAME_B);
880 Mockito.when(
881 this.serverManager.createDestinationServersList(SERVERNAME_A))
882 .thenReturn(serverList);
883 am.nodeDataChanged(path);
884
885
886 while (!gate.get()) {
887 Thread.sleep(10);
888 }
889
890
891
892 RegionPlan newRegionPlan = am.regionPlans
893 .get(REGIONINFO.getEncodedName());
894 while (newRegionPlan == null) {
895 Thread.sleep(10);
896 newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
897 }
898
899
900
901 assertNotSame("Same region plan should not come", regionPlan,
902 newRegionPlan);
903 assertTrue("Destination servers should be different.", !(regionPlan
904 .getDestination().equals(newRegionPlan.getDestination())));
905
906 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
907 } finally {
908 this.server.getConfiguration().setClass(
909 HConstants.HBASE_MASTER_LOADBALANCER_CLASS, StochasticLoadBalancer.class,
910 LoadBalancer.class);
911 am.getExecutorService().shutdown();
912 am.shutdown();
913 }
914 }
915
916
917
918
919
920
921 public static class MockedLoadBalancer extends StochasticLoadBalancer {
922 private AtomicBoolean gate;
923
924 public void setGateVariable(AtomicBoolean gate) {
925 this.gate = gate;
926 }
927
928 @Override
929 public ServerName randomAssignment(HRegionInfo regionInfo, List<ServerName> servers) {
930 ServerName randomServerName = super.randomAssignment(regionInfo, servers);
931 this.gate.set(true);
932 return randomServerName;
933 }
934
935 @Override
936 public Map<ServerName, List<HRegionInfo>> retainAssignment(
937 Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {
938 this.gate.set(true);
939 return super.retainAssignment(regions, servers);
940 }
941 }
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956 @Test(timeout = 60000)
957 public void testAssignmentOfRegionInSSHAndInFailedOpenState() throws IOException,
958 KeeperException, ServiceException, CoordinatedStateException, InterruptedException {
959 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
960 this.server, this.serverManager);
961 ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_B);
962 int v = ZKAssign.getVersion(this.watcher, REGIONINFO);
963 ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_B,
964 EventType.M_ZK_REGION_OFFLINE, EventType.RS_ZK_REGION_FAILED_OPEN, v);
965 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(true);
966 Mockito.when(this.serverManager.isServerReachable(SERVERNAME_B)).thenReturn(true);
967 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(false);
968 DeadServer deadServers = new DeadServer();
969 deadServers.add(SERVERNAME_A);
970 Mockito.when(this.serverManager.getDeadServers()).thenReturn(deadServers);
971 final Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
972 onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
973 Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(
974 new ArrayList<ServerName>(onlineServers.keySet()));
975 Mockito.when(this.serverManager.getOnlineServers()).thenReturn(onlineServers);
976 am.gate.set(false);
977
978 am.joinCluster();
979 while (!am.gate.get()) {
980 Thread.sleep(10);
981 }
982 assertTrue(am.getRegionStates().getRegionState(REGIONINFO).getState()
983 == RegionState.State.PENDING_OPEN);
984 am.shutdown();
985 }
986
987 @Test(timeout = 600000)
988 public void testAssignmentOfRegionRITWithOffline() throws IOException,
989 KeeperException, ServiceException, CoordinatedStateException, InterruptedException {
990 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
991 this.server, this.serverManager);
992 ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_B);
993 am.gate.set(false);
994
995 am.joinCluster();
996 while (!am.gate.get()) {
997 Thread.sleep(10);
998 }
999 assertTrue(am.getRegionStates().getRegionState(REGIONINFO).getState()
1000 == RegionState.State.PENDING_OPEN);
1001 am.shutdown();
1002 }
1003
1004
1005
1006
1007
1008 @Test(timeout = 60000)
1009 public void testRegionInOpeningStateOnDeadRSWhileMasterFailover() throws IOException,
1010 KeeperException, ServiceException, CoordinatedStateException, InterruptedException {
1011 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
1012 this.server, this.serverManager);
1013 ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A);
1014 int version = ZKAssign.getVersion(this.watcher, REGIONINFO);
1015 ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
1016 EventType.RS_ZK_REGION_OPENING, version);
1017 RegionTransition rt = RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_OPENING,
1018 REGIONINFO.getRegionName(), SERVERNAME_A, HConstants.EMPTY_BYTE_ARRAY);
1019 version = ZKAssign.getVersion(this.watcher, REGIONINFO);
1020 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(false);
1021 am.getRegionStates().logSplit(SERVERNAME_A);
1022 am.getRegionStates().createRegionState(REGIONINFO);
1023 am.gate.set(false);
1024
1025 BaseCoordinatedStateManager cp = new ZkCoordinatedStateManager();
1026 cp.initialize(server);
1027 cp.start();
1028
1029 OpenRegionCoordination orc = cp.getOpenRegionCoordination();
1030 ZkOpenRegionCoordination.ZkOpenRegionDetails zkOrd =
1031 new ZkOpenRegionCoordination.ZkOpenRegionDetails();
1032 zkOrd.setServerName(server.getServerName());
1033 zkOrd.setVersion(version);
1034
1035 assertFalse(am.processRegionsInTransition(rt, REGIONINFO, orc, zkOrd));
1036 am.getTableStateManager().setTableState(REGIONINFO.getTable(), Table.State.ENABLED);
1037 processServerShutdownHandler(am, false);
1038
1039 while (!am.gate.get()) {
1040 Thread.sleep(10);
1041 }
1042 assertTrue("The region should be assigned immediately.", null != am.regionPlans.get(REGIONINFO
1043 .getEncodedName()));
1044 am.shutdown();
1045 }
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055 @Test(timeout = 60000)
1056 public void testDisablingTableRegionsAssignmentDuringCleanClusterStartup()
1057 throws KeeperException, IOException, Exception {
1058 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
1059 MockedLoadBalancer.class, LoadBalancer.class);
1060 Mockito.when(this.serverManager.getOnlineServers()).thenReturn(
1061 new HashMap<ServerName, ServerLoad>(0));
1062 List<ServerName> destServers = new ArrayList<ServerName>(1);
1063 destServers.add(SERVERNAME_A);
1064 Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers);
1065
1066 HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
1067
1068 CoordinatedStateManager csm = CoordinatedStateManagerFactory.getCoordinatedStateManager(
1069 HTU.getConfiguration());
1070 Server server = new HMaster(HTU.getConfiguration(), csm);
1071 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
1072 this.serverManager);
1073
1074 Whitebox.setInternalState(server, "metaTableLocator", Mockito.mock(MetaTableLocator.class));
1075
1076
1077
1078 Whitebox.setInternalState(server, "clusterConnection", am.getConnection());
1079
1080 AtomicBoolean gate = new AtomicBoolean(false);
1081 if (balancer instanceof MockedLoadBalancer) {
1082 ((MockedLoadBalancer) balancer).setGateVariable(gate);
1083 }
1084 try{
1085
1086 am.getTableStateManager().setTableState(REGIONINFO.getTable(),
1087 Table.State.DISABLING);
1088 am.joinCluster();
1089
1090 assertFalse(
1091 "Assign should not be invoked for disabling table regions during clean cluster startup.",
1092 gate.get());
1093
1094 assertTrue("Table should be disabled.",
1095 am.getTableStateManager().isTableState(REGIONINFO.getTable(),
1096 Table.State.DISABLED));
1097 } finally {
1098 this.server.getConfiguration().setClass(
1099 HConstants.HBASE_MASTER_LOADBALANCER_CLASS, StochasticLoadBalancer.class,
1100 LoadBalancer.class);
1101 am.getTableStateManager().setTableState(REGIONINFO.getTable(),
1102 Table.State.ENABLED);
1103 am.shutdown();
1104 }
1105 }
1106
1107
1108
1109
1110
1111
1112
1113
1114 @Test (timeout=180000)
1115 public void testMasterRestartWhenTableInEnabling() throws KeeperException, IOException, Exception {
1116 enabling = true;
1117 List<ServerName> destServers = new ArrayList<ServerName>(1);
1118 destServers.add(SERVERNAME_A);
1119 Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers);
1120 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
1121 HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
1122 CoordinatedStateManager csm = CoordinatedStateManagerFactory.getCoordinatedStateManager(
1123 HTU.getConfiguration());
1124 Server server = new HMaster(HTU.getConfiguration(), csm);
1125 Whitebox.setInternalState(server, "serverManager", this.serverManager);
1126 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
1127 this.serverManager);
1128
1129 Whitebox.setInternalState(server, "metaTableLocator", Mockito.mock(MetaTableLocator.class));
1130
1131
1132
1133 Whitebox.setInternalState(server, "clusterConnection", am.getConnection());
1134
1135 try {
1136
1137 am.getTableStateManager().setTableState(REGIONINFO.getTable(),
1138 Table.State.ENABLING);
1139 new EnableTableHandler(server, REGIONINFO.getTable(),
1140 am, new NullTableLockManager(), true).prepare()
1141 .process();
1142 assertEquals("Number of assignments should be 1.", 1, assignmentCount);
1143 assertTrue("Table should be enabled.",
1144 am.getTableStateManager().isTableState(REGIONINFO.getTable(),
1145 Table.State.ENABLED));
1146 } finally {
1147 enabling = false;
1148 assignmentCount = 0;
1149 am.getTableStateManager().setTableState(REGIONINFO.getTable(),
1150 Table.State.ENABLED);
1151 am.shutdown();
1152 ZKAssign.deleteAllNodes(this.watcher);
1153 }
1154 }
1155
1156
1157
1158
1159
1160
1161
1162
1163 @Test (timeout=180000)
1164 public void testMasterRestartShouldRemoveStaleZnodesOfUnknownTableAsForMeta()
1165 throws Exception {
1166 List<ServerName> destServers = new ArrayList<ServerName>(1);
1167 destServers.add(SERVERNAME_A);
1168 Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers);
1169 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
1170 HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
1171 CoordinatedStateManager csm = CoordinatedStateManagerFactory.getCoordinatedStateManager(
1172 HTU.getConfiguration());
1173 Server server = new HMaster(HTU.getConfiguration(), csm);
1174 Whitebox.setInternalState(server, "serverManager", this.serverManager);
1175 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
1176 this.serverManager);
1177
1178 Whitebox.setInternalState(server, "metaTableLocator", Mockito.mock(MetaTableLocator.class));
1179
1180
1181
1182 Whitebox.setInternalState(server, "clusterConnection", am.getConnection());
1183
1184 try {
1185 TableName tableName = TableName.valueOf("dummyTable");
1186
1187 am.getTableStateManager().setTableState(tableName,
1188 Table.State.ENABLING);
1189 am.joinCluster();
1190 assertFalse("Table should not be present in zookeeper.",
1191 am.getTableStateManager().isTablePresent(tableName));
1192 } finally {
1193 am.shutdown();
1194 }
1195 }
1196
1197
1198
1199
1200
1201 @Test (timeout=180000)
1202 public void testSSHTimesOutOpeningRegionTransition()
1203 throws KeeperException, IOException, CoordinatedStateException, ServiceException {
1204
1205 AssignmentManagerWithExtrasForTesting am =
1206 setUpMockedAssignmentManager(this.server, this.serverManager);
1207
1208 RegionState state = new RegionState(REGIONINFO,
1209 State.OPENING, System.currentTimeMillis(), SERVERNAME_A);
1210 am.getRegionStates().regionOnline(REGIONINFO, SERVERNAME_B);
1211 am.getRegionStates().regionsInTransition.put(REGIONINFO.getEncodedName(), state);
1212
1213 am.regionPlans.put(REGIONINFO.getEncodedName(),
1214 new RegionPlan(REGIONINFO, SERVERNAME_B, SERVERNAME_A));
1215 am.getTableStateManager().setTableState(REGIONINFO.getTable(),
1216 Table.State.ENABLED);
1217
1218 try {
1219 am.assignInvoked = false;
1220 processServerShutdownHandler(am, false);
1221 assertTrue(am.assignInvoked);
1222 } finally {
1223 am.getRegionStates().regionsInTransition.remove(REGIONINFO.getEncodedName());
1224 am.regionPlans.remove(REGIONINFO.getEncodedName());
1225 am.shutdown();
1226 }
1227 }
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238 @Test (timeout=180000)
1239 public void testClosingFailureDuringRecovery() throws Exception {
1240
1241 AssignmentManagerWithExtrasForTesting am =
1242 setUpMockedAssignmentManager(this.server, this.serverManager);
1243 ZKAssign.createNodeClosing(this.watcher, REGIONINFO, SERVERNAME_A);
1244 try {
1245 am.getRegionStates().createRegionState(REGIONINFO);
1246
1247 assertFalse( am.getRegionStates().isRegionsInTransition() );
1248
1249 am.processRegionInTransition(REGIONINFO.getEncodedName(), REGIONINFO);
1250
1251 assertTrue( am.getRegionStates().isRegionsInTransition() );
1252 } finally {
1253 am.shutdown();
1254 }
1255 }
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273 private static int createNodeSplitting(final ZooKeeperWatcher zkw,
1274 final HRegionInfo region, final ServerName serverName)
1275 throws KeeperException, IOException {
1276 RegionTransition rt =
1277 RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_SPLITTING,
1278 region.getRegionName(), serverName);
1279
1280 String node = ZKAssign.getNodeName(zkw, region.getEncodedName());
1281 if (!ZKUtil.createEphemeralNodeAndWatch(zkw, node, rt.toByteArray())) {
1282 throw new IOException("Failed create of ephemeral " + node);
1283 }
1284
1285
1286 return transitionNodeSplitting(zkw, region, serverName, -1);
1287 }
1288
1289
1290
1291 private static int transitionNodeSplitting(final ZooKeeperWatcher zkw,
1292 final HRegionInfo parent,
1293 final ServerName serverName, final int version)
1294 throws KeeperException, IOException {
1295 return ZKAssign.transitionNode(zkw, parent, serverName,
1296 EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
1297 }
1298
1299 private void unassign(final AssignmentManager am, final ServerName sn,
1300 final HRegionInfo hri) throws RegionException {
1301
1302 am.regionOnline(hri, sn);
1303
1304 am.unassign(hri);
1305 }
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316 private AssignmentManagerWithExtrasForTesting setUpMockedAssignmentManager(final Server server,
1317 final ServerManager manager) throws IOException, KeeperException,
1318 ServiceException, CoordinatedStateException {
1319
1320
1321
1322
1323
1324 ClientProtos.ClientService.BlockingInterface ri =
1325 Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
1326
1327 Result r = MetaMockingUtil.getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
1328 final ScanResponse.Builder builder = ScanResponse.newBuilder();
1329 builder.setMoreResults(true);
1330 builder.addCellsPerResult(r.size());
1331 final List<CellScannable> rows = new ArrayList<CellScannable>(1);
1332 rows.add(r);
1333 Answer<ScanResponse> ans = new Answer<ClientProtos.ScanResponse>() {
1334 @Override
1335 public ScanResponse answer(InvocationOnMock invocation) throws Throwable {
1336 PayloadCarryingRpcController controller = (PayloadCarryingRpcController) invocation
1337 .getArguments()[0];
1338 if (controller != null) {
1339 controller.setCellScanner(CellUtil.createCellScanner(rows));
1340 }
1341 return builder.build();
1342 }
1343 };
1344 if (enabling) {
1345 Mockito.when(ri.scan((RpcController) Mockito.any(), (ScanRequest) Mockito.any()))
1346 .thenAnswer(ans).thenAnswer(ans).thenAnswer(ans).thenAnswer(ans).thenAnswer(ans)
1347 .thenReturn(ScanResponse.newBuilder().setMoreResults(false).build());
1348 } else {
1349 Mockito.when(ri.scan((RpcController) Mockito.any(), (ScanRequest) Mockito.any())).thenAnswer(
1350 ans);
1351 }
1352
1353 GetResponse.Builder getBuilder = GetResponse.newBuilder();
1354 getBuilder.setResult(ProtobufUtil.toResult(r));
1355 Mockito.when(ri.get((RpcController)Mockito.any(), (GetRequest) Mockito.any())).
1356 thenReturn(getBuilder.build());
1357
1358 ClusterConnection connection = (ClusterConnection)HConnectionTestingUtility.
1359 getMockedConnectionAndDecorate(HTU.getConfiguration(), null,
1360 ri, SERVERNAME_B, REGIONINFO);
1361 Mockito.when(this.server.getConnection()).thenReturn(connection);
1362
1363
1364
1365
1366
1367 Mockito.when(connection.isManaged()).thenReturn(true);
1368
1369
1370 ExecutorService executor = startupMasterExecutor("mockedAMExecutor");
1371 this.balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
1372 AssignmentManagerWithExtrasForTesting am = new AssignmentManagerWithExtrasForTesting(
1373 server, connection, manager, this.balancer, executor, new NullTableLockManager());
1374 return am;
1375 }
1376
1377
1378
1379
1380 class AssignmentManagerWithExtrasForTesting extends AssignmentManager {
1381
1382 private final ExecutorService es;
1383 boolean processRITInvoked = false;
1384 boolean assignInvoked = false;
1385 AtomicBoolean gate = new AtomicBoolean(true);
1386 private ClusterConnection connection;
1387
1388 public AssignmentManagerWithExtrasForTesting(
1389 final Server master, ClusterConnection connection, final ServerManager serverManager,
1390 final LoadBalancer balancer,
1391 final ExecutorService service, final TableLockManager tableLockManager)
1392 throws KeeperException, IOException, CoordinatedStateException {
1393 super(master, serverManager, balancer, service, null, tableLockManager);
1394 this.es = service;
1395 this.connection = connection;
1396 }
1397
1398 @Override
1399 boolean processRegionInTransition(String encodedRegionName,
1400 HRegionInfo regionInfo) throws KeeperException, IOException {
1401 this.processRITInvoked = true;
1402 return super.processRegionInTransition(encodedRegionName, regionInfo);
1403 }
1404
1405 @Override
1406 public void assign(HRegionInfo region, boolean setOfflineInZK, boolean forceNewPlan) {
1407 if (enabling) {
1408 assignmentCount++;
1409 this.regionOnline(region, SERVERNAME_A);
1410 } else {
1411 super.assign(region, setOfflineInZK, forceNewPlan);
1412 this.gate.set(true);
1413 }
1414 }
1415 @Override
1416 public void assign(RegionState state,
1417 boolean setOfflineInZK, final boolean forceNewPlan) {
1418 if (enabling) {
1419 assignmentCount++;
1420 } else {
1421 super.assign(state, setOfflineInZK, forceNewPlan);
1422 this.gate.set(true);
1423 }
1424 }
1425
1426 @Override
1427 boolean assign(ServerName destination, List<HRegionInfo> regions)
1428 throws InterruptedException {
1429 if (enabling) {
1430 for (HRegionInfo region : regions) {
1431 assignmentCount++;
1432 this.regionOnline(region, SERVERNAME_A);
1433 }
1434 return true;
1435 }
1436 return super.assign(destination, regions);
1437 }
1438
1439 @Override
1440 public void assign(Map<HRegionInfo, ServerName> regionServerMap)
1441 throws IOException, InterruptedException {
1442 assignInvoked = (regionServerMap != null && regionServerMap.size() > 0);
1443 super.assign(regionServerMap);
1444 this.gate.set(true);
1445 }
1446
1447 @Override
1448 public void assign(List<HRegionInfo> regions)
1449 throws IOException, InterruptedException {
1450 assignInvoked = (regions != null && regions.size() > 0);
1451 super.assign(regions);
1452 this.gate.set(true);
1453 }
1454
1455
1456 void setWatcher(ZooKeeperWatcher watcher) {
1457 this.watcher = watcher;
1458 }
1459
1460
1461
1462
1463 ExecutorService getExecutorService() {
1464 return this.es;
1465 }
1466
1467
1468
1469
1470 ClusterConnection getConnection() {
1471 return this.connection;
1472 }
1473
1474 @Override
1475 public void shutdown() {
1476 super.shutdown();
1477 if (this.connection != null)
1478 try {
1479 this.connection.close();
1480 } catch (IOException e) {
1481 fail("Failed to close connection");
1482 }
1483 }
1484 }
1485
1486
1487
1488
1489
1490
1491
1492 private void startFakeFailedOverMasterAssignmentManager(final AssignmentManager am,
1493 final ZooKeeperWatcher watcher) {
1494
1495
1496 watcher.registerListenerFirst(am);
1497 Thread t = new Thread("RunAmJoinCluster") {
1498 @Override
1499 public void run() {
1500
1501
1502
1503
1504
1505 am.getRegionStates().regionsInTransition.clear();
1506 am.regionPlans.clear();
1507 try {
1508 am.joinCluster();
1509 } catch (IOException e) {
1510 throw new RuntimeException(e);
1511 } catch (KeeperException e) {
1512 throw new RuntimeException(e);
1513 } catch (InterruptedException e) {
1514 throw new RuntimeException(e);
1515 } catch (CoordinatedStateException e) {
1516 throw new RuntimeException(e);
1517 }
1518 }
1519 };
1520 t.start();
1521 while (!t.isAlive()) Threads.sleep(1);
1522 }
1523
1524 @Test (timeout=180000)
1525 public void testForceAssignMergingRegion() throws Exception {
1526
1527 final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
1528
1529 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(
1530 server.getConfiguration());
1531
1532 AssignmentManager am = new AssignmentManager(this.server,
1533 this.serverManager, balancer, null, null, master.getTableLockManager());
1534 RegionStates regionStates = am.getRegionStates();
1535 try {
1536
1537 regionStates.updateRegionState(hri, RegionState.State.MERGING);
1538
1539 am.assign(hri, true, true);
1540 assertEquals("The region should be still in merging state",
1541 RegionState.State.MERGING, regionStates.getRegionState(hri).getState());
1542 } finally {
1543 am.shutdown();
1544 }
1545 }
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557 @Test (timeout=180000)
1558 public void testAssignmentEventIgnoredIfNotExpected() throws KeeperException, IOException,
1559 CoordinatedStateException {
1560
1561 final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
1562 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(
1563 server.getConfiguration());
1564 final AtomicBoolean zkEventProcessed = new AtomicBoolean(false);
1565
1566 AssignmentManager am = new AssignmentManager(this.server,
1567 this.serverManager, balancer, null, null, master.getTableLockManager()) {
1568
1569 @Override
1570 void handleRegion(final RegionTransition rt, OpenRegionCoordination coordination,
1571 OpenRegionCoordination.OpenRegionDetails ord) {
1572 super.handleRegion(rt, coordination, ord);
1573 if (rt != null && Bytes.equals(hri.getRegionName(),
1574 rt.getRegionName()) && rt.getEventType() == EventType.RS_ZK_REGION_OPENING) {
1575 zkEventProcessed.set(true);
1576 }
1577 }
1578 };
1579 try {
1580
1581 am.getRegionStates().regionOffline(hri);
1582 zkEventProcessed.set(false);
1583 this.watcher.registerListenerFirst(am);
1584 assertFalse("The region should not be in transition",
1585 am.getRegionStates().isRegionInTransition(hri));
1586 ZKAssign.createNodeOffline(this.watcher, hri, SERVERNAME_A);
1587
1588 ZKAssign.transitionNodeOpening(this.watcher, hri, SERVERNAME_A);
1589 long startTime = EnvironmentEdgeManager.currentTime();
1590 while (!zkEventProcessed.get()) {
1591 assertTrue("Timed out in waiting for ZK event to be processed",
1592 EnvironmentEdgeManager.currentTime() - startTime < 30000);
1593 Threads.sleepWithoutInterrupt(100);
1594 }
1595 assertFalse(am.getRegionStates().isRegionInTransition(hri));
1596 } finally {
1597 am.shutdown();
1598 }
1599 }
1600
1601
1602
1603
1604
1605
1606 @Test (timeout=180000)
1607 public void testBalanceRegionOfDeletedTable() throws Exception {
1608 AssignmentManager am = new AssignmentManager(this.server, this.serverManager,
1609 balancer, null, null, master.getTableLockManager());
1610 RegionStates regionStates = am.getRegionStates();
1611 HRegionInfo hri = REGIONINFO;
1612 regionStates.createRegionState(hri);
1613 assertFalse(regionStates.isRegionInTransition(hri));
1614 RegionPlan plan = new RegionPlan(hri, SERVERNAME_A, SERVERNAME_B);
1615
1616 regionStates.tableDeleted(hri.getTable());
1617 am.balance(plan);
1618 assertFalse("The region should not in transition",
1619 regionStates.isRegionInTransition(hri));
1620 am.shutdown();
1621 }
1622
1623
1624
1625
1626
1627 @SuppressWarnings("unchecked")
1628 @Test (timeout=180000)
1629 public void testOpenCloseRegionRPCIntendedForPreviousServer() throws Exception {
1630 Mockito.when(this.serverManager.sendRegionOpen(Mockito.eq(SERVERNAME_B), Mockito.eq(REGIONINFO),
1631 Mockito.anyInt(), (List<ServerName>)Mockito.any()))
1632 .thenThrow(new DoNotRetryIOException());
1633 this.server.getConfiguration().setInt("hbase.assignment.maximum.attempts", 100);
1634
1635 HRegionInfo hri = REGIONINFO;
1636 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(
1637 server.getConfiguration());
1638
1639 AssignmentManager am = new AssignmentManager(this.server,
1640 this.serverManager, balancer, null, null, master.getTableLockManager());
1641 RegionStates regionStates = am.getRegionStates();
1642 try {
1643 am.regionPlans.put(REGIONINFO.getEncodedName(),
1644 new RegionPlan(REGIONINFO, null, SERVERNAME_B));
1645
1646
1647 am.assign(hri, true, false);
1648 } finally {
1649 assertEquals(SERVERNAME_A, regionStates.getRegionState(REGIONINFO).getServerName());
1650 am.shutdown();
1651 }
1652 }
1653
1654
1655
1656
1657 @Test (timeout=180000)
1658 public void testCloseRegionOnAbortingRS() throws Exception {
1659 this.server.getConfiguration().setInt("hbase.assignment.maximum.attempts", 2);
1660
1661 HRegionInfo hri = REGIONINFO;
1662 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(
1663 server.getConfiguration());
1664
1665 AssignmentManager am = new AssignmentManager(this.server,
1666 this.serverManager, balancer, null, null, master.getTableLockManager());
1667 RegionStates regionStates = am.getRegionStates();
1668
1669 regionStates.createRegionState(hri, State.OPEN, SERVERNAME_B, SERVERNAME_B);
1670
1671
1672 Mockito.when(this.serverManager.sendRegionClose(Mockito.eq(SERVERNAME_B), Mockito.eq(REGIONINFO),
1673 Mockito.anyInt(), (ServerName)Mockito.any(), Mockito.anyBoolean()))
1674 .thenThrow(new RegionServerAbortedException(""));
1675
1676
1677 am.unassign(hri);
1678
1679
1680 assertEquals(State.FAILED_CLOSE, regionStates.getRegionState(REGIONINFO).getState());
1681 assertEquals(SERVERNAME_B, regionStates.getRegionState(REGIONINFO).getServerName());
1682
1683 am.shutdown();
1684 }
1685
1686
1687
1688
1689 @Test (timeout=180000)
1690 public void testCloseRegionOnServerNotOnline() throws Exception {
1691 this.server.getConfiguration().setInt("hbase.assignment.maximum.attempts", 2);
1692
1693 HRegionInfo hri = REGIONINFO;
1694 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(
1695 server.getConfiguration());
1696
1697 AssignmentManager am = new AssignmentManager(this.server,
1698 this.serverManager, balancer, null, null, master.getTableLockManager()) {
1699 @Override
1700 protected boolean wasRegionOnDeadServerByMeta(HRegionInfo region, ServerName sn) {
1701 return true;
1702 };
1703 };
1704 RegionStates regionStates = am.getRegionStates();
1705
1706 regionStates.createRegionState(hri, State.OPEN, SERVERNAME_B, SERVERNAME_B);
1707
1708
1709 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B))
1710 .thenReturn(false);
1711
1712
1713 am.unassign(hri);
1714
1715
1716 assertEquals(State.OFFLINE, regionStates.getRegionState(REGIONINFO).getState());
1717
1718
1719 am.regionPlans.put(REGIONINFO.getEncodedName(),
1720 new RegionPlan(REGIONINFO, null, SERVERNAME_A));
1721 am.assign(hri, true, false);
1722
1723
1724 assertEquals(State.OFFLINE, regionStates.getRegionState(REGIONINFO).getState());
1725
1726 am.shutdown();
1727 }
1728 }