View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertNotSame;
23  import static org.junit.Assert.assertTrue;
24  import static org.junit.Assert.fail;
25  
26  import java.io.IOException;
27  import java.util.ArrayList;
28  import java.util.HashMap;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.concurrent.atomic.AtomicBoolean;
32  
33  import org.apache.hadoop.conf.Configuration;
34  import org.apache.hadoop.hbase.CellScannable;
35  import org.apache.hadoop.hbase.CellUtil;
36  import org.apache.hadoop.hbase.CoordinatedStateException;
37  import org.apache.hadoop.hbase.CoordinatedStateManager;
38  import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
39  import org.apache.hadoop.hbase.DoNotRetryIOException;
40  import org.apache.hadoop.hbase.HBaseConfiguration;
41  import org.apache.hadoop.hbase.HBaseTestingUtility;
42  import org.apache.hadoop.hbase.HConstants;
43  import org.apache.hadoop.hbase.HRegionInfo;
44  import org.apache.hadoop.hbase.MetaMockingUtil;
45  import org.apache.hadoop.hbase.RegionException;
46  import org.apache.hadoop.hbase.RegionTransition;
47  import org.apache.hadoop.hbase.Server;
48  import org.apache.hadoop.hbase.ServerLoad;
49  import org.apache.hadoop.hbase.ServerName;
50  import org.apache.hadoop.hbase.TableName;
51  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
52  import org.apache.hadoop.hbase.client.ClusterConnection;
53  import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
54  import org.apache.hadoop.hbase.client.Result;
55  import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
56  import org.apache.hadoop.hbase.coordination.OpenRegionCoordination;
57  import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
58  import org.apache.hadoop.hbase.coordination.ZkOpenRegionCoordination;
59  import org.apache.hadoop.hbase.exceptions.DeserializationException;
60  import org.apache.hadoop.hbase.executor.EventType;
61  import org.apache.hadoop.hbase.executor.ExecutorService;
62  import org.apache.hadoop.hbase.executor.ExecutorType;
63  import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
64  import org.apache.hadoop.hbase.master.RegionState.State;
65  import org.apache.hadoop.hbase.master.TableLockManager.NullTableLockManager;
66  import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
67  import org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer;
68  import org.apache.hadoop.hbase.master.handler.EnableTableHandler;
69  import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
70  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
71  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
72  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest;
73  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetResponse;
74  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
75  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
76  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
77  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.Table;
78  import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
79  import org.apache.hadoop.hbase.regionserver.RegionServerAbortedException;
80  import org.apache.hadoop.hbase.testclassification.MediumTests;
81  import org.apache.hadoop.hbase.util.Bytes;
82  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
83  import org.apache.hadoop.hbase.util.Threads;
84  import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
85  import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper;
86  import org.apache.hadoop.hbase.zookeeper.ZKAssign;
87  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
88  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
89  import org.apache.zookeeper.KeeperException;
90  import org.apache.zookeeper.KeeperException.NodeExistsException;
91  import org.apache.zookeeper.Watcher;
92  import org.junit.After;
93  import org.junit.AfterClass;
94  import org.junit.Before;
95  import org.junit.BeforeClass;
96  import org.junit.Test;
97  import org.junit.experimental.categories.Category;
98  import org.mockito.Mockito;
99  import org.mockito.internal.util.reflection.Whitebox;
100 import org.mockito.invocation.InvocationOnMock;
101 import org.mockito.stubbing.Answer;
102 
103 import com.google.protobuf.RpcController;
104 import com.google.protobuf.ServiceException;
105 
106 
107 /**
108  * Test {@link AssignmentManager}
109  */
110 @Category(MediumTests.class)
111 public class TestAssignmentManager {
112   private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
113   private static final ServerName SERVERNAME_A =
114       ServerName.valueOf("example.org", 1234, 5678);
115   private static final ServerName SERVERNAME_AA =
116       ServerName.valueOf("example.org", 1234, 9999);
117   private static final ServerName SERVERNAME_B =
118       ServerName.valueOf("example.org", 0, 5678);
119   private static final HRegionInfo REGIONINFO =
120     new HRegionInfo(TableName.valueOf("t"),
121       HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
122   private static int assignmentCount;
123   private static boolean enabling = false;
124 
125   // Mocked objects or; get redone for each test.
126   private Server server;
127   private ServerManager serverManager;
128   private ZooKeeperWatcher watcher;
129   private CoordinatedStateManager cp;
130   private MetaTableLocator mtl;
131   private LoadBalancer balancer;
132   private HMaster master;
133   private ClusterConnection connection;
134 
135   @BeforeClass
136   public static void beforeClass() throws Exception {
137     HTU.getConfiguration().setBoolean("hbase.assignment.usezk", true);
138     HTU.startMiniZKCluster();
139   }
140 
141   @AfterClass
142   public static void afterClass() throws IOException {
143     HTU.shutdownMiniZKCluster();
144   }
145 
146   @Before
147   public void before() throws ZooKeeperConnectionException, IOException {
148     // TODO: Make generic versions of what we do below and put up in a mocking
149     // utility class or move up into HBaseTestingUtility.
150 
151     // Mock a Server.  Have it return a legit Configuration and ZooKeeperWatcher.
152     // If abort is called, be sure to fail the test (don't just swallow it
153     // silently as is mockito default).
154     this.server = Mockito.mock(Server.class);
155     Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("master,1,1"));
156     Mockito.when(server.getConfiguration()).thenReturn(HTU.getConfiguration());
157     this.watcher =
158       new ZooKeeperWatcher(HTU.getConfiguration(), "mockedServer", this.server, true);
159     Mockito.when(server.getZooKeeper()).thenReturn(this.watcher);
160     Mockito.doThrow(new RuntimeException("Aborted")).
161       when(server).abort(Mockito.anyString(), (Throwable)Mockito.anyObject());
162 
163     cp = new ZkCoordinatedStateManager();
164     cp.initialize(this.server);
165     cp.start();
166 
167     mtl = Mockito.mock(MetaTableLocator.class);
168 
169     Mockito.when(server.getCoordinatedStateManager()).thenReturn(cp);
170     Mockito.when(server.getMetaTableLocator()).thenReturn(mtl);
171 
172     // Get a connection w/ mocked up common methods.
173     this.connection =
174       (ClusterConnection)HConnectionTestingUtility.getMockedConnection(HTU.getConfiguration());
175 
176     // Make it so we can get a catalogtracker from servermanager.. .needed
177     // down in guts of server shutdown handler.
178     Mockito.when(server.getConnection()).thenReturn(connection);
179     Mockito.when(connection.isManaged()).thenReturn(true);
180 
181     // Mock a ServerManager.  Say server SERVERNAME_{A,B} are online.  Also
182     // make it so if close or open, we return 'success'.
183     this.serverManager = Mockito.mock(ServerManager.class);
184     Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
185     Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(true);
186     Mockito.when(this.serverManager.getDeadServers()).thenReturn(new DeadServer());
187     final Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
188     onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
189     onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
190     Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(
191         new ArrayList<ServerName>(onlineServers.keySet()));
192     Mockito.when(this.serverManager.getOnlineServers()).thenReturn(onlineServers);
193 
194     List<ServerName> avServers = new ArrayList<ServerName>();
195     avServers.addAll(onlineServers.keySet());
196     Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(avServers);
197     Mockito.when(this.serverManager.createDestinationServersList(null)).thenReturn(avServers);
198 
199     Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, REGIONINFO, -1)).
200       thenReturn(true);
201     Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_B, REGIONINFO, -1)).
202       thenReturn(true);
203     // Ditto on open.
204     Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_A, REGIONINFO, -1, null)).
205       thenReturn(RegionOpeningState.OPENED);
206     Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_B, REGIONINFO, -1, null)).
207       thenReturn(RegionOpeningState.OPENED);
208     this.master = Mockito.mock(HMaster.class);
209 
210     Mockito.when(this.master.getServerManager()).thenReturn(serverManager);
211   }
212 
213   @After public void after() throws KeeperException, IOException {
214     if (this.watcher != null) {
215       // Clean up all znodes
216       ZKAssign.deleteAllNodes(this.watcher);
217       this.watcher.close();
218       this.cp.stop();
219     }
220     if (this.connection != null) this.connection.close();
221   }
222 
223   /**
224    * Test a balance going on at same time as a master failover
225    *
226    * @throws IOException
227    * @throws KeeperException
228    * @throws InterruptedException
229    * @throws DeserializationException
230    */
231   @Test(timeout = 60000)
232   public void testBalanceOnMasterFailoverScenarioWithOpenedNode()
233       throws IOException, KeeperException, InterruptedException, ServiceException,
234       DeserializationException, CoordinatedStateException {
235     AssignmentManagerWithExtrasForTesting am =
236       setUpMockedAssignmentManager(this.server, this.serverManager);
237     try {
238       createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
239       startFakeFailedOverMasterAssignmentManager(am, this.watcher);
240       while (!am.processRITInvoked) Thread.sleep(1);
241       // As part of the failover cleanup, the balancing region plan is removed.
242       // So a random server will be used to open the region. For testing purpose,
243       // let's assume it is going to open on server b:
244       am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_B));
245 
246       Mocking.waitForRegionFailedToCloseAndSetToPendingClose(am, REGIONINFO);
247 
248       // Now fake the region closing successfully over on the regionserver; the
249       // regionserver will have set the region in CLOSED state. This will
250       // trigger callback into AM. The below zk close call is from the RS close
251       // region handler duplicated here because its down deep in a private
252       // method hard to expose.
253       int versionid =
254         ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
255       assertNotSame(versionid, -1);
256       Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
257 
258       // Get current versionid else will fail on transition from OFFLINE to
259       // OPENING below
260       versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
261       assertNotSame(-1, versionid);
262       // This uglyness below is what the openregionhandler on RS side does.
263       versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
264         SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
265         EventType.RS_ZK_REGION_OPENING, versionid);
266       assertNotSame(-1, versionid);
267       // Move znode from OPENING to OPENED as RS does on successful open.
268       versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
269         SERVERNAME_B, versionid);
270       assertNotSame(-1, versionid);
271       am.gate.set(false);
272       // Block here until our znode is cleared or until this test times out.
273       ZKAssign.blockUntilNoRIT(watcher);
274     } finally {
275       am.getExecutorService().shutdown();
276       am.shutdown();
277     }
278   }
279 
280   @Test(timeout = 60000)
281   public void testBalanceOnMasterFailoverScenarioWithClosedNode()
282       throws IOException, KeeperException, InterruptedException, ServiceException,
283         DeserializationException, CoordinatedStateException {
284     AssignmentManagerWithExtrasForTesting am =
285       setUpMockedAssignmentManager(this.server, this.serverManager);
286     try {
287       createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
288       startFakeFailedOverMasterAssignmentManager(am, this.watcher);
289       while (!am.processRITInvoked) Thread.sleep(1);
290       // As part of the failover cleanup, the balancing region plan is removed.
291       // So a random server will be used to open the region. For testing purpose,
292       // let's assume it is going to open on server b:
293       am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_B));
294 
295       Mocking.waitForRegionFailedToCloseAndSetToPendingClose(am, REGIONINFO);
296 
297       // Now fake the region closing successfully over on the regionserver; the
298       // regionserver will have set the region in CLOSED state. This will
299       // trigger callback into AM. The below zk close call is from the RS close
300       // region handler duplicated here because its down deep in a private
301       // method hard to expose.
302       int versionid =
303         ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
304       assertNotSame(versionid, -1);
305       am.gate.set(false);
306       Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
307 
308       // Get current versionid else will fail on transition from OFFLINE to
309       // OPENING below
310       versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
311       assertNotSame(-1, versionid);
312       // This uglyness below is what the openregionhandler on RS side does.
313       versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
314           SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
315           EventType.RS_ZK_REGION_OPENING, versionid);
316       assertNotSame(-1, versionid);
317       // Move znode from OPENING to OPENED as RS does on successful open.
318       versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
319           SERVERNAME_B, versionid);
320       assertNotSame(-1, versionid);
321 
322       // Block here until our znode is cleared or until this test timesout.
323       ZKAssign.blockUntilNoRIT(watcher);
324     } finally {
325       am.getExecutorService().shutdown();
326       am.shutdown();
327     }
328   }
329 
330   @Test(timeout = 60000)
331   public void testBalanceOnMasterFailoverScenarioWithOfflineNode()
332       throws IOException, KeeperException, InterruptedException, ServiceException,
333       DeserializationException, CoordinatedStateException {
334     AssignmentManagerWithExtrasForTesting am =
335       setUpMockedAssignmentManager(this.server, this.serverManager);
336     try {
337       createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
338       startFakeFailedOverMasterAssignmentManager(am, this.watcher);
339       while (!am.processRITInvoked) Thread.sleep(1);
340       // As part of the failover cleanup, the balancing region plan is removed.
341       // So a random server will be used to open the region. For testing purpose,
342       // let's assume it is going to open on server b:
343       am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_B));
344 
345       Mocking.waitForRegionFailedToCloseAndSetToPendingClose(am, REGIONINFO);
346 
347       // Now fake the region closing successfully over on the regionserver; the
348       // regionserver will have set the region in CLOSED state. This will
349       // trigger callback into AM. The below zk close call is from the RS close
350       // region handler duplicated here because its down deep in a private
351       // method hard to expose.
352       int versionid =
353         ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
354       assertNotSame(versionid, -1);
355       Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
356 
357       am.gate.set(false);
358       // Get current versionid else will fail on transition from OFFLINE to
359       // OPENING below
360       versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
361       assertNotSame(-1, versionid);
362       // This uglyness below is what the openregionhandler on RS side does.
363       versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
364           SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
365           EventType.RS_ZK_REGION_OPENING, versionid);
366       assertNotSame(-1, versionid);
367       // Move znode from OPENING to OPENED as RS does on successful open.
368       versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
369           SERVERNAME_B, versionid);
370       assertNotSame(-1, versionid);
371       // Block here until our znode is cleared or until this test timesout.
372       ZKAssign.blockUntilNoRIT(watcher);
373     } finally {
374       am.getExecutorService().shutdown();
375       am.shutdown();
376     }
377   }
378 
379   private void createRegionPlanAndBalance(
380       final AssignmentManager am, final ServerName from,
381       final ServerName to, final HRegionInfo hri) throws RegionException {
382     // Call the balance function but fake the region being online first at
383     // servername from.
384     am.regionOnline(hri, from);
385     // Balance region from 'from' to 'to'. It calls unassign setting CLOSING state
386     // up in zk.  Create a plan and balance
387     am.balance(new RegionPlan(hri, from, to));
388   }
389 
390   /**
391    * Tests AssignmentManager balance function.  Runs a balance moving a region
392    * from one server to another mocking regionserver responding over zk.
393    * @throws IOException
394    * @throws KeeperException
395    * @throws DeserializationException
396    */
397   @Test (timeout=180000)
398   public void testBalance() throws IOException, KeeperException, DeserializationException,
399       InterruptedException, CoordinatedStateException {
400     // Create and startup an executor.  This is used by AssignmentManager
401     // handling zk callbacks.
402     ExecutorService executor = startupMasterExecutor("testBalanceExecutor");
403 
404     // We need a mocked catalog tracker.
405     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
406         .getConfiguration());
407     // Create an AM.
408     AssignmentManager am = new AssignmentManager(this.server,
409       this.serverManager, balancer, executor, null, master.getTableLockManager());
410     am.failoverCleanupDone.set(true);
411     try {
412       // Make sure our new AM gets callbacks; once registered, can't unregister.
413       // Thats ok because we make a new zk watcher for each test.
414       this.watcher.registerListenerFirst(am);
415       // Call the balance function but fake the region being online first at
416       // SERVERNAME_A.  Create a balance plan.
417       am.regionOnline(REGIONINFO, SERVERNAME_A);
418       // Balance region from A to B.
419       RegionPlan plan = new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B);
420       am.balance(plan);
421 
422       RegionStates regionStates = am.getRegionStates();
423       // Must be failed to close since the server is fake
424       assertTrue(regionStates.isRegionInTransition(REGIONINFO)
425         && regionStates.isRegionInState(REGIONINFO, State.FAILED_CLOSE));
426       // Move it back to pending_close
427       regionStates.updateRegionState(REGIONINFO, State.PENDING_CLOSE);
428 
429       // Now fake the region closing successfully over on the regionserver; the
430       // regionserver will have set the region in CLOSED state.  This will
431       // trigger callback into AM. The below zk close call is from the RS close
432       // region handler duplicated here because its down deep in a private
433       // method hard to expose.
434       int versionid =
435         ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
436       assertNotSame(versionid, -1);
437       // AM is going to notice above CLOSED and queue up a new assign.  The
438       // assign will go to open the region in the new location set by the
439       // balancer.  The zk node will be OFFLINE waiting for regionserver to
440       // transition it through OPENING, OPENED.  Wait till we see the OFFLINE
441       // zk node before we proceed.
442       Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
443 
444       // Get current versionid else will fail on transition from OFFLINE to OPENING below
445       versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
446       assertNotSame(-1, versionid);
447       // This uglyness below is what the openregionhandler on RS side does.
448       versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
449         SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
450         EventType.RS_ZK_REGION_OPENING, versionid);
451       assertNotSame(-1, versionid);
452       // Move znode from OPENING to OPENED as RS does on successful open.
453       versionid =
454         ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO, SERVERNAME_B, versionid);
455       assertNotSame(-1, versionid);
456       // Wait on the handler removing the OPENED znode.
457       while(regionStates.isRegionInTransition(REGIONINFO)) Threads.sleep(1);
458     } finally {
459       executor.shutdown();
460       am.shutdown();
461       // Clean up all znodes
462       ZKAssign.deleteAllNodes(this.watcher);
463     }
464   }
465 
466   /**
467    * Run a simple server shutdown handler.
468    * @throws KeeperException
469    * @throws IOException
470    */
471   @Test (timeout=180000)
472   public void testShutdownHandler()
473       throws KeeperException, IOException, CoordinatedStateException, ServiceException {
474     // Create and startup an executor.  This is used by AssignmentManager
475     // handling zk callbacks.
476     ExecutorService executor = startupMasterExecutor("testShutdownHandler");
477 
478     // Create an AM.
479     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
480         this.server, this.serverManager);
481     try {
482       processServerShutdownHandler(am, false);
483     } finally {
484       executor.shutdown();
485       am.shutdown();
486       // Clean up all znodes
487       ZKAssign.deleteAllNodes(this.watcher);
488     }
489   }
490 
491   /**
492    * Run a simple server shutdown handler after the same server restarts.
493    * @throws KeeperException
494    * @throws IOException
495    */
496   @Test (timeout=180000)
497   public void testShutdownHandlerWithRestartedServer()
498       throws KeeperException, IOException, CoordinatedStateException, ServiceException {
499     // Create and startup an executor.  This is used by AssignmentManager
500     // handling zk callbacks.
501     ExecutorService executor = startupMasterExecutor("testShutdownHandlerWithRestartedServer");
502 
503     // Create an AM.
504     AssignmentManagerWithExtrasForTesting am =
505       setUpMockedAssignmentManager(this.server, this.serverManager);
506     am.getRegionStates().regionOnline(REGIONINFO, SERVERNAME_A);
507     am.getTableStateManager().setTableState(REGIONINFO.getTable(), Table.State.ENABLED);
508     try {
509       processServerShutdownHandler(am, false, true);
510     } finally {
511       executor.shutdown();
512       am.shutdown();
513       // Clean up all znodes
514       ZKAssign.deleteAllNodes(this.watcher);
515     }
516   }
517 
518   /**
519    * To test closed region handler to remove rit and delete corresponding znode
520    * if region in pending close or closing while processing shutdown of a region
521    * server.(HBASE-5927).
522    *
523    * @throws KeeperException
524    * @throws IOException
525    * @throws ServiceException
526    */
527   @Test (timeout=180000)
528   public void testSSHWhenDisableTableInProgress() throws KeeperException, IOException,
529     CoordinatedStateException, ServiceException {
530     testCaseWithPartiallyDisabledState(Table.State.DISABLING);
531     testCaseWithPartiallyDisabledState(Table.State.DISABLED);
532   }
533 
534 
535   /**
536    * To test if the split region is removed from RIT if the region was in SPLITTING state but the RS
537    * has actually completed the splitting in hbase:meta but went down. See HBASE-6070 and also HBASE-5806
538    *
539    * @throws KeeperException
540    * @throws IOException
541    */
542   @Test (timeout=180000)
543   public void testSSHWhenSplitRegionInProgress() throws KeeperException, IOException, Exception {
544     // true indicates the region is split but still in RIT
545     testCaseWithSplitRegionPartial(true);
546     // false indicate the region is not split
547     testCaseWithSplitRegionPartial(false);
548   }
549 
550   private void testCaseWithSplitRegionPartial(boolean regionSplitDone) throws KeeperException,
551       IOException, InterruptedException,
552     CoordinatedStateException, ServiceException {
553     // Create and startup an executor. This is used by AssignmentManager
554     // handling zk callbacks.
555     ExecutorService executor = startupMasterExecutor("testSSHWhenSplitRegionInProgress");
556     // We need a mocked catalog tracker.
557     ZKAssign.deleteAllNodes(this.watcher);
558 
559     // Create an AM.
560     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
561       this.server, this.serverManager);
562     // adding region to regions and servers maps.
563     am.regionOnline(REGIONINFO, SERVERNAME_A);
564     // adding region in pending close.
565     am.getRegionStates().updateRegionState(
566       REGIONINFO, State.SPLITTING, SERVERNAME_A);
567     am.getTableStateManager().setTableState(REGIONINFO.getTable(),
568       Table.State.ENABLED);
569     RegionTransition data = RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_SPLITTING,
570         REGIONINFO.getRegionName(), SERVERNAME_A);
571     String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
572     // create znode in M_ZK_REGION_CLOSING state.
573     ZKUtil.createAndWatch(this.watcher, node, data.toByteArray());
574 
575     try {
576       processServerShutdownHandler(am, regionSplitDone);
577       // check znode deleted or not.
578       // In both cases the znode should be deleted.
579 
580       if (regionSplitDone) {
581         assertFalse("Region state of region in SPLITTING should be removed from rit.",
582             am.getRegionStates().isRegionsInTransition());
583       } else {
584         while (!am.assignInvoked) {
585           Thread.sleep(1);
586         }
587         assertTrue("Assign should be invoked.", am.assignInvoked);
588       }
589     } finally {
590       REGIONINFO.setOffline(false);
591       REGIONINFO.setSplit(false);
592       executor.shutdown();
593       am.shutdown();
594       // Clean up all znodes
595       ZKAssign.deleteAllNodes(this.watcher);
596     }
597   }
598 
599   private void testCaseWithPartiallyDisabledState(Table.State state) throws KeeperException,
600       IOException, CoordinatedStateException, ServiceException {
601     // Create and startup an executor. This is used by AssignmentManager
602     // handling zk callbacks.
603     ExecutorService executor = startupMasterExecutor("testSSHWhenDisableTableInProgress");
604     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
605     ZKAssign.deleteAllNodes(this.watcher);
606 
607     // Create an AM.
608     AssignmentManager am = new AssignmentManager(this.server,
609       this.serverManager, balancer, executor, null, master.getTableLockManager());
610     // adding region to regions and servers maps.
611     am.regionOnline(REGIONINFO, SERVERNAME_A);
612     // adding region in pending close.
613     am.getRegionStates().updateRegionState(REGIONINFO, State.PENDING_CLOSE);
614     if (state == Table.State.DISABLING) {
615       am.getTableStateManager().setTableState(REGIONINFO.getTable(),
616         Table.State.DISABLING);
617     } else {
618       am.getTableStateManager().setTableState(REGIONINFO.getTable(),
619         Table.State.DISABLED);
620     }
621     RegionTransition data = RegionTransition.createRegionTransition(EventType.M_ZK_REGION_CLOSING,
622         REGIONINFO.getRegionName(), SERVERNAME_A);
623     // RegionTransitionData data = new
624     // RegionTransitionData(EventType.M_ZK_REGION_CLOSING,
625     // REGIONINFO.getRegionName(), SERVERNAME_A);
626     String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
627     // create znode in M_ZK_REGION_CLOSING state.
628     ZKUtil.createAndWatch(this.watcher, node, data.toByteArray());
629 
630     try {
631       processServerShutdownHandler(am, false);
632       // check znode deleted or not.
633       // In both cases the znode should be deleted.
634       assertTrue("The znode should be deleted.", ZKUtil.checkExists(this.watcher, node) == -1);
635       // check whether in rit or not. In the DISABLING case also the below
636       // assert will be true but the piece of code added for HBASE-5927 will not
637       // do that.
638       if (state == Table.State.DISABLED) {
639         assertFalse("Region state of region in pending close should be removed from rit.",
640             am.getRegionStates().isRegionsInTransition());
641       }
642     } finally {
643       am.setEnabledTable(REGIONINFO.getTable());
644       executor.shutdown();
645       am.shutdown();
646       // Clean up all znodes
647       ZKAssign.deleteAllNodes(this.watcher);
648     }
649   }
650 
651   private void processServerShutdownHandler(AssignmentManager am, boolean splitRegion)
652       throws IOException, ServiceException {
653     processServerShutdownHandler(am, splitRegion, false);
654   }
655 
656   private void processServerShutdownHandler(
657       AssignmentManager am, boolean splitRegion, boolean deadserverRestarted)
658       throws IOException, ServiceException {
659     // Make sure our new AM gets callbacks; once registered, can't unregister.
660     // Thats ok because we make a new zk watcher for each test.
661     this.watcher.registerListenerFirst(am);
662 
663     // Need to set up a fake scan of meta for the servershutdown handler
664     // Make an RS Interface implementation.  Make it so a scanner can go against it.
665     ClientProtos.ClientService.BlockingInterface implementation =
666       Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
667     // Get a meta row result that has region up on SERVERNAME_A
668 
669     Result r;
670     if (splitRegion) {
671       r = MetaMockingUtil.getMetaTableRowResultAsSplitRegion(REGIONINFO, SERVERNAME_A);
672     } else {
673       r = MetaMockingUtil.getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
674     }
675 
676     final ScanResponse.Builder builder = ScanResponse.newBuilder();
677     builder.setMoreResults(true);
678     builder.addCellsPerResult(r.size());
679     final List<CellScannable> cellScannables = new ArrayList<CellScannable>(1);
680     cellScannables.add(r);
681     Mockito.when(implementation.scan(
682       (RpcController)Mockito.any(), (ScanRequest)Mockito.any())).
683       thenAnswer(new Answer<ScanResponse>() {
684           @Override
685           public ScanResponse answer(InvocationOnMock invocation) throws Throwable {
686             PayloadCarryingRpcController controller = (PayloadCarryingRpcController) invocation
687                 .getArguments()[0];
688             if (controller != null) {
689               controller.setCellScanner(CellUtil.createCellScanner(cellScannables));
690             }
691             return builder.build();
692           }
693       });
694 
695     // Get a connection w/ mocked up common methods.
696     ClusterConnection connection =
697       HConnectionTestingUtility.getMockedConnectionAndDecorate(HTU.getConfiguration(),
698         null, implementation, SERVERNAME_B, REGIONINFO);
699     // These mocks were done up when all connections were managed.  World is different now we
700     // moved to unmanaged connections.  It messes up the intercepts done in these tests.
701     // Just mark connections as marked and then down in MetaTableAccessor, it will go the path
702     // that picks up the above mocked up 'implementation' so 'scans' of meta return the expected
703     // result.  Redo in new realm of unmanaged connections.
704     Mockito.when(connection.isManaged()).thenReturn(true);
705     try {
706       // Make it so we can get a catalogtracker from servermanager.. .needed
707       // down in guts of server shutdown handler.
708       Mockito.when(this.server.getConnection()).thenReturn(connection);
709 
710       // Now make a server shutdown handler instance and invoke process.
711       // Have it that SERVERNAME_A died.
712       DeadServer deadServers = new DeadServer();
713       deadServers.add(SERVERNAME_A);
714       Mockito.when(this.serverManager.isServerReachable(SERVERNAME_B)).thenReturn(true);
715       Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(false);
716       final Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
717       onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
718       if (deadserverRestarted) {
719         // Now make the same server (same host name and port) online again with a different
720         // start code.
721         Mockito.when(this.serverManager.isServerOnline(SERVERNAME_AA)).thenReturn(true);
722         Mockito.when(this.serverManager.isServerReachable(SERVERNAME_AA)).thenReturn(true);
723         Mockito.when(
724           this.serverManager.isServerWithSameHostnamePortOnline(SERVERNAME_A)).thenReturn(true);
725         onlineServers.put(SERVERNAME_AA, ServerLoad.EMPTY_SERVERLOAD);
726       }
727       Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(
728           new ArrayList<ServerName>(onlineServers.keySet()));
729       Mockito.when(this.serverManager.getOnlineServers()).thenReturn(onlineServers);
730       List<ServerName> avServers = new ArrayList<ServerName>();
731       avServers.addAll(onlineServers.keySet());
732       Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(avServers);
733       // I need a services instance that will return the AM
734       MasterFileSystem fs = Mockito.mock(MasterFileSystem.class);
735       Mockito.doNothing().when(fs).setLogRecoveryMode();
736       Mockito.when(fs.getLogRecoveryMode()).thenReturn(RecoveryMode.LOG_REPLAY);
737       MasterServices services = Mockito.mock(MasterServices.class);
738       Mockito.when(services.getAssignmentManager()).thenReturn(am);
739       Mockito.when(services.getServerManager()).thenReturn(this.serverManager);
740       Mockito.when(services.getZooKeeper()).thenReturn(this.watcher);
741       Mockito.when(services.getMasterFileSystem()).thenReturn(fs);
742       Mockito.when(services.getConnection()).thenReturn(connection);
743       Configuration conf = server.getConfiguration();
744       Mockito.when(services.getConfiguration()).thenReturn(conf);
745       ServerShutdownHandler handler = new ServerShutdownHandler(this.server,
746           services, deadServers, SERVERNAME_A, false);
747       am.failoverCleanupDone.set(true);
748       handler.process();
749       // The region in r will have been assigned.  It'll be up in zk as unassigned.
750     } finally {
751       if (connection != null) connection.close();
752     }
753   }
754 
755   /**
756    * Create and startup executor pools. Start same set as master does (just
757    * run a few less).
758    * @param name Name to give our executor
759    * @return Created executor (be sure to call shutdown when done).
760    */
761   private ExecutorService startupMasterExecutor(final String name) {
762     // TODO: Move up into HBaseTestingUtility?  Generally useful.
763     ExecutorService executor = new ExecutorService(name);
764     executor.startExecutorService(ExecutorType.MASTER_OPEN_REGION, 3);
765     executor.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, 3);
766     executor.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS, 3);
767     executor.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS, 3);
768     return executor;
769   }
770 
771   @Test (timeout=180000)
772   public void testUnassignWithSplitAtSameTime() throws KeeperException,
773       IOException, CoordinatedStateException {
774     // Region to use in test.
775     final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
776     // First amend the servermanager mock so that when we do send close of the
777     // first meta region on SERVERNAME_A, it will return true rather than
778     // default null.
779     Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, hri, -1)).thenReturn(true);
780     // Need a mocked catalog tracker.
781     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
782         .getConfiguration());
783     // Create an AM.
784     AssignmentManager am = new AssignmentManager(this.server,
785       this.serverManager, balancer, null, null, master.getTableLockManager());
786     try {
787       // First make sure my mock up basically works.  Unassign a region.
788       unassign(am, SERVERNAME_A, hri);
789       // This delete will fail if the previous unassign did wrong thing.
790       ZKAssign.deleteClosingNode(this.watcher, hri, SERVERNAME_A);
791       // Now put a SPLITTING region in the way.  I don't have to assert it
792       // go put in place.  This method puts it in place then asserts it still
793       // owns it by moving state from SPLITTING to SPLITTING.
794       int version = createNodeSplitting(this.watcher, hri, SERVERNAME_A);
795       // Now, retry the unassign with the SPLTTING in place.  It should just
796       // complete without fail; a sort of 'silent' recognition that the
797       // region to unassign has been split and no longer exists: TOOD: what if
798       // the split fails and the parent region comes back to life?
799       unassign(am, SERVERNAME_A, hri);
800       // This transition should fail if the znode has been messed with.
801       ZKAssign.transitionNode(this.watcher, hri, SERVERNAME_A,
802         EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
803       assertFalse(am.getRegionStates().isRegionInTransition(hri));
804     } finally {
805       am.shutdown();
806     }
807   }
808 
809   /**
810    * Tests the processDeadServersAndRegionsInTransition should not fail with NPE
811    * when it failed to get the children. Let's abort the system in this
812    * situation
813    * @throws ServiceException
814    */
815   @Test(timeout = 60000)
816   public void testProcessDeadServersAndRegionsInTransitionShouldNotFailWithNPE()
817       throws IOException, KeeperException, CoordinatedStateException,
818       InterruptedException, ServiceException {
819     final RecoverableZooKeeper recoverableZk = Mockito
820         .mock(RecoverableZooKeeper.class);
821     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
822       this.server, this.serverManager);
823     Watcher zkw = new ZooKeeperWatcher(HBaseConfiguration.create(), "unittest",
824         null) {
825       @Override
826       public RecoverableZooKeeper getRecoverableZooKeeper() {
827         return recoverableZk;
828       }
829     };
830     ((ZooKeeperWatcher) zkw).registerListener(am);
831     Mockito.doThrow(new InterruptedException()).when(recoverableZk)
832         .getChildren("/hbase/region-in-transition", null);
833     am.setWatcher((ZooKeeperWatcher) zkw);
834     try {
835       am.processDeadServersAndRegionsInTransition(null);
836       fail("Expected to abort");
837     } catch (NullPointerException e) {
838       fail("Should not throw NPE");
839     } catch (RuntimeException e) {
840       assertEquals("Aborted", e.getLocalizedMessage());
841     } finally {
842       am.shutdown();
843     }
844   }
845   /**
846    * TestCase verifies that the regionPlan is updated whenever a region fails to open
847    * and the master tries to process RS_ZK_FAILED_OPEN state.(HBASE-5546).
848    */
849   @Test(timeout = 60000)
850   public void testRegionPlanIsUpdatedWhenRegionFailsToOpen() throws IOException, KeeperException,
851       ServiceException, InterruptedException, CoordinatedStateException {
852     this.server.getConfiguration().setClass(
853       HConstants.HBASE_MASTER_LOADBALANCER_CLASS, MockedLoadBalancer.class,
854       LoadBalancer.class);
855     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
856       this.server, this.serverManager);
857     try {
858       // Boolean variable used for waiting until randomAssignment is called and
859       // new
860       // plan is generated.
861       AtomicBoolean gate = new AtomicBoolean(false);
862       if (balancer instanceof MockedLoadBalancer) {
863         ((MockedLoadBalancer) balancer).setGateVariable(gate);
864       }
865       ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A);
866       int v = ZKAssign.getVersion(this.watcher, REGIONINFO);
867       ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A,
868           EventType.M_ZK_REGION_OFFLINE, EventType.RS_ZK_REGION_FAILED_OPEN, v);
869       String path = ZKAssign.getNodeName(this.watcher, REGIONINFO
870           .getEncodedName());
871       am.getRegionStates().updateRegionState(
872         REGIONINFO, State.OPENING, SERVERNAME_A);
873       // a dummy plan inserted into the regionPlans. This plan is cleared and
874       // new one is formed
875       am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(
876           REGIONINFO, null, SERVERNAME_A));
877       RegionPlan regionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
878       List<ServerName> serverList = new ArrayList<ServerName>(2);
879       serverList.add(SERVERNAME_B);
880       Mockito.when(
881           this.serverManager.createDestinationServersList(SERVERNAME_A))
882           .thenReturn(serverList);
883       am.nodeDataChanged(path);
884       // here we are waiting until the random assignment in the load balancer is
885       // called.
886       while (!gate.get()) {
887         Thread.sleep(10);
888       }
889       // new region plan may take some time to get updated after random
890       // assignment is called and
891       // gate is set to true.
892       RegionPlan newRegionPlan = am.regionPlans
893           .get(REGIONINFO.getEncodedName());
894       while (newRegionPlan == null) {
895         Thread.sleep(10);
896         newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
897       }
898       // the new region plan created may contain the same RS as destination but
899       // it should
900       // be new plan.
901       assertNotSame("Same region plan should not come", regionPlan,
902           newRegionPlan);
903       assertTrue("Destination servers should be different.", !(regionPlan
904           .getDestination().equals(newRegionPlan.getDestination())));
905 
906       Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
907     } finally {
908       this.server.getConfiguration().setClass(
909           HConstants.HBASE_MASTER_LOADBALANCER_CLASS, StochasticLoadBalancer.class,
910           LoadBalancer.class);
911       am.getExecutorService().shutdown();
912       am.shutdown();
913     }
914   }
915 
916 
917   /**
918    * Mocked load balancer class used in the testcase to make sure that the testcase waits until
919    * random assignment is called and the gate variable is set to true.
920    */
921   public static class MockedLoadBalancer extends StochasticLoadBalancer {
922     private AtomicBoolean gate;
923 
924     public void setGateVariable(AtomicBoolean gate) {
925       this.gate = gate;
926     }
927 
928     @Override
929     public ServerName randomAssignment(HRegionInfo regionInfo, List<ServerName> servers) {
930       ServerName randomServerName = super.randomAssignment(regionInfo, servers);
931       this.gate.set(true);
932       return randomServerName;
933     }
934 
935     @Override
936     public Map<ServerName, List<HRegionInfo>> retainAssignment(
937         Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {
938       this.gate.set(true);
939       return super.retainAssignment(regions, servers);
940     }
941   }
942 
943   /*
944    * Tests the scenario
945    * - a regionserver (SERVERNAME_A) owns a region (hence the meta would have
946    *   the SERVERNAME_A as the host for the region),
947    * - SERVERNAME_A goes down
948    * - one of the affected regions is assigned to a live regionserver (SERVERNAME_B) but that
949    *   assignment somehow fails. The region ends up in the FAILED_OPEN state on ZK
950    * - [Issue that the patch on HBASE-13330 fixes] when the master is restarted,
951    *   the SSH for SERVERNAME_A rightly thinks that the region is now on transition on
952    *   SERVERNAME_B. But the owner for the region is still SERVERNAME_A in the AM's states.
953    *   The AM thinks that the SSH for SERVERNAME_A will assign the region. The region remains
954    *   unassigned for ever.
955    */
956   @Test(timeout = 60000)
957   public void testAssignmentOfRegionInSSHAndInFailedOpenState() throws IOException,
958   KeeperException, ServiceException, CoordinatedStateException, InterruptedException {
959     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
960         this.server, this.serverManager);
961     ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_B);
962     int v = ZKAssign.getVersion(this.watcher, REGIONINFO);
963     ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_B,
964         EventType.M_ZK_REGION_OFFLINE, EventType.RS_ZK_REGION_FAILED_OPEN, v);
965     Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(true);
966     Mockito.when(this.serverManager.isServerReachable(SERVERNAME_B)).thenReturn(true);
967     Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(false);
968     DeadServer deadServers = new DeadServer();
969     deadServers.add(SERVERNAME_A);
970     Mockito.when(this.serverManager.getDeadServers()).thenReturn(deadServers);
971     final Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
972     onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
973     Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(
974         new ArrayList<ServerName>(onlineServers.keySet()));
975     Mockito.when(this.serverManager.getOnlineServers()).thenReturn(onlineServers);
976     am.gate.set(false);
977     // join the cluster - that's when the AM is really kicking in after a restart
978     am.joinCluster();
979     while (!am.gate.get()) {
980       Thread.sleep(10);
981     }
982     assertTrue(am.getRegionStates().getRegionState(REGIONINFO).getState()
983         == RegionState.State.PENDING_OPEN);
984     am.shutdown();
985   }
986 
987   @Test(timeout = 600000)
988   public void testAssignmentOfRegionRITWithOffline() throws IOException,
989       KeeperException, ServiceException, CoordinatedStateException, InterruptedException {
990     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
991         this.server, this.serverManager);
992     ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_B);
993     am.gate.set(false);
994     // join the cluster - that's when the AM is really kicking in after a restart
995     am.joinCluster();
996     while (!am.gate.get()) {
997       Thread.sleep(10);
998     }
999     assertTrue(am.getRegionStates().getRegionState(REGIONINFO).getState()
1000         == RegionState.State.PENDING_OPEN);
1001     am.shutdown();
1002   }
1003   /**
1004    * Test the scenario when the master is in failover and trying to process a
1005    * region which is in Opening state on a dead RS. Master will force offline the
1006    * region and put it in transition. AM relies on SSH to reassign it.
1007    */
1008   @Test(timeout = 60000)
1009   public void testRegionInOpeningStateOnDeadRSWhileMasterFailover() throws IOException,
1010       KeeperException, ServiceException, CoordinatedStateException, InterruptedException {
1011     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
1012       this.server, this.serverManager);
1013     ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A);
1014     int version = ZKAssign.getVersion(this.watcher, REGIONINFO);
1015     ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
1016         EventType.RS_ZK_REGION_OPENING, version);
1017     RegionTransition rt = RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_OPENING,
1018         REGIONINFO.getRegionName(), SERVERNAME_A, HConstants.EMPTY_BYTE_ARRAY);
1019     version = ZKAssign.getVersion(this.watcher, REGIONINFO);
1020     Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(false);
1021     am.getRegionStates().logSplit(SERVERNAME_A); // Assume log splitting is done
1022     am.getRegionStates().createRegionState(REGIONINFO);
1023     am.gate.set(false);
1024 
1025     BaseCoordinatedStateManager cp = new ZkCoordinatedStateManager();
1026     cp.initialize(server);
1027     cp.start();
1028 
1029     OpenRegionCoordination orc = cp.getOpenRegionCoordination();
1030     ZkOpenRegionCoordination.ZkOpenRegionDetails zkOrd =
1031       new ZkOpenRegionCoordination.ZkOpenRegionDetails();
1032     zkOrd.setServerName(server.getServerName());
1033     zkOrd.setVersion(version);
1034 
1035     assertFalse(am.processRegionsInTransition(rt, REGIONINFO, orc, zkOrd));
1036     am.getTableStateManager().setTableState(REGIONINFO.getTable(), Table.State.ENABLED);
1037     processServerShutdownHandler(am, false);
1038     // Waiting for the assignment to get completed.
1039     while (!am.gate.get()) {
1040       Thread.sleep(10);
1041     }
1042     assertTrue("The region should be assigned immediately.", null != am.regionPlans.get(REGIONINFO
1043         .getEncodedName()));
1044     am.shutdown();
1045   }
1046 
1047   /**
1048    * Test verifies whether assignment is skipped for regions of tables in DISABLING state during
1049    * clean cluster startup. See HBASE-6281.
1050    *
1051    * @throws KeeperException
1052    * @throws IOException
1053    * @throws Exception
1054    */
1055   @Test(timeout = 60000)
1056   public void testDisablingTableRegionsAssignmentDuringCleanClusterStartup()
1057       throws KeeperException, IOException, Exception {
1058     this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
1059         MockedLoadBalancer.class, LoadBalancer.class);
1060     Mockito.when(this.serverManager.getOnlineServers()).thenReturn(
1061         new HashMap<ServerName, ServerLoad>(0));
1062     List<ServerName> destServers = new ArrayList<ServerName>(1);
1063     destServers.add(SERVERNAME_A);
1064     Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers);
1065     // To avoid cast exception in DisableTableHandler process.
1066     HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
1067 
1068     CoordinatedStateManager csm = CoordinatedStateManagerFactory.getCoordinatedStateManager(
1069       HTU.getConfiguration());
1070     Server server = new HMaster(HTU.getConfiguration(), csm);
1071     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
1072         this.serverManager);
1073 
1074     Whitebox.setInternalState(server, "metaTableLocator", Mockito.mock(MetaTableLocator.class));
1075 
1076     // Make it so we can get a catalogtracker from servermanager.. .needed
1077     // down in guts of server shutdown handler.
1078     Whitebox.setInternalState(server, "clusterConnection", am.getConnection());
1079 
1080     AtomicBoolean gate = new AtomicBoolean(false);
1081     if (balancer instanceof MockedLoadBalancer) {
1082       ((MockedLoadBalancer) balancer).setGateVariable(gate);
1083     }
1084     try{
1085       // set table in disabling state.
1086       am.getTableStateManager().setTableState(REGIONINFO.getTable(),
1087         Table.State.DISABLING);
1088       am.joinCluster();
1089       // should not call retainAssignment if we get empty regions in assignAllUserRegions.
1090       assertFalse(
1091           "Assign should not be invoked for disabling table regions during clean cluster startup.",
1092           gate.get());
1093       // need to change table state from disabling to disabled.
1094       assertTrue("Table should be disabled.",
1095           am.getTableStateManager().isTableState(REGIONINFO.getTable(),
1096             Table.State.DISABLED));
1097     } finally {
1098       this.server.getConfiguration().setClass(
1099         HConstants.HBASE_MASTER_LOADBALANCER_CLASS, StochasticLoadBalancer.class,
1100         LoadBalancer.class);
1101       am.getTableStateManager().setTableState(REGIONINFO.getTable(),
1102         Table.State.ENABLED);
1103       am.shutdown();
1104     }
1105   }
1106 
1107   /**
1108    * Test verifies whether all the enabling table regions assigned only once during master startup.
1109    *
1110    * @throws KeeperException
1111    * @throws IOException
1112    * @throws Exception
1113    */
1114   @Test (timeout=180000)
1115   public void testMasterRestartWhenTableInEnabling() throws KeeperException, IOException, Exception {
1116     enabling = true;
1117     List<ServerName> destServers = new ArrayList<ServerName>(1);
1118     destServers.add(SERVERNAME_A);
1119     Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers);
1120     Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
1121     HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
1122     CoordinatedStateManager csm = CoordinatedStateManagerFactory.getCoordinatedStateManager(
1123       HTU.getConfiguration());
1124     Server server = new HMaster(HTU.getConfiguration(), csm);
1125     Whitebox.setInternalState(server, "serverManager", this.serverManager);
1126     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
1127         this.serverManager);
1128 
1129     Whitebox.setInternalState(server, "metaTableLocator", Mockito.mock(MetaTableLocator.class));
1130 
1131     // Make it so we can get a catalogtracker from servermanager.. .needed
1132     // down in guts of server shutdown handler.
1133     Whitebox.setInternalState(server, "clusterConnection", am.getConnection());
1134 
1135     try {
1136       // set table in enabling state.
1137       am.getTableStateManager().setTableState(REGIONINFO.getTable(),
1138         Table.State.ENABLING);
1139       new EnableTableHandler(server, REGIONINFO.getTable(),
1140           am, new NullTableLockManager(), true).prepare()
1141           .process();
1142       assertEquals("Number of assignments should be 1.", 1, assignmentCount);
1143       assertTrue("Table should be enabled.",
1144           am.getTableStateManager().isTableState(REGIONINFO.getTable(),
1145             Table.State.ENABLED));
1146     } finally {
1147       enabling = false;
1148       assignmentCount = 0;
1149       am.getTableStateManager().setTableState(REGIONINFO.getTable(),
1150         Table.State.ENABLED);
1151       am.shutdown();
1152       ZKAssign.deleteAllNodes(this.watcher);
1153     }
1154   }
1155 
1156   /**
1157    * Test verifies whether stale znodes of unknown tables as for the hbase:meta will be removed or
1158    * not.
1159    * @throws KeeperException
1160    * @throws IOException
1161    * @throws Exception
1162    */
1163   @Test (timeout=180000)
1164   public void testMasterRestartShouldRemoveStaleZnodesOfUnknownTableAsForMeta()
1165       throws Exception {
1166     List<ServerName> destServers = new ArrayList<ServerName>(1);
1167     destServers.add(SERVERNAME_A);
1168     Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers);
1169     Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
1170     HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
1171     CoordinatedStateManager csm = CoordinatedStateManagerFactory.getCoordinatedStateManager(
1172       HTU.getConfiguration());
1173     Server server = new HMaster(HTU.getConfiguration(), csm);
1174     Whitebox.setInternalState(server, "serverManager", this.serverManager);
1175     AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
1176         this.serverManager);
1177 
1178     Whitebox.setInternalState(server, "metaTableLocator", Mockito.mock(MetaTableLocator.class));
1179 
1180     // Make it so we can get a catalogtracker from servermanager.. .needed
1181     // down in guts of server shutdown handler.
1182     Whitebox.setInternalState(server, "clusterConnection", am.getConnection());
1183 
1184     try {
1185       TableName tableName = TableName.valueOf("dummyTable");
1186       // set table in enabling state.
1187       am.getTableStateManager().setTableState(tableName,
1188         Table.State.ENABLING);
1189       am.joinCluster();
1190       assertFalse("Table should not be present in zookeeper.",
1191         am.getTableStateManager().isTablePresent(tableName));
1192     } finally {
1193       am.shutdown();
1194     }
1195   }
1196   /**
1197    * When a region is in transition, if the region server opening the region goes down,
1198    * the region assignment takes a long time normally (waiting for timeout monitor to trigger assign).
1199    * This test is to make sure SSH reassigns it right away.
1200    */
1201   @Test (timeout=180000)
1202   public void testSSHTimesOutOpeningRegionTransition()
1203       throws KeeperException, IOException, CoordinatedStateException, ServiceException {
1204     // Create an AM.
1205     AssignmentManagerWithExtrasForTesting am =
1206       setUpMockedAssignmentManager(this.server, this.serverManager);
1207     // adding region in pending open.
1208     RegionState state = new RegionState(REGIONINFO,
1209       State.OPENING, System.currentTimeMillis(), SERVERNAME_A);
1210     am.getRegionStates().regionOnline(REGIONINFO, SERVERNAME_B);
1211     am.getRegionStates().regionsInTransition.put(REGIONINFO.getEncodedName(), state);
1212     // adding region plan
1213     am.regionPlans.put(REGIONINFO.getEncodedName(),
1214       new RegionPlan(REGIONINFO, SERVERNAME_B, SERVERNAME_A));
1215     am.getTableStateManager().setTableState(REGIONINFO.getTable(),
1216       Table.State.ENABLED);
1217 
1218     try {
1219       am.assignInvoked = false;
1220       processServerShutdownHandler(am, false);
1221       assertTrue(am.assignInvoked);
1222     } finally {
1223       am.getRegionStates().regionsInTransition.remove(REGIONINFO.getEncodedName());
1224       am.regionPlans.remove(REGIONINFO.getEncodedName());
1225       am.shutdown();
1226     }
1227   }
1228 
1229   /**
1230    * Scenario:<ul>
1231    *  <li> master starts a close, and creates a znode</li>
1232    *  <li> it fails just at this moment, before contacting the RS</li>
1233    *  <li> while the second master is coming up, the targeted RS dies. But it's before ZK timeout so
1234    *    we don't know, and we have an exception.</li>
1235    *  <li> the master must handle this nicely and reassign.
1236    *  </ul>
1237    */
1238   @Test (timeout=180000)
1239   public void testClosingFailureDuringRecovery() throws Exception {
1240 
1241     AssignmentManagerWithExtrasForTesting am =
1242         setUpMockedAssignmentManager(this.server, this.serverManager);
1243     ZKAssign.createNodeClosing(this.watcher, REGIONINFO, SERVERNAME_A);
1244     try {
1245       am.getRegionStates().createRegionState(REGIONINFO);
1246 
1247       assertFalse( am.getRegionStates().isRegionsInTransition() );
1248 
1249       am.processRegionInTransition(REGIONINFO.getEncodedName(), REGIONINFO);
1250 
1251       assertTrue( am.getRegionStates().isRegionsInTransition() );
1252     } finally {
1253       am.shutdown();
1254     }
1255   }
1256 
1257   /**
1258    * Creates a new ephemeral node in the SPLITTING state for the specified region.
1259    * Create it ephemeral in case regionserver dies mid-split.
1260    *
1261    * <p>Does not transition nodes from other states.  If a node already exists
1262    * for this region, a {@link NodeExistsException} will be thrown.
1263    *
1264    * @param zkw zk reference
1265    * @param region region to be created as offline
1266    * @param serverName server event originates from
1267    * @return Version of znode created.
1268    * @throws KeeperException
1269    * @throws IOException
1270    */
1271   // Copied from SplitTransaction rather than open the method over there in
1272   // the regionserver package.
1273   private static int createNodeSplitting(final ZooKeeperWatcher zkw,
1274       final HRegionInfo region, final ServerName serverName)
1275   throws KeeperException, IOException {
1276     RegionTransition rt =
1277       RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_SPLITTING,
1278         region.getRegionName(), serverName);
1279 
1280     String node = ZKAssign.getNodeName(zkw, region.getEncodedName());
1281     if (!ZKUtil.createEphemeralNodeAndWatch(zkw, node, rt.toByteArray())) {
1282       throw new IOException("Failed create of ephemeral " + node);
1283     }
1284     // Transition node from SPLITTING to SPLITTING and pick up version so we
1285     // can be sure this znode is ours; version is needed deleting.
1286     return transitionNodeSplitting(zkw, region, serverName, -1);
1287   }
1288 
1289   // Copied from SplitTransaction rather than open the method over there in
1290   // the regionserver package.
1291   private static int transitionNodeSplitting(final ZooKeeperWatcher zkw,
1292       final HRegionInfo parent,
1293       final ServerName serverName, final int version)
1294   throws KeeperException, IOException {
1295     return ZKAssign.transitionNode(zkw, parent, serverName,
1296       EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
1297   }
1298 
1299   private void unassign(final AssignmentManager am, final ServerName sn,
1300       final HRegionInfo hri) throws RegionException {
1301     // Before I can unassign a region, I need to set it online.
1302     am.regionOnline(hri, sn);
1303     // Unassign region.
1304     am.unassign(hri);
1305   }
1306 
1307   /**
1308    * Create an {@link AssignmentManagerWithExtrasForTesting} that has mocked
1309    * {@link CatalogTracker} etc.
1310    * @param server
1311    * @param manager
1312    * @return An AssignmentManagerWithExtras with mock connections, etc.
1313    * @throws IOException
1314    * @throws KeeperException
1315    */
1316   private AssignmentManagerWithExtrasForTesting setUpMockedAssignmentManager(final Server server,
1317       final ServerManager manager) throws IOException, KeeperException,
1318         ServiceException, CoordinatedStateException {
1319     // Make an RS Interface implementation. Make it so a scanner can go against
1320     // it and a get to return the single region, REGIONINFO, this test is
1321     // messing with. Needed when "new master" joins cluster. AM will try and
1322     // rebuild its list of user regions and it will also get the HRI that goes
1323     // with an encoded name by doing a Get on hbase:meta
1324     ClientProtos.ClientService.BlockingInterface ri =
1325       Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
1326     // Get a meta row result that has region up on SERVERNAME_A for REGIONINFO
1327     Result r = MetaMockingUtil.getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
1328     final ScanResponse.Builder builder = ScanResponse.newBuilder();
1329     builder.setMoreResults(true);
1330     builder.addCellsPerResult(r.size());
1331     final List<CellScannable> rows = new ArrayList<CellScannable>(1);
1332     rows.add(r);
1333     Answer<ScanResponse> ans = new Answer<ClientProtos.ScanResponse>() {
1334       @Override
1335       public ScanResponse answer(InvocationOnMock invocation) throws Throwable {
1336         PayloadCarryingRpcController controller = (PayloadCarryingRpcController) invocation
1337             .getArguments()[0];
1338         if (controller != null) {
1339           controller.setCellScanner(CellUtil.createCellScanner(rows));
1340         }
1341         return builder.build();
1342       }
1343     };
1344     if (enabling) {
1345       Mockito.when(ri.scan((RpcController) Mockito.any(), (ScanRequest) Mockito.any()))
1346           .thenAnswer(ans).thenAnswer(ans).thenAnswer(ans).thenAnswer(ans).thenAnswer(ans)
1347           .thenReturn(ScanResponse.newBuilder().setMoreResults(false).build());
1348     } else {
1349       Mockito.when(ri.scan((RpcController) Mockito.any(), (ScanRequest) Mockito.any())).thenAnswer(
1350           ans);
1351     }
1352     // If a get, return the above result too for REGIONINFO
1353     GetResponse.Builder getBuilder = GetResponse.newBuilder();
1354     getBuilder.setResult(ProtobufUtil.toResult(r));
1355     Mockito.when(ri.get((RpcController)Mockito.any(), (GetRequest) Mockito.any())).
1356       thenReturn(getBuilder.build());
1357     // Get a connection w/ mocked up common methods.
1358     ClusterConnection connection = (ClusterConnection)HConnectionTestingUtility.
1359       getMockedConnectionAndDecorate(HTU.getConfiguration(), null,
1360         ri, SERVERNAME_B, REGIONINFO);
1361     Mockito.when(this.server.getConnection()).thenReturn(connection);
1362     // These mocks were done up when all connections were managed.  World is different now we
1363     // moved to unmanaged connections.  It messes up the intercepts done in these tests.
1364     // Just mark connections as marked and then down in MetaTableAccessor, it will go the path
1365     // that picks up the above mocked up 'implementation' so 'scans' of meta return the expected
1366     // result.  Redo in new realm of unmanaged connections.
1367     Mockito.when(connection.isManaged()).thenReturn(true);
1368     // Make it so we can get the connection from our mocked catalogtracker
1369     // Create and startup an executor. Used by AM handling zk callbacks.
1370     ExecutorService executor = startupMasterExecutor("mockedAMExecutor");
1371     this.balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
1372     AssignmentManagerWithExtrasForTesting am = new AssignmentManagerWithExtrasForTesting(
1373       server, connection, manager, this.balancer, executor, new NullTableLockManager());
1374     return am;
1375   }
1376 
1377   /**
1378    * An {@link AssignmentManager} with some extra facility used testing
1379    */
1380   class AssignmentManagerWithExtrasForTesting extends AssignmentManager {
1381     // Keep a reference so can give it out below in {@link #getExecutorService}
1382     private final ExecutorService es;
1383     boolean processRITInvoked = false;
1384     boolean assignInvoked = false;
1385     AtomicBoolean gate = new AtomicBoolean(true);
1386     private ClusterConnection connection;
1387 
1388     public AssignmentManagerWithExtrasForTesting(
1389         final Server master, ClusterConnection connection, final ServerManager serverManager,
1390         final LoadBalancer balancer,
1391         final ExecutorService service, final TableLockManager tableLockManager)
1392             throws KeeperException, IOException, CoordinatedStateException {
1393       super(master, serverManager, balancer, service, null, tableLockManager);
1394       this.es = service;
1395       this.connection = connection;
1396     }
1397 
1398     @Override
1399     boolean processRegionInTransition(String encodedRegionName,
1400         HRegionInfo regionInfo) throws KeeperException, IOException {
1401       this.processRITInvoked = true;
1402       return super.processRegionInTransition(encodedRegionName, regionInfo);
1403     }
1404 
1405     @Override
1406     public void assign(HRegionInfo region, boolean setOfflineInZK, boolean forceNewPlan) {
1407       if (enabling) {
1408         assignmentCount++;
1409         this.regionOnline(region, SERVERNAME_A);
1410       } else {
1411         super.assign(region, setOfflineInZK, forceNewPlan);
1412         this.gate.set(true);
1413       }
1414     }
1415     @Override
1416     public void assign(RegionState state,
1417         boolean setOfflineInZK, final boolean forceNewPlan) {
1418       if (enabling) {
1419         assignmentCount++;
1420       } else {
1421         super.assign(state, setOfflineInZK, forceNewPlan);
1422         this.gate.set(true);
1423       }
1424     }
1425 
1426     @Override
1427     boolean assign(ServerName destination, List<HRegionInfo> regions)
1428         throws InterruptedException {
1429       if (enabling) {
1430         for (HRegionInfo region : regions) {
1431           assignmentCount++;
1432           this.regionOnline(region, SERVERNAME_A);
1433         }
1434         return true;
1435       }
1436       return super.assign(destination, regions);
1437     }
1438 
1439     @Override
1440     public void assign(Map<HRegionInfo, ServerName> regionServerMap)
1441         throws IOException, InterruptedException {
1442       assignInvoked = (regionServerMap != null && regionServerMap.size() > 0);
1443       super.assign(regionServerMap);
1444       this.gate.set(true);
1445     }
1446 
1447     @Override
1448     public void assign(List<HRegionInfo> regions)
1449         throws IOException, InterruptedException {
1450       assignInvoked = (regions != null && regions.size() > 0);
1451       super.assign(regions);
1452       this.gate.set(true);
1453     }
1454 
1455     /** reset the watcher */
1456     void setWatcher(ZooKeeperWatcher watcher) {
1457       this.watcher = watcher;
1458     }
1459 
1460     /**
1461      * @return ExecutorService used by this instance.
1462      */
1463     ExecutorService getExecutorService() {
1464       return this.es;
1465     }
1466 
1467     /*
1468      * Convenient method to retrieve mocked up connection
1469      */
1470     ClusterConnection getConnection() {
1471       return this.connection;
1472     }
1473 
1474     @Override
1475     public void shutdown() {
1476       super.shutdown();
1477       if (this.connection != null)
1478         try {
1479           this.connection.close();
1480         } catch (IOException e) {
1481           fail("Failed to close connection");
1482         }
1483     }
1484   }
1485 
1486   /**
1487    * Call joinCluster on the passed AssignmentManager.  Do it in a thread
1488    * so it runs independent of what all else is going on.  Try to simulate
1489    * an AM running insided a failed over master by clearing all in-memory
1490    * AM state first.
1491   */
1492   private void startFakeFailedOverMasterAssignmentManager(final AssignmentManager am,
1493       final ZooKeeperWatcher watcher) {
1494     // Make sure our new AM gets callbacks; once registered, we can't unregister.
1495     // Thats ok because we make a new zk watcher for each test.
1496     watcher.registerListenerFirst(am);
1497     Thread t = new Thread("RunAmJoinCluster") {
1498       @Override
1499       public void run() {
1500         // Call the joinCluster function as though we were doing a master
1501         // failover at this point. It will stall just before we go to add
1502         // the RIT region to our RIT Map in AM at processRegionsInTransition.
1503         // First clear any inmemory state from AM so it acts like a new master
1504         // coming on line.
1505         am.getRegionStates().regionsInTransition.clear();
1506         am.regionPlans.clear();
1507         try {
1508           am.joinCluster();
1509         } catch (IOException e) {
1510           throw new RuntimeException(e);
1511         } catch (KeeperException e) {
1512           throw new RuntimeException(e);
1513         } catch (InterruptedException e) {
1514           throw new RuntimeException(e);
1515         } catch (CoordinatedStateException e) {
1516           throw new RuntimeException(e);
1517         }
1518       }
1519     };
1520     t.start();
1521     while (!t.isAlive()) Threads.sleep(1);
1522   }
1523 
1524   @Test (timeout=180000)
1525   public void testForceAssignMergingRegion() throws Exception {
1526     // Region to use in test.
1527     final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
1528     // Need a mocked catalog tracker.
1529     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(
1530       server.getConfiguration());
1531     // Create an AM.
1532     AssignmentManager am = new AssignmentManager(this.server,
1533       this.serverManager, balancer, null, null, master.getTableLockManager());
1534     RegionStates regionStates = am.getRegionStates();
1535     try {
1536       // First set the state of the region to merging
1537       regionStates.updateRegionState(hri, RegionState.State.MERGING);
1538       // Now, try to assign it with force new plan
1539       am.assign(hri, true, true);
1540       assertEquals("The region should be still in merging state",
1541         RegionState.State.MERGING, regionStates.getRegionState(hri).getState());
1542     } finally {
1543       am.shutdown();
1544     }
1545   }
1546 
1547   /**
1548    * Test assignment related ZK events are ignored by AM if the region is not known
1549    * by AM to be in transition. During normal operation, all assignments are started
1550    * by AM (not considering split/merge), if an event is received but the region
1551    * is not in transition, the event must be a very late one. So it can be ignored.
1552    * During master failover, since AM watches assignment znodes after failover cleanup
1553    * is completed, when an event comes in, AM should already have the region in transition
1554    * if ZK is used during the assignment action (only hbck doesn't use ZK for region
1555    * assignment). So during master failover, we can ignored such events too.
1556    */
1557   @Test (timeout=180000)
1558   public void testAssignmentEventIgnoredIfNotExpected() throws KeeperException, IOException,
1559       CoordinatedStateException {
1560     // Region to use in test.
1561     final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
1562     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(
1563       server.getConfiguration());
1564     final AtomicBoolean zkEventProcessed = new AtomicBoolean(false);
1565     // Create an AM.
1566     AssignmentManager am = new AssignmentManager(this.server,
1567       this.serverManager, balancer, null, null, master.getTableLockManager()) {
1568 
1569       @Override
1570       void handleRegion(final RegionTransition rt, OpenRegionCoordination coordination,
1571                         OpenRegionCoordination.OpenRegionDetails ord) {
1572         super.handleRegion(rt, coordination, ord);
1573         if (rt != null && Bytes.equals(hri.getRegionName(),
1574           rt.getRegionName()) && rt.getEventType() == EventType.RS_ZK_REGION_OPENING) {
1575           zkEventProcessed.set(true);
1576         }
1577       }
1578     };
1579     try {
1580       // First make sure the region is not in transition
1581       am.getRegionStates().regionOffline(hri);
1582       zkEventProcessed.set(false); // Reset it before faking zk transition
1583       this.watcher.registerListenerFirst(am);
1584       assertFalse("The region should not be in transition",
1585         am.getRegionStates().isRegionInTransition(hri));
1586       ZKAssign.createNodeOffline(this.watcher, hri, SERVERNAME_A);
1587       // Trigger a transition event
1588       ZKAssign.transitionNodeOpening(this.watcher, hri, SERVERNAME_A);
1589       long startTime = EnvironmentEdgeManager.currentTime();
1590       while (!zkEventProcessed.get()) {
1591         assertTrue("Timed out in waiting for ZK event to be processed",
1592           EnvironmentEdgeManager.currentTime() - startTime < 30000);
1593         Threads.sleepWithoutInterrupt(100);
1594       }
1595       assertFalse(am.getRegionStates().isRegionInTransition(hri));
1596     } finally {
1597       am.shutdown();
1598     }
1599   }
1600 
1601   /**
1602    * If a table is deleted, we should not be able to balance it anymore.
1603    * Otherwise, the region will be brought back.
1604    * @throws Exception
1605    */
1606   @Test (timeout=180000)
1607   public void testBalanceRegionOfDeletedTable() throws Exception {
1608     AssignmentManager am = new AssignmentManager(this.server, this.serverManager,
1609       balancer, null, null, master.getTableLockManager());
1610     RegionStates regionStates = am.getRegionStates();
1611     HRegionInfo hri = REGIONINFO;
1612     regionStates.createRegionState(hri);
1613     assertFalse(regionStates.isRegionInTransition(hri));
1614     RegionPlan plan = new RegionPlan(hri, SERVERNAME_A, SERVERNAME_B);
1615     // Fake table is deleted
1616     regionStates.tableDeleted(hri.getTable());
1617     am.balance(plan);
1618     assertFalse("The region should not in transition",
1619       regionStates.isRegionInTransition(hri));
1620     am.shutdown();
1621   }
1622 
1623   /**
1624    * Tests an on-the-fly RPC that was scheduled for the earlier RS on the same port
1625    * for openRegion. AM should assign this somewhere else. (HBASE-9721)
1626    */
1627   @SuppressWarnings("unchecked")
1628   @Test (timeout=180000)
1629   public void testOpenCloseRegionRPCIntendedForPreviousServer() throws Exception {
1630     Mockito.when(this.serverManager.sendRegionOpen(Mockito.eq(SERVERNAME_B), Mockito.eq(REGIONINFO),
1631       Mockito.anyInt(), (List<ServerName>)Mockito.any()))
1632       .thenThrow(new DoNotRetryIOException());
1633     this.server.getConfiguration().setInt("hbase.assignment.maximum.attempts", 100);
1634 
1635     HRegionInfo hri = REGIONINFO;
1636     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(
1637       server.getConfiguration());
1638     // Create an AM.
1639     AssignmentManager am = new AssignmentManager(this.server,
1640       this.serverManager, balancer, null, null, master.getTableLockManager());
1641     RegionStates regionStates = am.getRegionStates();
1642     try {
1643       am.regionPlans.put(REGIONINFO.getEncodedName(),
1644         new RegionPlan(REGIONINFO, null, SERVERNAME_B));
1645 
1646       // Should fail once, but succeed on the second attempt for the SERVERNAME_A
1647       am.assign(hri, true, false);
1648     } finally {
1649       assertEquals(SERVERNAME_A, regionStates.getRegionState(REGIONINFO).getServerName());
1650       am.shutdown();
1651     }
1652   }
1653 
1654   /**
1655    * Tests close region call on a region server that is aborting
1656    */
1657   @Test (timeout=180000)
1658   public void testCloseRegionOnAbortingRS() throws Exception {
1659     this.server.getConfiguration().setInt("hbase.assignment.maximum.attempts", 2);
1660 
1661     HRegionInfo hri = REGIONINFO;
1662     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(
1663       server.getConfiguration());
1664     // Create an AM.
1665     AssignmentManager am = new AssignmentManager(this.server,
1666       this.serverManager, balancer, null, null, master.getTableLockManager());
1667     RegionStates regionStates = am.getRegionStates();
1668 
1669     regionStates.createRegionState(hri, State.OPEN, SERVERNAME_B, SERVERNAME_B);
1670 
1671     // mock aborting region server
1672     Mockito.when(this.serverManager.sendRegionClose(Mockito.eq(SERVERNAME_B), Mockito.eq(REGIONINFO),
1673       Mockito.anyInt(), (ServerName)Mockito.any(), Mockito.anyBoolean()))
1674       .thenThrow(new RegionServerAbortedException(""));
1675 
1676     // try to unassign the region
1677     am.unassign(hri);
1678 
1679     // assert that the we have FAILED_CLOSE for region state
1680     assertEquals(State.FAILED_CLOSE, regionStates.getRegionState(REGIONINFO).getState());
1681     assertEquals(SERVERNAME_B, regionStates.getRegionState(REGIONINFO).getServerName());
1682 
1683     am.shutdown();
1684   }
1685 
1686   /**
1687    * Tests close region call on a region server that is not in onlineServer list
1688    */
1689   @Test (timeout=180000)
1690   public void testCloseRegionOnServerNotOnline() throws Exception {
1691     this.server.getConfiguration().setInt("hbase.assignment.maximum.attempts", 2);
1692 
1693     HRegionInfo hri = REGIONINFO;
1694     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(
1695       server.getConfiguration());
1696     // Create an AM.
1697     AssignmentManager am = new AssignmentManager(this.server,
1698       this.serverManager, balancer, null, null, master.getTableLockManager()) {
1699       @Override
1700       protected boolean wasRegionOnDeadServerByMeta(HRegionInfo region, ServerName sn) {
1701         return true;
1702       };
1703     };
1704     RegionStates regionStates = am.getRegionStates();
1705 
1706     regionStates.createRegionState(hri, State.OPEN, SERVERNAME_B, SERVERNAME_B);
1707 
1708     // mock that RS is expired, but not processed
1709     Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B))
1710       .thenReturn(false);
1711 
1712     // try to unassign the region
1713     am.unassign(hri);
1714 
1715     // assert that the we have OFFLINE
1716     assertEquals(State.OFFLINE, regionStates.getRegionState(REGIONINFO).getState());
1717 
1718     // try to assign the region before SSH
1719     am.regionPlans.put(REGIONINFO.getEncodedName(),
1720       new RegionPlan(REGIONINFO, null, SERVERNAME_A));
1721     am.assign(hri, true, false);
1722 
1723     // assert that the we still have OFFLINE
1724     assertEquals(State.OFFLINE, regionStates.getRegionState(REGIONINFO).getState());
1725 
1726     am.shutdown();
1727   }
1728 }