1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertNotEquals;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertNull;
25 import static org.junit.Assert.assertTrue;
26 import static org.junit.Assert.fail;
27
28 import java.io.IOException;
29 import java.util.ArrayList;
30 import java.util.Arrays;
31 import java.util.HashMap;
32 import java.util.List;
33 import java.util.Map;
34 import java.util.Set;
35 import java.util.concurrent.atomic.AtomicBoolean;
36 import java.util.concurrent.atomic.AtomicInteger;
37
38 import com.google.common.collect.Lists;
39 import com.google.common.collect.Maps;
40 import org.apache.hadoop.conf.Configuration;
41 import org.apache.hadoop.fs.FileSystem;
42 import org.apache.hadoop.fs.Path;
43 import org.apache.hadoop.hbase.CoordinatedStateManager;
44 import org.apache.hadoop.hbase.HBaseTestingUtility;
45 import org.apache.hadoop.hbase.HColumnDescriptor;
46 import org.apache.hadoop.hbase.HConstants;
47 import org.apache.hadoop.hbase.HRegionInfo;
48 import org.apache.hadoop.hbase.HTableDescriptor;
49 import org.apache.hadoop.hbase.testclassification.MediumTests;
50 import org.apache.hadoop.hbase.MetaTableAccessor;
51 import org.apache.hadoop.hbase.MiniHBaseCluster;
52 import org.apache.hadoop.hbase.MiniHBaseCluster.MiniHBaseClusterRegionServer;
53 import org.apache.hadoop.hbase.ServerLoad;
54 import org.apache.hadoop.hbase.ServerName;
55 import org.apache.hadoop.hbase.TableName;
56 import org.apache.hadoop.hbase.UnknownRegionException;
57 import org.apache.hadoop.hbase.Waiter;
58 import org.apache.hadoop.hbase.client.Admin;
59 import org.apache.hadoop.hbase.client.HBaseAdmin;
60 import org.apache.hadoop.hbase.client.HTable;
61 import org.apache.hadoop.hbase.client.Result;
62 import org.apache.hadoop.hbase.client.Table;
63 import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
64 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
65 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
66 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
67 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
68 import org.apache.hadoop.hbase.coprocessor.RegionObserver;
69 import org.apache.hadoop.hbase.executor.EventType;
70 import org.apache.hadoop.hbase.master.RegionState.State;
71 import org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer;
72 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
73 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
74 import org.apache.hadoop.hbase.regionserver.HRegionServer;
75 import org.apache.hadoop.hbase.util.Bytes;
76 import org.apache.hadoop.hbase.util.ConfigUtil;
77 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
78 import org.apache.hadoop.hbase.util.FSUtils;
79 import org.apache.hadoop.hbase.util.JVMClusterUtil;
80 import org.apache.hadoop.hbase.util.Threads;
81 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
82 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
83 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
84 import org.apache.zookeeper.KeeperException;
85 import org.junit.AfterClass;
86 import org.junit.BeforeClass;
87 import org.junit.Test;
88 import org.junit.experimental.categories.Category;
89
90
91
92
93
94 @Category(MediumTests.class)
95 @SuppressWarnings("deprecation")
96 public class TestAssignmentManagerOnCluster {
97 private final static byte[] FAMILY = Bytes.toBytes("FAMILY");
98 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
99 final static Configuration conf = TEST_UTIL.getConfiguration();
100 private static HBaseAdmin admin;
101
102 static void setupOnce() throws Exception {
103
104 conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
105 MyLoadBalancer.class, LoadBalancer.class);
106 conf.setClass(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
107 MyRegionObserver.class, RegionObserver.class);
108
109 conf.setInt("hbase.assignment.maximum.attempts", 3);
110
111 conf.set("hbase.balancer.tablesOnMaster", "hbase:meta");
112 conf.setInt("hbase.master.maximum.ping.server.attempts", 3);
113 conf.setInt("hbase.master.ping.server.retry.sleep.interval", 1);
114
115 TEST_UTIL.startMiniCluster(1, 4, null, MyMaster.class, MyRegionServer.class);
116 admin = TEST_UTIL.getHBaseAdmin();
117 }
118
119 @BeforeClass
120 public static void setUpBeforeClass() throws Exception {
121
122 conf.setBoolean("hbase.assignment.usezk", true);
123 setupOnce();
124 }
125
126 @AfterClass
127 public static void tearDownAfterClass() throws Exception {
128 TEST_UTIL.shutdownMiniCluster();
129 }
130
131
132
133
134 @Test (timeout=180000)
135 public void testRestartMetaRegionServer() throws Exception {
136 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
137 boolean stoppedARegionServer = false;
138 try {
139 HMaster master = cluster.getMaster();
140 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
141 ServerName metaServerName = regionStates.getRegionServerOfRegion(
142 HRegionInfo.FIRST_META_REGIONINFO);
143 if (master.getServerName().equals(metaServerName) || metaServerName == null
144 || !metaServerName.equals(cluster.getServerHoldingMeta())) {
145
146 metaServerName = cluster.getLiveRegionServerThreads()
147 .get(0).getRegionServer().getServerName();
148 master.move(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(),
149 Bytes.toBytes(metaServerName.getServerName()));
150 master.assignmentManager.waitUntilNoRegionsInTransition(60000);
151 }
152 RegionState metaState =
153 MetaTableLocator.getMetaRegionState(master.getZooKeeper());
154 assertEquals("Meta should be not in transition",
155 metaState.getState(), RegionState.State.OPEN);
156 assertNotEquals("Meta should be moved off master",
157 metaServerName, master.getServerName());
158 cluster.killRegionServer(metaServerName);
159 stoppedARegionServer = true;
160 cluster.waitForRegionServerToStop(metaServerName, 60000);
161
162 final ServerManager serverManager = master.getServerManager();
163 TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {
164 @Override
165 public boolean evaluate() throws Exception {
166 return !serverManager.areDeadServersInProgress();
167 }
168 });
169
170
171 assertTrue("Meta should be assigned",
172 regionStates.isRegionOnline(HRegionInfo.FIRST_META_REGIONINFO));
173
174 metaState = MetaTableLocator.getMetaRegionState(master.getZooKeeper());
175 assertEquals("Meta should be not in transition",
176 metaState.getState(), RegionState.State.OPEN);
177 assertEquals("Meta should be assigned", metaState.getServerName(),
178 regionStates.getRegionServerOfRegion(HRegionInfo.FIRST_META_REGIONINFO));
179 assertNotEquals("Meta should be assigned on a different server",
180 metaState.getServerName(), metaServerName);
181 } finally {
182 if (stoppedARegionServer) {
183 cluster.startRegionServer();
184 }
185 }
186 }
187
188
189
190
191 @Test (timeout=60000)
192 public void testAssignRegion() throws Exception {
193 String table = "testAssignRegion";
194 try {
195 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(table));
196 desc.addFamily(new HColumnDescriptor(FAMILY));
197 admin.createTable(desc);
198
199 Table meta = new HTable(conf, TableName.META_TABLE_NAME);
200 HRegionInfo hri = new HRegionInfo(
201 desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
202 MetaTableAccessor.addRegionToMeta(meta, hri);
203
204 HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
205 AssignmentManager am = master.getAssignmentManager();
206 TEST_UTIL.assignRegion(hri);
207
208 RegionStates regionStates = am.getRegionStates();
209 ServerName serverName = regionStates.getRegionServerOfRegion(hri);
210 TEST_UTIL.assertRegionOnServer(hri, serverName, 6000);
211
212
213
214 RegionState oldState = regionStates.getRegionState(hri);
215 TEST_UTIL.getHBaseAdmin().assign(hri.getRegionName());
216 master.getAssignmentManager().waitForAssignment(hri);
217 RegionState newState = regionStates.getRegionState(hri);
218 assertTrue(newState.isOpened()
219 && newState.getStamp() != oldState.getStamp());
220 } finally {
221 TEST_UTIL.deleteTable(Bytes.toBytes(table));
222 }
223 }
224
225
226 @Test (timeout=60000)
227 public void testAssignRegionBySSH() throws Exception {
228 if (!conf.getBoolean("hbase.assignment.usezk", true)) {
229 return;
230 }
231 String table = "testAssignRegionBySSH";
232 MyMaster master = (MyMaster) TEST_UTIL.getHBaseCluster().getMaster();
233 try {
234 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(table));
235 desc.addFamily(new HColumnDescriptor(FAMILY));
236 admin.createTable(desc);
237
238 HTable meta = new HTable(conf, TableName.META_TABLE_NAME);
239 HRegionInfo hri = new HRegionInfo(
240 desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
241 MetaTableAccessor.addRegionToMeta(meta, hri);
242
243 MetaTableAccessor.updateRegionLocation(TEST_UTIL.getHBaseCluster().getMaster().getConnection(), hri,
244 ServerName.valueOf("example.org", 1234, System.currentTimeMillis()), 0, -1);
245 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
246 int i = TEST_UTIL.getHBaseCluster().getServerWithMeta();
247 HRegionServer rs = TEST_UTIL.getHBaseCluster().getRegionServer(i == 0 ? 1 : 0);
248
249 ServerName controlledServer = rs.getServerName();
250 master.enableSSH(false);
251 TEST_UTIL.getHBaseCluster().killRegionServer(controlledServer);
252 TEST_UTIL.getHBaseCluster().waitForRegionServerToStop(controlledServer, -1);
253 AssignmentManager am = master.getAssignmentManager();
254
255
256
257 regionStates.createRegionState(hri, State.OFFLINE, controlledServer, null);
258 am.assign(hri, true, true);
259
260 assertEquals(State.OFFLINE, regionStates.getRegionState(hri).getState());
261 assertTrue (regionStates.isRegionInTransition(hri));
262
263 master.enableSSH(true);
264 am.waitForAssignment(hri);
265 assertTrue (regionStates.getRegionState(hri).isOpened());
266 ServerName serverName = regionStates.getRegionServerOfRegion(hri);
267 TEST_UTIL.assertRegionOnlyOnServer(hri, serverName, 6000);
268 } finally {
269 if (master != null) {
270 master.enableSSH(true);
271 }
272 TEST_UTIL.deleteTable(Bytes.toBytes(table));
273 TEST_UTIL.getHBaseCluster().startRegionServer();
274 }
275 }
276
277
278
279
280 @Test (timeout=120000)
281 public void testAssignRegionOnRestartedServer() throws Exception {
282 String table = "testAssignRegionOnRestartedServer";
283 TEST_UTIL.getMiniHBaseCluster().getConf().setInt("hbase.assignment.maximum.attempts", 20);
284 TEST_UTIL.getMiniHBaseCluster().stopMaster(0);
285 TEST_UTIL.getMiniHBaseCluster().startMaster();
286
287 ServerName deadServer = null;
288 HMaster master = null;
289 try {
290 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(table));
291 desc.addFamily(new HColumnDescriptor(FAMILY));
292 admin.createTable(desc);
293
294 Table meta = new HTable(conf, TableName.META_TABLE_NAME);
295 final HRegionInfo hri = new HRegionInfo(
296 desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
297 MetaTableAccessor.addRegionToMeta(meta, hri);
298
299 master = TEST_UTIL.getHBaseCluster().getMaster();
300 Set<ServerName> onlineServers = master.serverManager.getOnlineServers().keySet();
301 assertFalse("There should be some servers online", onlineServers.isEmpty());
302
303
304 ServerName destServer = onlineServers.iterator().next();
305
306
307 deadServer = ServerName.valueOf(destServer.getHostname(),
308 destServer.getPort(), destServer.getStartcode() - 100L);
309 master.serverManager.recordNewServerWithLock(deadServer, ServerLoad.EMPTY_SERVERLOAD);
310
311 final AssignmentManager am = master.getAssignmentManager();
312 RegionPlan plan = new RegionPlan(hri, null, deadServer);
313 am.addPlan(hri.getEncodedName(), plan);
314 TEST_UTIL.assignRegion(hri);
315
316 int version = ZKAssign.transitionNode(master.getZooKeeper(), hri,
317 destServer, EventType.M_ZK_REGION_OFFLINE,
318 EventType.RS_ZK_REGION_OPENING, 0);
319 assertEquals("TansitionNode should fail", -1, version);
320
321 TEST_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
322 @Override
323 public boolean evaluate() throws Exception {
324 return ! am.getRegionStates().isRegionInTransition(hri);
325 }
326 });
327
328 assertFalse("Region should be assigned", am.getRegionStates().isRegionInTransition(hri));
329 } finally {
330 if (deadServer != null) {
331 master.serverManager.expireServer(deadServer);
332 }
333
334 TEST_UTIL.deleteTable(Bytes.toBytes(table));
335
336
337 TEST_UTIL.getMiniHBaseCluster().getConf().setInt("hbase.assignment.maximum.attempts", 3);
338 ServerName masterServerName = TEST_UTIL.getMiniHBaseCluster().getMaster().getServerName();
339 TEST_UTIL.getMiniHBaseCluster().stopMaster(masterServerName);
340 TEST_UTIL.getMiniHBaseCluster().startMaster();
341
342 while (TEST_UTIL.getMiniHBaseCluster().getMaster() == null ||
343 !TEST_UTIL.getMiniHBaseCluster().getMaster().isInitialized()) {
344 Threads.sleep(1);
345 }
346 }
347 }
348
349
350
351
352 @Test (timeout=60000)
353 public void testOfflineRegion() throws Exception {
354 TableName table =
355 TableName.valueOf("testOfflineRegion");
356 try {
357 HRegionInfo hri = createTableAndGetOneRegion(table);
358
359 RegionStates regionStates = TEST_UTIL.getHBaseCluster().
360 getMaster().getAssignmentManager().getRegionStates();
361 ServerName serverName = regionStates.getRegionServerOfRegion(hri);
362 TEST_UTIL.assertRegionOnServer(hri, serverName, 6000);
363 admin.offline(hri.getRegionName());
364
365 long timeoutTime = System.currentTimeMillis() + 800;
366 while (true) {
367 if (regionStates.getRegionByStateOfTable(table)
368 .get(RegionState.State.OFFLINE).contains(hri))
369 break;
370 long now = System.currentTimeMillis();
371 if (now > timeoutTime) {
372 fail("Failed to offline the region in time");
373 break;
374 }
375 Thread.sleep(10);
376 }
377 RegionState regionState = regionStates.getRegionState(hri);
378 assertTrue(regionState.isOffline());
379 } finally {
380 TEST_UTIL.deleteTable(table);
381 }
382 }
383
384
385
386
387 @Test (timeout=50000)
388 public void testMoveRegion() throws Exception {
389 TableName table =
390 TableName.valueOf("testMoveRegion");
391 try {
392 HRegionInfo hri = createTableAndGetOneRegion(table);
393
394 HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
395 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
396 ServerName serverName = regionStates.getRegionServerOfRegion(hri);
397 ServerManager serverManager = master.getServerManager();
398 ServerName destServerName = null;
399 List<JVMClusterUtil.RegionServerThread> regionServers =
400 TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads();
401 for (JVMClusterUtil.RegionServerThread regionServer: regionServers) {
402 HRegionServer destServer = regionServer.getRegionServer();
403 destServerName = destServer.getServerName();
404 if (!destServerName.equals(serverName)
405 && serverManager.isServerOnline(destServerName)) {
406 break;
407 }
408 }
409 assertTrue(destServerName != null
410 && !destServerName.equals(serverName));
411 TEST_UTIL.getHBaseAdmin().move(hri.getEncodedNameAsBytes(),
412 Bytes.toBytes(destServerName.getServerName()));
413
414 long timeoutTime = System.currentTimeMillis() + 30000;
415 while (true) {
416 ServerName sn = regionStates.getRegionServerOfRegion(hri);
417 if (sn != null && sn.equals(destServerName)) {
418 TEST_UTIL.assertRegionOnServer(hri, sn, 6000);
419 break;
420 }
421 long now = System.currentTimeMillis();
422 if (now > timeoutTime) {
423 fail("Failed to move the region in time: "
424 + regionStates.getRegionState(hri));
425 }
426 regionStates.waitForUpdate(50);
427 }
428
429 } finally {
430 TEST_UTIL.deleteTable(table);
431 }
432 }
433
434
435
436
437
438
439 @Test (timeout=50000)
440 public void testMoveRegionOfDeletedTable() throws Exception {
441 TableName table =
442 TableName.valueOf("testMoveRegionOfDeletedTable");
443 Admin admin = TEST_UTIL.getHBaseAdmin();
444 try {
445 HRegionInfo hri = createTableAndGetOneRegion(table);
446
447 HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
448 AssignmentManager am = master.getAssignmentManager();
449 RegionStates regionStates = am.getRegionStates();
450 ServerName serverName = regionStates.getRegionServerOfRegion(hri);
451 ServerName destServerName = null;
452 for (int i = 0; i < 3; i++) {
453 HRegionServer destServer = TEST_UTIL.getHBaseCluster().getRegionServer(i);
454 if (!destServer.getServerName().equals(serverName)) {
455 destServerName = destServer.getServerName();
456 break;
457 }
458 }
459 assertTrue(destServerName != null
460 && !destServerName.equals(serverName));
461
462 TEST_UTIL.deleteTable(table);
463
464 try {
465 admin.move(hri.getEncodedNameAsBytes(),
466 Bytes.toBytes(destServerName.getServerName()));
467 fail("We should not find the region");
468 } catch (IOException ioe) {
469 assertTrue(ioe instanceof UnknownRegionException);
470 }
471
472 am.balance(new RegionPlan(hri, serverName, destServerName));
473 assertFalse("The region should not be in transition",
474 regionStates.isRegionInTransition(hri));
475 } finally {
476 if (admin.tableExists(table)) {
477 TEST_UTIL.deleteTable(table);
478 }
479 }
480 }
481
482 HRegionInfo createTableAndGetOneRegion(
483 final TableName tableName) throws IOException, InterruptedException {
484 HTableDescriptor desc = new HTableDescriptor(tableName);
485 desc.addFamily(new HColumnDescriptor(FAMILY));
486 admin.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), 5);
487
488
489 HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
490 long timeoutTime = System.currentTimeMillis() + 1000;
491 while (true) {
492 List<HRegionInfo> regions = master.getAssignmentManager().
493 getRegionStates().getRegionsOfTable(tableName);
494 if (regions.size() > 3) {
495 return regions.get(2);
496 }
497 long now = System.currentTimeMillis();
498 if (now > timeoutTime) {
499 fail("Could not find an online region");
500 }
501 Thread.sleep(10);
502 }
503 }
504
505
506
507
508
509
510
511 @Test (timeout=60000)
512 public void testForceAssignWhileClosing() throws Exception {
513 String table = "testForceAssignWhileClosing";
514 try {
515 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(table));
516 desc.addFamily(new HColumnDescriptor(FAMILY));
517 admin.createTable(desc);
518
519 Table meta = new HTable(conf, TableName.META_TABLE_NAME);
520 HRegionInfo hri = new HRegionInfo(
521 desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
522 MetaTableAccessor.addRegionToMeta(meta, hri);
523
524 HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
525 AssignmentManager am = master.getAssignmentManager();
526 assertTrue(TEST_UTIL.assignRegion(hri));
527
528 ServerName sn = am.getRegionStates().getRegionServerOfRegion(hri);
529 TEST_UTIL.assertRegionOnServer(hri, sn, 6000);
530 MyRegionObserver.preCloseEnabled.set(true);
531 am.unassign(hri);
532 RegionState state = am.getRegionStates().getRegionState(hri);
533 assertEquals(RegionState.State.FAILED_CLOSE, state.getState());
534
535 MyRegionObserver.preCloseEnabled.set(false);
536 am.unassign(hri, true);
537
538
539
540
541 am.assign(hri, true, true);
542
543
544 am.waitOnRegionToClearRegionsInTransition(hri);
545 assertTrue(am.waitForAssignment(hri));
546
547 ServerName serverName = master.getAssignmentManager().
548 getRegionStates().getRegionServerOfRegion(hri);
549 TEST_UTIL.assertRegionOnlyOnServer(hri, serverName, 200);
550 } finally {
551 MyRegionObserver.preCloseEnabled.set(false);
552 TEST_UTIL.deleteTable(Bytes.toBytes(table));
553 }
554 }
555
556
557
558
559 @Test (timeout=60000)
560 public void testCloseFailed() throws Exception {
561 String table = "testCloseFailed";
562 try {
563 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(table));
564 desc.addFamily(new HColumnDescriptor(FAMILY));
565 admin.createTable(desc);
566
567 Table meta = new HTable(conf, TableName.META_TABLE_NAME);
568 HRegionInfo hri = new HRegionInfo(
569 desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
570 MetaTableAccessor.addRegionToMeta(meta, hri);
571
572 HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
573 AssignmentManager am = master.getAssignmentManager();
574 assertTrue(TEST_UTIL.assignRegion(hri));
575 ServerName sn = am.getRegionStates().getRegionServerOfRegion(hri);
576 TEST_UTIL.assertRegionOnServer(hri, sn, 6000);
577
578 MyRegionObserver.preCloseEnabled.set(true);
579 am.unassign(hri);
580 RegionState state = am.getRegionStates().getRegionState(hri);
581 assertEquals(RegionState.State.FAILED_CLOSE, state.getState());
582
583 MyRegionObserver.preCloseEnabled.set(false);
584 am.unassign(hri, true);
585
586
587
588 am.waitOnRegionToClearRegionsInTransition(hri);
589
590
591 assertTrue(am.waitForAssignment(hri));
592 ServerName serverName = master.getAssignmentManager().
593 getRegionStates().getRegionServerOfRegion(hri);
594 TEST_UTIL.assertRegionOnServer(hri, serverName, 6000);
595 } finally {
596 MyRegionObserver.preCloseEnabled.set(false);
597 TEST_UTIL.deleteTable(Bytes.toBytes(table));
598 }
599 }
600
601
602
603
604 @Test (timeout=60000)
605 public void testOpenFailed() throws Exception {
606 String table = "testOpenFailed";
607 try {
608 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(table));
609 desc.addFamily(new HColumnDescriptor(FAMILY));
610 admin.createTable(desc);
611
612 Table meta = new HTable(conf, TableName.META_TABLE_NAME);
613 HRegionInfo hri = new HRegionInfo(
614 desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
615 MetaTableAccessor.addRegionToMeta(meta, hri);
616
617 MyLoadBalancer.controledRegion = hri;
618
619 HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
620 AssignmentManager am = master.getAssignmentManager();
621 assertFalse(TEST_UTIL.assignRegion(hri));
622
623 RegionState state = am.getRegionStates().getRegionState(hri);
624 assertEquals(RegionState.State.FAILED_OPEN, state.getState());
625
626 assertNull(state.getServerName());
627
628 MyLoadBalancer.controledRegion = null;
629 assertTrue(TEST_UTIL.assignRegion(hri));
630
631 ServerName serverName = master.getAssignmentManager().
632 getRegionStates().getRegionServerOfRegion(hri);
633 TEST_UTIL.assertRegionOnServer(hri, serverName, 6000);
634 } finally {
635 MyLoadBalancer.controledRegion = null;
636 TEST_UTIL.deleteTable(Bytes.toBytes(table));
637 }
638 }
639
640
641
642
643 @Test (timeout=60000)
644 public void testRoundRobinAssignmentFailed() throws Exception {
645 TableName tableName = TableName.valueOf("testRoundRobinAssignmentFailed");
646 try {
647 HTableDescriptor desc = new HTableDescriptor(tableName);
648 desc.addFamily(new HColumnDescriptor(FAMILY));
649 admin.createTable(desc);
650
651 Table meta = admin.getConnection().getTable(TableName.META_TABLE_NAME);
652 HRegionInfo hri = new HRegionInfo(
653 desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
654 MetaTableAccessor.addRegionToMeta(meta, hri);
655
656 MyLoadBalancer.controledRegion = hri;
657
658 HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
659 AssignmentManager am = master.getAssignmentManager();
660
661
662 am.assign(Arrays.asList(hri));
663
664
665
666 assertFalse(am.waitForAssignment(hri));
667 RegionState state = am.getRegionStates().getRegionState(hri);
668 assertEquals(RegionState.State.FAILED_OPEN, state.getState());
669
670 assertNull(state.getServerName());
671
672
673 MyLoadBalancer.controledRegion = null;
674 am.assign(Arrays.asList(hri));
675 assertTrue(am.waitForAssignment(hri));
676
677 ServerName serverName = master.getAssignmentManager().
678 getRegionStates().getRegionServerOfRegion(hri);
679 TEST_UTIL.assertRegionOnServer(hri, serverName, 200);
680 } finally {
681 MyLoadBalancer.controledRegion = null;
682 TEST_UTIL.deleteTable(tableName);
683 }
684 }
685
686
687
688
689 @Test (timeout=60000)
690 public void testRetainAssignmentFailed() throws Exception {
691 TableName tableName = TableName.valueOf("testRetainAssignmentFailed");
692 try {
693 HTableDescriptor desc = new HTableDescriptor(tableName);
694 desc.addFamily(new HColumnDescriptor(FAMILY));
695 admin.createTable(desc);
696
697 Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
698 HRegionInfo hri = new HRegionInfo(
699 desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
700 MetaTableAccessor.addRegionToMeta(meta, hri);
701
702 MyLoadBalancer.controledRegion = hri;
703
704 HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
705 AssignmentManager am = master.getAssignmentManager();
706
707 Map<HRegionInfo, ServerName> regions = new HashMap<HRegionInfo, ServerName>();
708 ServerName dest = TEST_UTIL.getHBaseCluster().getRegionServer(0).getServerName();
709 regions.put(hri, dest);
710
711
712 am.assign(regions);
713
714
715
716 assertFalse(am.waitForAssignment(hri));
717 RegionState state = am.getRegionStates().getRegionState(hri);
718 assertEquals(RegionState.State.FAILED_OPEN, state.getState());
719
720 assertNull(state.getServerName());
721
722
723 MyLoadBalancer.controledRegion = null;
724 am.assign(regions);
725 assertTrue(am.waitForAssignment(hri));
726
727 ServerName serverName = master.getAssignmentManager().
728 getRegionStates().getRegionServerOfRegion(hri);
729 TEST_UTIL.assertRegionOnServer(hri, serverName, 200);
730
731
732 assertEquals(serverName, dest);
733 } finally {
734 MyLoadBalancer.controledRegion = null;
735 TEST_UTIL.deleteTable(tableName);
736 }
737 }
738
739
740
741
742 @Test (timeout=60000)
743 public void testOpenFailedUnrecoverable() throws Exception {
744 TableName table =
745 TableName.valueOf("testOpenFailedUnrecoverable");
746 try {
747 HTableDescriptor desc = new HTableDescriptor(table);
748 desc.addFamily(new HColumnDescriptor(FAMILY));
749 admin.createTable(desc);
750
751 Table meta = new HTable(conf, TableName.META_TABLE_NAME);
752 HRegionInfo hri = new HRegionInfo(
753 desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
754 MetaTableAccessor.addRegionToMeta(meta, hri);
755
756 FileSystem fs = FileSystem.get(conf);
757 Path tableDir= FSUtils.getTableDir(FSUtils.getRootDir(conf), table);
758 Path regionDir = new Path(tableDir, hri.getEncodedName());
759
760
761 fs.create(regionDir, true);
762
763 HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
764 AssignmentManager am = master.getAssignmentManager();
765 assertFalse(TEST_UTIL.assignRegion(hri));
766
767 RegionState state = am.getRegionStates().getRegionState(hri);
768 assertEquals(RegionState.State.FAILED_OPEN, state.getState());
769
770
771
772 assertNotNull(state.getServerName());
773
774
775 fs.delete(regionDir, true);
776 assertTrue(TEST_UTIL.assignRegion(hri));
777
778 ServerName serverName = master.getAssignmentManager().
779 getRegionStates().getRegionServerOfRegion(hri);
780 TEST_UTIL.assertRegionOnServer(hri, serverName, 6000);
781 } finally {
782 TEST_UTIL.deleteTable(table);
783 }
784 }
785
786 @Test (timeout=60000)
787 public void testSSHWhenDisablingTableRegionsInOpeningOrPendingOpenState() throws Exception {
788 final TableName table =
789 TableName.valueOf
790 ("testSSHWhenDisablingTableRegionsInOpeningOrPendingOpenState");
791 AssignmentManager am = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager();
792 HRegionInfo hri = null;
793 ServerName serverName = null;
794 try {
795 hri = createTableAndGetOneRegion(table);
796 serverName = am.getRegionStates().getRegionServerOfRegion(hri);
797 ServerName destServerName = null;
798 HRegionServer destServer = null;
799 for (int i = 0; i < 3; i++) {
800 destServer = TEST_UTIL.getHBaseCluster().getRegionServer(i);
801 if (!destServer.getServerName().equals(serverName)) {
802 destServerName = destServer.getServerName();
803 break;
804 }
805 }
806 am.regionOffline(hri);
807 ZooKeeperWatcher zkw = TEST_UTIL.getHBaseCluster().getMaster().getZooKeeper();
808 am.getRegionStates().updateRegionState(hri, State.PENDING_OPEN, destServerName);
809 if (ConfigUtil.useZKForAssignment(conf)) {
810 ZKAssign.createNodeOffline(zkw, hri, destServerName);
811 ZKAssign.transitionNodeOpening(zkw, hri, destServerName);
812
813
814 long timeoutTime = System.currentTimeMillis() + 20000;
815 while (!am.getRegionStates().isRegionInTransition(hri)) {
816 assertTrue("Failed to process ZK opening event in time",
817 System.currentTimeMillis() < timeoutTime);
818 Thread.sleep(100);
819 }
820 }
821
822 am.getTableStateManager().setTableState(table, ZooKeeperProtos.Table.State.DISABLING);
823 List<HRegionInfo> toAssignRegions = am.cleanOutCrashedServerReferences(destServerName);
824 assertTrue("Regions to be assigned should be empty.", toAssignRegions.isEmpty());
825 assertTrue("Regions to be assigned should be empty.", am.getRegionStates()
826 .getRegionState(hri).isOffline());
827 } finally {
828 if (hri != null && serverName != null) {
829 am.regionOnline(hri, serverName);
830 }
831 am.getTableStateManager().setTableState(table, ZooKeeperProtos.Table.State.DISABLED);
832 TEST_UTIL.deleteTable(table);
833 }
834 }
835
836
837
838
839 @Test (timeout=60000)
840 public void testCloseHang() throws Exception {
841 String table = "testCloseHang";
842 try {
843 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(table));
844 desc.addFamily(new HColumnDescriptor(FAMILY));
845 admin.createTable(desc);
846
847 Table meta = new HTable(conf, TableName.META_TABLE_NAME);
848 HRegionInfo hri = new HRegionInfo(
849 desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
850 MetaTableAccessor.addRegionToMeta(meta, hri);
851
852 HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
853 AssignmentManager am = master.getAssignmentManager();
854 assertTrue(TEST_UTIL.assignRegion(hri));
855 ServerName sn = am.getRegionStates().getRegionServerOfRegion(hri);
856 TEST_UTIL.assertRegionOnServer(hri, sn, 6000);
857
858 MyRegionObserver.postCloseEnabled.set(true);
859 am.unassign(hri);
860
861
862
863
864 am.server.getConfiguration().setLong(
865 AssignmentManager.ALREADY_IN_TRANSITION_WAITTIME, 1000);
866 am.unassign(hri, true);
867 RegionState state = am.getRegionStates().getRegionState(hri);
868 assertEquals(RegionState.State.FAILED_CLOSE, state.getState());
869
870
871
872 MyRegionObserver.postCloseEnabled.set(false);
873
874
875
876 am.waitOnRegionToClearRegionsInTransition(hri);
877
878
879 assertTrue(am.waitForAssignment(hri));
880 ServerName serverName = master.getAssignmentManager().
881 getRegionStates().getRegionServerOfRegion(hri);
882 TEST_UTIL.assertRegionOnServer(hri, serverName, 6000);
883 } finally {
884 MyRegionObserver.postCloseEnabled.set(false);
885 TEST_UTIL.deleteTable(Bytes.toBytes(table));
886 }
887 }
888
889
890
891
892 @Test (timeout=60000)
893 public void testOpenCloseRacing() throws Exception {
894 String table = "testOpenCloseRacing";
895 try {
896 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(table));
897 desc.addFamily(new HColumnDescriptor(FAMILY));
898 admin.createTable(desc);
899
900 Table meta = new HTable(conf, TableName.META_TABLE_NAME);
901 HRegionInfo hri = new HRegionInfo(
902 desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
903 MetaTableAccessor.addRegionToMeta(meta, hri);
904 meta.close();
905
906 MyRegionObserver.postOpenEnabled.set(true);
907 MyRegionObserver.postOpenCalled = false;
908 HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
909 AssignmentManager am = master.getAssignmentManager();
910
911 am.assign(hri, true);
912 long end = EnvironmentEdgeManager.currentTime() + 20000;
913
914 while (!MyRegionObserver.postOpenCalled ) {
915 assertFalse("Timed out waiting for postOpen to be called",
916 EnvironmentEdgeManager.currentTime() > end);
917 Thread.sleep(300);
918 }
919
920
921 am.unassign(hri);
922 RegionState state = am.getRegionStates().getRegionState(hri);
923 ServerName oldServerName = state.getServerName();
924 assertTrue(state.isPendingOpenOrOpening() && oldServerName != null);
925
926
927
928
929
930 ServerName destServerName = null;
931 int numRS = TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads().size();
932 for (int i = 0; i < numRS; i++) {
933 HRegionServer destServer = TEST_UTIL.getHBaseCluster().getRegionServer(i);
934 if (!destServer.getServerName().equals(oldServerName)) {
935 destServerName = destServer.getServerName();
936 break;
937 }
938 }
939 assertNotNull(destServerName);
940 assertFalse("Region should be assigned on a new region server",
941 oldServerName.equals(destServerName));
942 List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
943 regions.add(hri);
944 am.assign(destServerName, regions);
945
946
947 MyRegionObserver.postOpenEnabled.set(false);
948
949
950 am.waitOnRegionToClearRegionsInTransition(hri);
951 assertTrue(am.waitForAssignment(hri));
952
953 ServerName serverName = master.getAssignmentManager().
954 getRegionStates().getRegionServerOfRegion(hri);
955 TEST_UTIL.assertRegionOnlyOnServer(hri, serverName, 6000);
956 } finally {
957 MyRegionObserver.postOpenEnabled.set(false);
958 TEST_UTIL.deleteTable(Bytes.toBytes(table));
959 }
960 }
961
962
963
964
965 @Test (timeout=60000)
966 public void testAssignRacingWithSSH() throws Exception {
967 String table = "testAssignRacingWithSSH";
968 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
969 MyMaster master = null;
970 try {
971 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(table));
972 desc.addFamily(new HColumnDescriptor(FAMILY));
973 admin.createTable(desc);
974
975 Table meta = new HTable(conf, TableName.META_TABLE_NAME);
976 HRegionInfo hri = new HRegionInfo(
977 desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
978 MetaTableAccessor.addRegionToMeta(meta, hri);
979
980
981 master = (MyMaster)cluster.getMaster();
982 AssignmentManager am = master.getAssignmentManager();
983
984 am.assign(hri, true);
985
986
987 master.enableSSH(false);
988
989
990 RegionStates regionStates = am.getRegionStates();
991 ServerName metaServer = regionStates.getRegionServerOfRegion(
992 HRegionInfo.FIRST_META_REGIONINFO);
993 while (true) {
994 assertTrue(am.waitForAssignment(hri));
995 RegionState state = regionStates.getRegionState(hri);
996 ServerName oldServerName = state.getServerName();
997 if (!ServerName.isSameHostnameAndPort(oldServerName, metaServer)) {
998
999 cluster.killRegionServer(oldServerName);
1000 cluster.waitForRegionServerToStop(oldServerName, -1);
1001 break;
1002 }
1003 int i = cluster.getServerWithMeta();
1004 HRegionServer rs = cluster.getRegionServer(i == 0 ? 1 : 0);
1005 oldServerName = rs.getServerName();
1006 master.move(hri.getEncodedNameAsBytes(),
1007 Bytes.toBytes(oldServerName.getServerName()));
1008 }
1009
1010
1011 am.assign(hri, true, true);
1012 RegionState state = regionStates.getRegionState(hri);
1013 assertTrue(state.isFailedClose());
1014
1015
1016 am.unassign(hri, true);
1017 assertTrue(state.isFailedClose());
1018
1019
1020 master.enableSSH(true);
1021
1022
1023
1024 am.waitOnRegionToClearRegionsInTransition(hri);
1025 assertTrue(am.waitForAssignment(hri));
1026
1027 ServerName serverName = master.getAssignmentManager().
1028 getRegionStates().getRegionServerOfRegion(hri);
1029 TEST_UTIL.assertRegionOnlyOnServer(hri, serverName, 6000);
1030 } finally {
1031 if (master != null) {
1032 master.enableSSH(true);
1033 }
1034 TEST_UTIL.deleteTable(Bytes.toBytes(table));
1035 cluster.startRegionServer();
1036 }
1037 }
1038
1039
1040
1041
1042 @Test (timeout=60000)
1043 public void testAssignDisabledRegion() throws Exception {
1044 TableName table = TableName.valueOf("testAssignDisabledRegion");
1045 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1046 MyMaster master = null;
1047 try {
1048 HTableDescriptor desc = new HTableDescriptor(table);
1049 desc.addFamily(new HColumnDescriptor(FAMILY));
1050 admin.createTable(desc);
1051
1052 Table meta = new HTable(conf, TableName.META_TABLE_NAME);
1053 HRegionInfo hri = new HRegionInfo(
1054 desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
1055 MetaTableAccessor.addRegionToMeta(meta, hri);
1056
1057
1058 master = (MyMaster)cluster.getMaster();
1059 AssignmentManager am = master.getAssignmentManager();
1060 RegionStates regionStates = am.getRegionStates();
1061 assertTrue(TEST_UTIL.assignRegion(hri));
1062
1063
1064 admin.disableTable(table);
1065 assertTrue(regionStates.isRegionOffline(hri));
1066
1067
1068 am.assign(hri, true, true);
1069 assertTrue(regionStates.isRegionOffline(hri));
1070
1071
1072 am.unassign(hri, true);
1073 assertTrue(regionStates.isRegionOffline(hri));
1074 } finally {
1075 TEST_UTIL.deleteTable(table);
1076 }
1077 }
1078
1079
1080
1081
1082 @Test (timeout=60000)
1083 public void testAssignOfflinedRegionBySSH() throws Exception {
1084 String table = "testAssignOfflinedRegionBySSH";
1085 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1086 MyMaster master = null;
1087 try {
1088 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(table));
1089 desc.addFamily(new HColumnDescriptor(FAMILY));
1090 admin.createTable(desc);
1091
1092 Table meta = new HTable(conf, TableName.META_TABLE_NAME);
1093 HRegionInfo hri = new HRegionInfo(
1094 desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
1095 MetaTableAccessor.addRegionToMeta(meta, hri);
1096
1097
1098 master = (MyMaster)cluster.getMaster();
1099 AssignmentManager am = master.getAssignmentManager();
1100 am.assign(hri, true);
1101
1102 RegionStates regionStates = am.getRegionStates();
1103 ServerName metaServer = regionStates.getRegionServerOfRegion(
1104 HRegionInfo.FIRST_META_REGIONINFO);
1105 ServerName oldServerName = null;
1106 while (true) {
1107 assertTrue(am.waitForAssignment(hri));
1108 RegionState state = regionStates.getRegionState(hri);
1109 oldServerName = state.getServerName();
1110 if (!ServerName.isSameHostnameAndPort(oldServerName, metaServer)) {
1111
1112
1113 MyRegionServer.abortedServer = oldServerName;
1114 break;
1115 }
1116 int i = cluster.getServerWithMeta();
1117 HRegionServer rs = cluster.getRegionServer(i == 0 ? 1 : 0);
1118 oldServerName = rs.getServerName();
1119 master.move(hri.getEncodedNameAsBytes(),
1120 Bytes.toBytes(oldServerName.getServerName()));
1121 }
1122
1123
1124 assertTrue(regionStates.isRegionOnline(hri));
1125 assertEquals(oldServerName, regionStates.getRegionServerOfRegion(hri));
1126
1127
1128 cluster.killRegionServer(oldServerName);
1129 cluster.waitForRegionServerToStop(oldServerName, -1);
1130
1131 ServerManager serverManager = master.getServerManager();
1132 while (!serverManager.isServerDead(oldServerName)
1133 || serverManager.getDeadServers().areDeadServersInProgress()) {
1134 Thread.sleep(100);
1135 }
1136
1137
1138
1139 am.waitOnRegionToClearRegionsInTransition(hri);
1140 assertTrue(am.waitForAssignment(hri));
1141
1142 ServerName serverName = master.getAssignmentManager().
1143 getRegionStates().getRegionServerOfRegion(hri);
1144 TEST_UTIL.assertRegionOnlyOnServer(hri, serverName, 200);
1145 } finally {
1146 MyRegionServer.abortedServer = null;
1147 TEST_UTIL.deleteTable(Bytes.toBytes(table));
1148 cluster.startRegionServer();
1149 }
1150 }
1151
1152
1153
1154
1155 @Test (timeout=300000)
1156 public void testSSHWaitForServerToAssignRegion() throws Exception {
1157 TableName table = TableName.valueOf("testSSHWaitForServerToAssignRegion");
1158 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1159 boolean startAServer = false;
1160 try {
1161 HTableDescriptor desc = new HTableDescriptor(table);
1162 desc.addFamily(new HColumnDescriptor(FAMILY));
1163 admin.createTable(desc);
1164
1165 HMaster master = cluster.getMaster();
1166 final ServerManager serverManager = master.getServerManager();
1167 MyLoadBalancer.countRegionServers = Integer.valueOf(
1168 serverManager.countOfRegionServers());
1169 HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(table);
1170 assertNotNull("First region should be assigned", rs);
1171 final ServerName serverName = rs.getServerName();
1172
1173 int counter = MyLoadBalancer.counter.get() + 5;
1174 cluster.killRegionServer(serverName);
1175 startAServer = true;
1176 cluster.waitForRegionServerToStop(serverName, -1);
1177 while (counter > MyLoadBalancer.counter.get()) {
1178 Thread.sleep(1000);
1179 }
1180 cluster.startRegionServer();
1181 startAServer = false;
1182
1183 TEST_UTIL.waitFor(120000, 1000, new Waiter.Predicate<Exception>() {
1184 @Override
1185 public boolean evaluate() throws Exception {
1186 return serverManager.isServerDead(serverName)
1187 && !serverManager.areDeadServersInProgress();
1188 }
1189 });
1190 TEST_UTIL.waitUntilAllRegionsAssigned(table, 300000);
1191
1192 rs = TEST_UTIL.getRSForFirstRegionInTable(table);
1193 assertTrue("First region should be re-assigned to a different server",
1194 rs != null && !serverName.equals(rs.getServerName()));
1195 } finally {
1196 MyLoadBalancer.countRegionServers = null;
1197 TEST_UTIL.deleteTable(table);
1198 if (startAServer) {
1199 cluster.startRegionServer();
1200 }
1201 }
1202 }
1203
1204
1205
1206
1207 @Test (timeout=60000)
1208 public void testAssignDisabledRegionBySSH() throws Exception {
1209 String table = "testAssignDisabledRegionBySSH";
1210 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1211 MyMaster master = null;
1212 try {
1213 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(table));
1214 desc.addFamily(new HColumnDescriptor(FAMILY));
1215 admin.createTable(desc);
1216
1217 Table meta = new HTable(conf, TableName.META_TABLE_NAME);
1218 HRegionInfo hri = new HRegionInfo(
1219 desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
1220 MetaTableAccessor.addRegionToMeta(meta, hri);
1221
1222
1223 master = (MyMaster)cluster.getMaster();
1224 AssignmentManager am = master.getAssignmentManager();
1225 am.assign(hri, true);
1226
1227 RegionStates regionStates = am.getRegionStates();
1228 ServerName metaServer = regionStates.getRegionServerOfRegion(
1229 HRegionInfo.FIRST_META_REGIONINFO);
1230 ServerName oldServerName = null;
1231 while (true) {
1232 assertTrue(am.waitForAssignment(hri));
1233 RegionState state = regionStates.getRegionState(hri);
1234 oldServerName = state.getServerName();
1235 if (!ServerName.isSameHostnameAndPort(oldServerName, metaServer)) {
1236
1237
1238 MyRegionServer.abortedServer = oldServerName;
1239 break;
1240 }
1241 int i = cluster.getServerWithMeta();
1242 HRegionServer rs = cluster.getRegionServer(i == 0 ? 1 : 0);
1243 oldServerName = rs.getServerName();
1244 master.move(hri.getEncodedNameAsBytes(),
1245 Bytes.toBytes(oldServerName.getServerName()));
1246 }
1247
1248
1249 assertTrue(regionStates.isRegionOnline(hri));
1250 assertEquals(oldServerName, regionStates.getRegionServerOfRegion(hri));
1251
1252
1253 master.disableTable(hri.getTable(), HConstants.NO_NONCE, HConstants.NO_NONCE);
1254
1255
1256 cluster.killRegionServer(oldServerName);
1257 cluster.waitForRegionServerToStop(oldServerName, -1);
1258
1259 ServerManager serverManager = master.getServerManager();
1260 while (!serverManager.isServerDead(oldServerName)
1261 || serverManager.getDeadServers().areDeadServersInProgress()) {
1262 Thread.sleep(100);
1263 }
1264
1265
1266 am.waitUntilNoRegionsInTransition(60000);
1267 assertTrue(regionStates.isRegionOffline(hri));
1268 } finally {
1269 MyRegionServer.abortedServer = null;
1270 TEST_UTIL.deleteTable(Bytes.toBytes(table));
1271 cluster.startRegionServer();
1272 }
1273 }
1274
1275
1276
1277
1278 @Test(timeout = 60000)
1279 public void testReportRegionStateTransition() throws Exception {
1280 String table = "testReportRegionStateTransition";
1281 try {
1282 MyRegionServer.simulateRetry = true;
1283 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(table));
1284 desc.addFamily(new HColumnDescriptor(FAMILY));
1285 admin.createTable(desc);
1286 Table meta = new HTable(conf, TableName.META_TABLE_NAME);
1287 HRegionInfo hri =
1288 new HRegionInfo(desc.getTableName(), Bytes.toBytes("A"), Bytes.toBytes("Z"));
1289 MetaTableAccessor.addRegionToMeta(meta, hri);
1290 HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
1291 AssignmentManager am = master.getAssignmentManager();
1292 TEST_UTIL.assignRegion(hri);
1293 RegionStates regionStates = am.getRegionStates();
1294 ServerName serverName = regionStates.getRegionServerOfRegion(hri);
1295
1296 TEST_UTIL.assertRegionOnServer(hri, serverName, 6000);
1297
1298 admin.disableTable(TableName.valueOf(table));
1299 assertTrue(regionStates.isRegionOffline(hri));
1300 List<HRegionInfo> regions = TEST_UTIL.getHBaseAdmin().getOnlineRegions(serverName);
1301 assertTrue(!regions.contains(hri));
1302 } finally {
1303 MyRegionServer.simulateRetry = false;
1304 TEST_UTIL.deleteTable(Bytes.toBytes(table));
1305 }
1306 }
1307
1308
1309
1310
1311
1312 @Test(timeout = 30000)
1313 public void testUpdatesRemoteMeta() throws Exception {
1314
1315 if (conf.getBoolean("hbase.assignment.usezk", true)) {
1316 return;
1317 }
1318 conf.setInt("hbase.regionstatestore.meta.connection", 3);
1319 final RegionStateStore rss =
1320 new RegionStateStore(new MyRegionServer(conf, new ZkCoordinatedStateManager()));
1321 rss.start();
1322
1323 Thread[] th = new Thread[10];
1324 List<String> nameList = new ArrayList<String>();
1325 List<TableName> tableNameList = new ArrayList<TableName>();
1326 for (int i = 0; i < th.length; i++) {
1327 th[i] = new Thread() {
1328 @Override
1329 public void run() {
1330 HRegionInfo[] hri = new HRegionInfo[10];
1331 ServerName serverName = ServerName.valueOf("dummyhost", 1000, 1234);
1332 for (int i = 0; i < 10; i++) {
1333 hri[i] = new HRegionInfo(TableName.valueOf(Thread.currentThread().getName() + "_" + i));
1334 RegionState newState = new RegionState(hri[i], RegionState.State.OPEN, serverName);
1335 RegionState oldState =
1336 new RegionState(hri[i], RegionState.State.PENDING_OPEN, serverName);
1337 rss.updateRegionState(1, newState, oldState);
1338 }
1339 }
1340 };
1341 th[i].start();
1342 nameList.add(th[i].getName());
1343 }
1344 for (int i = 0; i < th.length; i++) {
1345 th[i].join();
1346 }
1347
1348 for (String name : nameList) {
1349 for (int i = 0; i < 10; i++) {
1350 tableNameList.add(TableName.valueOf(name + "_" + i));
1351 }
1352 }
1353 List<Result> metaRows = MetaTableAccessor.fullScanOfMeta(admin.getConnection());
1354 int count = 0;
1355
1356 for (Result result : metaRows) {
1357 if (tableNameList.contains(HRegionInfo.getTable(result.getRow()))) {
1358 count++;
1359 if (count == 100) {
1360 break;
1361 }
1362 }
1363 }
1364 assertTrue(count == 100);
1365 rss.stop();
1366 }
1367
1368 static class MyLoadBalancer extends StochasticLoadBalancer {
1369
1370 static volatile HRegionInfo controledRegion = null;
1371
1372 static volatile Integer countRegionServers = null;
1373 static AtomicInteger counter = new AtomicInteger(0);
1374
1375 @Override
1376 public ServerName randomAssignment(HRegionInfo regionInfo,
1377 List<ServerName> servers) {
1378 if (regionInfo.equals(controledRegion)) {
1379 return null;
1380 }
1381 return super.randomAssignment(regionInfo, servers);
1382 }
1383
1384 @Override
1385 public Map<ServerName, List<HRegionInfo>> roundRobinAssignment(
1386 List<HRegionInfo> regions, List<ServerName> servers) {
1387 if (countRegionServers != null && services != null) {
1388 int regionServers = services.getServerManager().countOfRegionServers();
1389 if (regionServers < countRegionServers.intValue()) {
1390
1391
1392 counter.incrementAndGet();
1393 return null;
1394 }
1395 }
1396 if (regions.get(0).equals(controledRegion)) {
1397 Map<ServerName, List<HRegionInfo>> m = Maps.newHashMap();
1398 m.put(LoadBalancer.BOGUS_SERVER_NAME, regions);
1399 return m;
1400 }
1401 return super.roundRobinAssignment(regions, servers);
1402 }
1403
1404 @Override
1405 public Map<ServerName, List<HRegionInfo>> retainAssignment(
1406 Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {
1407 for (HRegionInfo hri : regions.keySet()) {
1408 if (hri.equals(controledRegion)) {
1409 Map<ServerName, List<HRegionInfo>> m = Maps.newHashMap();
1410 m.put(LoadBalancer.BOGUS_SERVER_NAME, Lists.newArrayList(regions.keySet()));
1411 return m;
1412 }
1413 }
1414 return super.retainAssignment(regions, servers);
1415 }
1416 }
1417
1418 public static class MyMaster extends HMaster {
1419 AtomicBoolean enabled = new AtomicBoolean(true);
1420
1421 public MyMaster(Configuration conf, CoordinatedStateManager cp)
1422 throws IOException, KeeperException,
1423 InterruptedException {
1424 super(conf, cp);
1425 }
1426
1427 @Override
1428 public boolean isServerCrashProcessingEnabled() {
1429 return enabled.get() && super.isServerCrashProcessingEnabled();
1430 }
1431
1432 public void enableSSH(boolean enabled) {
1433 this.enabled.set(enabled);
1434 if (enabled) {
1435 serverManager.processQueuedDeadServers();
1436 }
1437 }
1438 }
1439
1440 public static class MyRegionServer extends MiniHBaseClusterRegionServer {
1441 static volatile ServerName abortedServer = null;
1442 static volatile boolean simulateRetry = false;
1443
1444 public MyRegionServer(Configuration conf, CoordinatedStateManager cp)
1445 throws IOException, KeeperException,
1446 InterruptedException {
1447 super(conf, cp);
1448 }
1449
1450 @Override
1451 public boolean reportRegionStateTransition(TransitionCode code, long openSeqNum,
1452 HRegionInfo... hris) {
1453 if (simulateRetry) {
1454
1455 super.reportRegionStateTransition(code, openSeqNum, hris);
1456 return super.reportRegionStateTransition(code, openSeqNum, hris);
1457 }
1458 return super.reportRegionStateTransition(code, openSeqNum, hris);
1459 }
1460
1461 @Override
1462 public boolean isAborted() {
1463 return getServerName().equals(abortedServer) || super.isAborted();
1464 }
1465 }
1466
1467 public static class MyRegionObserver extends BaseRegionObserver {
1468
1469 static AtomicBoolean preCloseEnabled = new AtomicBoolean(false);
1470
1471
1472 static AtomicBoolean postCloseEnabled = new AtomicBoolean(false);
1473
1474
1475 static AtomicBoolean postOpenEnabled = new AtomicBoolean(false);
1476
1477
1478 static volatile boolean postOpenCalled = false;
1479
1480 @Override
1481 public void preClose(ObserverContext<RegionCoprocessorEnvironment> c,
1482 boolean abortRequested) throws IOException {
1483 if (preCloseEnabled.get()) throw new IOException("fail preClose from coprocessor");
1484 }
1485
1486 @Override
1487 public void postClose(ObserverContext<RegionCoprocessorEnvironment> c,
1488 boolean abortRequested) {
1489 stallOnFlag(postCloseEnabled);
1490 }
1491
1492 @Override
1493 public void postOpen(ObserverContext<RegionCoprocessorEnvironment> c) {
1494 postOpenCalled = true;
1495 stallOnFlag(postOpenEnabled);
1496 }
1497
1498 private void stallOnFlag(final AtomicBoolean flag) {
1499 try {
1500
1501 while (flag.get()) {
1502 Thread.sleep(1000);
1503 }
1504 } catch (InterruptedException ie) {
1505 Thread.currentThread().interrupt();
1506 }
1507 }
1508 }
1509 }