1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import java.io.IOException;
22 import java.io.InterruptedIOException;
23 import java.util.ArrayList;
24 import java.util.Arrays;
25 import java.util.Collection;
26 import java.util.Collections;
27 import java.util.HashMap;
28 import java.util.HashSet;
29 import java.util.Iterator;
30 import java.util.List;
31 import java.util.Map;
32 import java.util.NavigableMap;
33 import java.util.Random;
34 import java.util.Set;
35 import java.util.TreeMap;
36 import java.util.concurrent.Callable;
37 import java.util.concurrent.ConcurrentHashMap;
38 import java.util.concurrent.CopyOnWriteArrayList;
39 import java.util.concurrent.ThreadFactory;
40 import java.util.concurrent.TimeUnit;
41 import java.util.concurrent.atomic.AtomicBoolean;
42 import java.util.concurrent.atomic.AtomicInteger;
43 import java.util.concurrent.locks.Lock;
44 import java.util.concurrent.locks.ReentrantLock;
45
46 import org.apache.commons.logging.Log;
47 import org.apache.commons.logging.LogFactory;
48 import org.apache.hadoop.conf.Configuration;
49 import org.apache.hadoop.fs.FileSystem;
50 import org.apache.hadoop.fs.Path;
51 import org.apache.hadoop.hbase.CoordinatedStateException;
52 import org.apache.hadoop.hbase.HBaseIOException;
53 import org.apache.hadoop.hbase.HConstants;
54 import org.apache.hadoop.hbase.HRegionInfo;
55 import org.apache.hadoop.hbase.HRegionLocation;
56 import org.apache.hadoop.hbase.HTableDescriptor;
57 import org.apache.hadoop.hbase.MetaTableAccessor;
58 import org.apache.hadoop.hbase.NotServingRegionException;
59 import org.apache.hadoop.hbase.RegionLocations;
60 import org.apache.hadoop.hbase.RegionTransition;
61 import org.apache.hadoop.hbase.Server;
62 import org.apache.hadoop.hbase.ServerName;
63 import org.apache.hadoop.hbase.TableName;
64 import org.apache.hadoop.hbase.TableNotFoundException;
65 import org.apache.hadoop.hbase.TableStateManager;
66 import org.apache.hadoop.hbase.classification.InterfaceAudience;
67 import org.apache.hadoop.hbase.client.Admin;
68 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
69 import org.apache.hadoop.hbase.client.Result;
70 import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
71 import org.apache.hadoop.hbase.coordination.OpenRegionCoordination;
72 import org.apache.hadoop.hbase.coordination.RegionMergeCoordination;
73 import org.apache.hadoop.hbase.coordination.SplitTransactionCoordination.SplitTransactionDetails;
74 import org.apache.hadoop.hbase.coordination.ZkOpenRegionCoordination;
75 import org.apache.hadoop.hbase.coordination.ZkRegionMergeCoordination;
76 import org.apache.hadoop.hbase.exceptions.DeserializationException;
77 import org.apache.hadoop.hbase.executor.EventHandler;
78 import org.apache.hadoop.hbase.executor.EventType;
79 import org.apache.hadoop.hbase.executor.ExecutorService;
80 import org.apache.hadoop.hbase.ipc.FailedServerException;
81 import org.apache.hadoop.hbase.ipc.RpcClient;
82 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
83 import org.apache.hadoop.hbase.master.RegionState.State;
84 import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper;
85 import org.apache.hadoop.hbase.master.balancer.FavoredNodeLoadBalancer;
86 import org.apache.hadoop.hbase.master.handler.ClosedRegionHandler;
87 import org.apache.hadoop.hbase.master.handler.DisableTableHandler;
88 import org.apache.hadoop.hbase.master.handler.EnableTableHandler;
89 import org.apache.hadoop.hbase.master.handler.OpenedRegionHandler;
90 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition;
91 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
92 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
93 import org.apache.hadoop.hbase.quotas.RegionStateListener;
94 import org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException;
95 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
96 import org.apache.hadoop.hbase.regionserver.RegionServerAbortedException;
97 import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
98 import org.apache.hadoop.hbase.util.ConfigUtil;
99 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
100 import org.apache.hadoop.hbase.util.FSUtils;
101 import org.apache.hadoop.hbase.util.KeyLocker;
102 import org.apache.hadoop.hbase.util.Pair;
103 import org.apache.hadoop.hbase.util.PairOfSameType;
104 import org.apache.hadoop.hbase.util.Threads;
105 import org.apache.hadoop.hbase.util.Triple;
106 import org.apache.hadoop.hbase.wal.DefaultWALProvider;
107 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
108 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
109 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
110 import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
111 import org.apache.hadoop.ipc.RemoteException;
112 import org.apache.hadoop.util.StringUtils;
113 import org.apache.zookeeper.AsyncCallback;
114 import org.apache.zookeeper.KeeperException;
115 import org.apache.zookeeper.KeeperException.NoNodeException;
116 import org.apache.zookeeper.KeeperException.NodeExistsException;
117 import org.apache.zookeeper.data.Stat;
118
119 import com.google.common.annotations.VisibleForTesting;
120 import com.google.common.collect.LinkedHashMultimap;
121
122
123
124
125
126
127
128
129 @InterfaceAudience.Private
130 public class AssignmentManager extends ZooKeeperListener {
131 private static final Log LOG = LogFactory.getLog(AssignmentManager.class);
132
133 public static final ServerName HBCK_CODE_SERVERNAME = ServerName.valueOf(HConstants.HBCK_CODE_NAME,
134 -1, -1L);
135
136 static final String ALREADY_IN_TRANSITION_WAITTIME
137 = "hbase.assignment.already.intransition.waittime";
138 static final int DEFAULT_ALREADY_IN_TRANSITION_WAITTIME = 60000;
139
140 protected final Server server;
141
142 private ServerManager serverManager;
143
144 private boolean shouldAssignRegionsWithFavoredNodes;
145
146 private LoadBalancer balancer;
147
148 private final MetricsAssignmentManager metricsAssignmentManager;
149
150 private final TableLockManager tableLockManager;
151
152 private AtomicInteger numRegionsOpened = new AtomicInteger(0);
153
154 final private KeyLocker<String> locker = new KeyLocker<String>();
155
156 Set<HRegionInfo> replicasToClose = Collections.synchronizedSet(new HashSet<HRegionInfo>());
157
158
159
160
161
162 private final Map <String, HRegionInfo> regionsToReopen;
163
164
165
166
167
168 private final int maximumAttempts;
169
170
171
172
173 private final Map<String, PairOfSameType<HRegionInfo>> mergingRegions
174 = new HashMap<String, PairOfSameType<HRegionInfo>>();
175
176 private final Map<HRegionInfo, PairOfSameType<HRegionInfo>> splitRegions
177 = new HashMap<HRegionInfo, PairOfSameType<HRegionInfo>>();
178
179
180
181
182
183 private final long sleepTimeBeforeRetryingMetaAssignment;
184
185
186
187
188
189 final NavigableMap<String, RegionPlan> regionPlans =
190 new TreeMap<String, RegionPlan>();
191
192 private final TableStateManager tableStateManager;
193
194 private final ExecutorService executorService;
195
196
197 private Map<HRegionInfo, AtomicBoolean> closedRegionHandlerCalled = null;
198
199
200 private Map<HRegionInfo, AtomicBoolean> openedRegionHandlerCalled = null;
201
202
203 private java.util.concurrent.ExecutorService threadPoolExecutorService;
204
205
206 private final java.util.concurrent.ExecutorService zkEventWorkers;
207
208 private List<EventType> ignoreStatesRSOffline = Arrays.asList(
209 EventType.RS_ZK_REGION_FAILED_OPEN, EventType.RS_ZK_REGION_CLOSED);
210
211 private final RegionStates regionStates;
212
213
214
215
216
217 private final int bulkAssignThresholdRegions;
218 private final int bulkAssignThresholdServers;
219 private final int bulkPerRegionOpenTimeGuesstimate;
220
221
222
223
224 private final boolean bulkAssignWaitTillAllAssigned;
225
226
227
228
229
230
231
232
233
234 protected final AtomicBoolean failoverCleanupDone = new AtomicBoolean(false);
235
236
237
238
239
240
241
242
243 private final ConcurrentHashMap<String, AtomicInteger>
244 failedOpenTracker = new ConcurrentHashMap<String, AtomicInteger>();
245
246
247 private final boolean useZKForAssignment;
248
249
250
251 private final RegionStateStore regionStateStore;
252
253
254
255
256 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="MS_SHOULD_BE_FINAL")
257 public static boolean TEST_SKIP_SPLIT_HANDLING = false;
258
259
260 private List<AssignmentListener> listeners = new CopyOnWriteArrayList<AssignmentListener>();
261
262 private RegionStateListener regionStateListener;
263
264 public enum ServerHostRegion {
265 NOT_HOSTING_REGION, HOSTING_REGION, UNKNOWN,
266 }
267
268
269
270
271
272
273
274
275
276
277
278
279
280 public AssignmentManager(Server server, ServerManager serverManager,
281 final LoadBalancer balancer,
282 final ExecutorService service, MetricsMaster metricsMaster,
283 final TableLockManager tableLockManager) throws KeeperException,
284 IOException, CoordinatedStateException {
285 super(server.getZooKeeper());
286 this.server = server;
287 this.serverManager = serverManager;
288 this.executorService = service;
289 this.regionStateStore = new RegionStateStore(server);
290 this.regionsToReopen = Collections.synchronizedMap
291 (new HashMap<String, HRegionInfo> ());
292 Configuration conf = server.getConfiguration();
293
294 this.shouldAssignRegionsWithFavoredNodes = conf.getClass(
295 HConstants.HBASE_MASTER_LOADBALANCER_CLASS, Object.class).equals(
296 FavoredNodeLoadBalancer.class);
297 try {
298 if (server.getCoordinatedStateManager() != null) {
299 this.tableStateManager = server.getCoordinatedStateManager().getTableStateManager();
300 } else {
301 this.tableStateManager = null;
302 }
303 } catch (InterruptedException e) {
304 throw new InterruptedIOException();
305 }
306
307 this.maximumAttempts = Math.max(1,
308 this.server.getConfiguration().getInt("hbase.assignment.maximum.attempts", 10));
309 this.sleepTimeBeforeRetryingMetaAssignment = this.server.getConfiguration().getLong(
310 "hbase.meta.assignment.retry.sleeptime", 1000l);
311 this.balancer = balancer;
312 int maxThreads = conf.getInt("hbase.assignment.threads.max", 30);
313 this.threadPoolExecutorService = Threads.getBoundedCachedThreadPool(
314 maxThreads, 60L, TimeUnit.SECONDS, Threads.newDaemonThreadFactory("AM."));
315 this.regionStates = new RegionStates(
316 server, tableStateManager, serverManager, regionStateStore);
317
318 this.bulkAssignWaitTillAllAssigned =
319 conf.getBoolean("hbase.bulk.assignment.waittillallassigned", false);
320 this.bulkAssignThresholdRegions = conf.getInt("hbase.bulk.assignment.threshold.regions", 7);
321 this.bulkAssignThresholdServers = conf.getInt("hbase.bulk.assignment.threshold.servers", 3);
322 this.bulkPerRegionOpenTimeGuesstimate =
323 conf.getInt("hbase.bulk.assignment.perregion.open.time", 10000);
324
325 int workers = conf.getInt("hbase.assignment.zkevent.workers", 20);
326 ThreadFactory threadFactory = Threads.newDaemonThreadFactory("AM.ZK.Worker");
327 zkEventWorkers = Threads.getBoundedCachedThreadPool(workers, 60L,
328 TimeUnit.SECONDS, threadFactory);
329 this.tableLockManager = tableLockManager;
330
331 this.metricsAssignmentManager = new MetricsAssignmentManager();
332 useZKForAssignment = ConfigUtil.useZKForAssignment(conf);
333 }
334
335 MetricsAssignmentManager getAssignmentManagerMetrics() {
336 return this.metricsAssignmentManager;
337 }
338
339
340
341
342
343 public void registerListener(final AssignmentListener listener) {
344 this.listeners.add(listener);
345 }
346
347
348
349
350
351 public boolean unregisterListener(final AssignmentListener listener) {
352 return this.listeners.remove(listener);
353 }
354
355
356
357
358 public TableStateManager getTableStateManager() {
359
360
361 return this.tableStateManager;
362 }
363
364
365
366
367
368
369
370 public RegionStates getRegionStates() {
371 return regionStates;
372 }
373
374
375
376
377 @VisibleForTesting
378 RegionStateStore getRegionStateStore() {
379 return regionStateStore;
380 }
381
382 public RegionPlan getRegionReopenPlan(HRegionInfo hri) {
383 return new RegionPlan(hri, null, regionStates.getRegionServerOfRegion(hri));
384 }
385
386
387
388
389
390
391 public void addPlan(String encodedName, RegionPlan plan) {
392 synchronized (regionPlans) {
393 regionPlans.put(encodedName, plan);
394 }
395 }
396
397
398
399
400 public void addPlans(Map<String, RegionPlan> plans) {
401 synchronized (regionPlans) {
402 regionPlans.putAll(plans);
403 }
404 }
405
406
407
408
409
410
411
412
413 public void setRegionsToReopen(List <HRegionInfo> regions) {
414 for(HRegionInfo hri : regions) {
415 regionsToReopen.put(hri.getEncodedName(), hri);
416 }
417 }
418
419
420
421
422
423
424
425
426 public Pair<Integer, Integer> getReopenStatus(TableName tableName)
427 throws IOException {
428 List<HRegionInfo> hris;
429 if (TableName.META_TABLE_NAME.equals(tableName)) {
430 hris = new MetaTableLocator().getMetaRegions(server.getZooKeeper());
431 } else {
432 hris = MetaTableAccessor.getTableRegions(server.getZooKeeper(),
433 server.getConnection(), tableName, true);
434 }
435
436 Integer pending = 0;
437 for (HRegionInfo hri : hris) {
438 String name = hri.getEncodedName();
439
440 if (regionsToReopen.containsKey(name)
441 || regionStates.isRegionInTransition(name)) {
442 pending++;
443 }
444 }
445 return new Pair<Integer, Integer>(pending, hris.size());
446 }
447
448
449
450
451
452
453 public boolean isFailoverCleanupDone() {
454 return failoverCleanupDone.get();
455 }
456
457
458
459
460
461 public Lock acquireRegionLock(final String encodedName) {
462 return locker.acquireLock(encodedName);
463 }
464
465
466
467
468
469 void failoverCleanupDone() {
470 failoverCleanupDone.set(true);
471 serverManager.processQueuedDeadServers();
472 }
473
474
475
476
477
478
479
480
481
482 void joinCluster() throws IOException,
483 KeeperException, InterruptedException, CoordinatedStateException {
484 long startTime = System.currentTimeMillis();
485
486
487
488
489
490
491
492
493
494
495
496 Set<ServerName> deadServers = rebuildUserRegions();
497
498
499
500
501 boolean failover = processDeadServersAndRegionsInTransition(deadServers);
502
503 if (!useZKForAssignment) {
504
505 ZKUtil.deleteNodeRecursively(watcher, watcher.assignmentZNode);
506 }
507 recoverTableInDisablingState();
508 recoverTableInEnablingState();
509 LOG.info("Joined the cluster in " + (System.currentTimeMillis()
510 - startTime) + "ms, failover=" + failover);
511 }
512
513
514
515
516
517
518
519
520
521
522
523
524 boolean processDeadServersAndRegionsInTransition(
525 final Set<ServerName> deadServers) throws KeeperException,
526 IOException, InterruptedException, CoordinatedStateException {
527 List<String> nodes = ZKUtil.listChildrenNoWatch(watcher,
528 watcher.assignmentZNode);
529
530 if (useZKForAssignment && nodes == null) {
531 String errorMessage = "Failed to get the children from ZK";
532 server.abort(errorMessage, new IOException(errorMessage));
533 return true;
534 }
535
536 boolean failover = !serverManager.getDeadServers().isEmpty();
537 if (failover) {
538
539 if (LOG.isDebugEnabled()) {
540 LOG.debug("Found dead servers out on cluster " + serverManager.getDeadServers());
541 }
542 } else {
543
544 Set<ServerName> onlineServers = serverManager.getOnlineServers().keySet();
545 for (Map.Entry<HRegionInfo, ServerName> en:
546 regionStates.getRegionAssignments().entrySet()) {
547 HRegionInfo hri = en.getKey();
548 if (!hri.isMetaTable()
549 && onlineServers.contains(en.getValue())) {
550 LOG.debug("Found " + hri + " out on cluster");
551 failover = true;
552 break;
553 }
554 }
555 if (!failover && nodes != null) {
556
557 for (String encodedName: nodes) {
558 RegionState regionState = regionStates.getRegionState(encodedName);
559 if (regionState != null && !regionState.getRegion().isMetaRegion()) {
560 LOG.debug("Found " + regionState + " in RITs");
561 failover = true;
562 break;
563 }
564 }
565 }
566 }
567 if (!failover && !useZKForAssignment) {
568
569 Map<String, RegionState> regionsInTransition = regionStates.getRegionsInTransition();
570 if (!regionsInTransition.isEmpty()) {
571 Set<ServerName> onlineServers = serverManager.getOnlineServers().keySet();
572 for (RegionState regionState: regionsInTransition.values()) {
573 ServerName serverName = regionState.getServerName();
574 if (!regionState.getRegion().isMetaRegion()
575 && serverName != null && onlineServers.contains(serverName)) {
576 LOG.debug("Found " + regionState + " in RITs");
577 failover = true;
578 break;
579 }
580 }
581 }
582 }
583 if (!failover) {
584
585
586
587
588 Set<ServerName> queuedDeadServers = serverManager.getRequeuedDeadServers().keySet();
589 if (!queuedDeadServers.isEmpty()) {
590 Configuration conf = server.getConfiguration();
591 Path walRootDir = FSUtils.getWALRootDir(conf);
592 FileSystem walFs = FSUtils.getWALFileSystem(conf);
593 for (ServerName serverName: queuedDeadServers) {
594
595
596 Path walDir = new Path(walRootDir,
597 DefaultWALProvider.getWALDirectoryName(serverName.toString()));
598 Path splitDir = walDir.suffix(DefaultWALProvider.SPLITTING_EXT);
599 if (walFs.exists(walDir) || walFs.exists(splitDir)) {
600 LOG.debug("Found queued dead server " + serverName);
601 failover = true;
602 break;
603 }
604 }
605 if (!failover) {
606
607
608 LOG.info("AM figured that it's not a failover and cleaned up "
609 + queuedDeadServers.size() + " queued dead servers");
610 serverManager.removeRequeuedDeadServers();
611 }
612 }
613 }
614
615 Set<TableName> disabledOrDisablingOrEnabling = null;
616 Map<HRegionInfo, ServerName> allRegions = null;
617
618 if (!failover) {
619 disabledOrDisablingOrEnabling = tableStateManager.getTablesInStates(
620 ZooKeeperProtos.Table.State.DISABLED, ZooKeeperProtos.Table.State.DISABLING,
621 ZooKeeperProtos.Table.State.ENABLING);
622
623
624 allRegions = regionStates.closeAllUserRegions(
625 disabledOrDisablingOrEnabling);
626 }
627
628
629 regionStateStore.start();
630
631
632 if (failover) {
633 LOG.info("Found regions out on cluster or in RIT; presuming failover");
634
635
636 processDeadServersAndRecoverLostRegions(deadServers);
637 }
638
639 if (!failover && useZKForAssignment) {
640
641 ZKAssign.deleteAllNodes(watcher);
642 ZKUtil.listChildrenAndWatchForNewChildren(this.watcher,
643 this.watcher.assignmentZNode);
644 }
645
646
647
648
649
650 failoverCleanupDone();
651 if (!failover) {
652
653 LOG.info("Clean cluster startup. Assigning user regions");
654 assignAllUserRegions(allRegions);
655 }
656
657
658
659 for (HRegionInfo h : replicasToClose) {
660 unassign(h);
661 }
662 replicasToClose.clear();
663 return failover;
664 }
665
666
667
668
669
670
671
672
673
674
675
676
677 boolean processRegionInTransitionAndBlockUntilAssigned(final HRegionInfo hri)
678 throws InterruptedException, KeeperException, IOException {
679 String encodedRegionName = hri.getEncodedName();
680 if (!processRegionInTransition(encodedRegionName, hri)) {
681 return false;
682 }
683 LOG.debug("Waiting on " + HRegionInfo.prettyPrint(encodedRegionName));
684 while (!this.server.isStopped() &&
685 this.regionStates.isRegionInTransition(encodedRegionName)) {
686 RegionState state = this.regionStates.getRegionTransitionState(encodedRegionName);
687 if (state == null || !serverManager.isServerOnline(state.getServerName())) {
688
689
690
691 break;
692 }
693 this.regionStates.waitForUpdate(100);
694 }
695 return true;
696 }
697
698
699
700
701
702
703
704
705
706
707 boolean processRegionInTransition(final String encodedRegionName,
708 final HRegionInfo regionInfo) throws KeeperException, IOException {
709
710
711
712
713 Lock lock = locker.acquireLock(encodedRegionName);
714 try {
715 Stat stat = new Stat();
716 byte [] data = ZKAssign.getDataAndWatch(watcher, encodedRegionName, stat);
717 if (data == null) return false;
718 RegionTransition rt;
719 try {
720 rt = RegionTransition.parseFrom(data);
721 } catch (DeserializationException e) {
722 LOG.warn("Failed parse znode data", e);
723 return false;
724 }
725 HRegionInfo hri = regionInfo;
726 if (hri == null) {
727
728
729
730
731
732 hri = regionStates.getRegionInfo(rt.getRegionName());
733 EventType et = rt.getEventType();
734 if (hri == null && et != EventType.RS_ZK_REGION_MERGING
735 && et != EventType.RS_ZK_REQUEST_REGION_MERGE) {
736 LOG.warn("Couldn't find the region in recovering " + rt);
737 return false;
738 }
739 }
740
741
742
743 BaseCoordinatedStateManager cp =
744 (BaseCoordinatedStateManager) this.server.getCoordinatedStateManager();
745 OpenRegionCoordination openRegionCoordination = cp.getOpenRegionCoordination();
746
747 ZkOpenRegionCoordination.ZkOpenRegionDetails zkOrd =
748 new ZkOpenRegionCoordination.ZkOpenRegionDetails();
749 zkOrd.setVersion(stat.getVersion());
750 zkOrd.setServerName(cp.getServer().getServerName());
751
752 return processRegionsInTransition(
753 rt, hri, openRegionCoordination, zkOrd);
754 } finally {
755 lock.unlock();
756 }
757 }
758
759
760
761
762
763
764
765
766
767 boolean processRegionsInTransition(
768 final RegionTransition rt, final HRegionInfo regionInfo,
769 OpenRegionCoordination coordination,
770 final OpenRegionCoordination.OpenRegionDetails ord) throws KeeperException {
771 EventType et = rt.getEventType();
772
773 final ServerName sn = rt.getServerName();
774 final byte[] regionName = rt.getRegionName();
775 final String encodedName = HRegionInfo.encodeRegionName(regionName);
776 final String prettyPrintedRegionName = HRegionInfo.prettyPrint(encodedName);
777 LOG.info("Processing " + prettyPrintedRegionName + " in state: " + et);
778
779 if (regionStates.isRegionInTransition(encodedName)
780 && (regionInfo.isMetaRegion() || !useZKForAssignment)) {
781 LOG.info("Processed region " + prettyPrintedRegionName + " in state: "
782 + et + ", does nothing since the region is already in transition "
783 + regionStates.getRegionTransitionState(encodedName));
784
785 return true;
786 }
787 if (!serverManager.isServerOnline(sn)) {
788
789
790
791 LOG.debug("RIT " + encodedName + " in state=" + rt.getEventType() +
792 " was on deadserver; forcing offline");
793 if (regionStates.isRegionOnline(regionInfo)) {
794
795
796
797 regionStates.regionOffline(regionInfo);
798 sendRegionClosedNotification(regionInfo);
799 }
800
801 regionStates.updateRegionState(regionInfo, State.OFFLINE, sn);
802
803 if (regionInfo.isMetaRegion()) {
804
805
806 MetaTableLocator.setMetaLocation(watcher, sn, State.OPEN);
807 } else {
808
809
810 regionStates.setLastRegionServerOfRegion(sn, encodedName);
811
812 if (!serverManager.isServerDead(sn)) {
813 serverManager.expireServer(sn);
814 }
815 }
816 return false;
817 }
818 switch (et) {
819 case M_ZK_REGION_CLOSING:
820
821
822 final RegionState rsClosing = regionStates.updateRegionState(rt, State.CLOSING);
823 this.executorService.submit(
824 new EventHandler(server, EventType.M_MASTER_RECOVERY) {
825 @Override
826 public void process() throws IOException {
827 ReentrantLock lock = locker.acquireLock(regionInfo.getEncodedName());
828 try {
829 final int expectedVersion = ((ZkOpenRegionCoordination.ZkOpenRegionDetails) ord)
830 .getVersion();
831 unassign(regionInfo, rsClosing, expectedVersion, null, useZKForAssignment, null);
832 if (regionStates.isRegionOffline(regionInfo)) {
833 assign(regionInfo, true);
834 }
835 } finally {
836 lock.unlock();
837 }
838 }
839 });
840 break;
841
842 case RS_ZK_REGION_CLOSED:
843 case RS_ZK_REGION_FAILED_OPEN:
844
845 regionStates.setRegionStateTOCLOSED(regionInfo, sn);
846 if (!replicasToClose.contains(regionInfo)) {
847 invokeAssign(regionInfo);
848 } else {
849 offlineDisabledRegion(regionInfo);
850 }
851 break;
852
853 case M_ZK_REGION_OFFLINE:
854
855 regionStates.updateRegionState(rt, State.OFFLINE);
856 final RegionState rsOffline = regionStates.getRegionState(regionInfo);
857 this.executorService.submit(
858 new EventHandler(server, EventType.M_MASTER_RECOVERY) {
859 @Override
860 public void process() throws IOException {
861 ReentrantLock lock = locker.acquireLock(regionInfo.getEncodedName());
862 try {
863 RegionPlan plan = new RegionPlan(regionInfo, null, sn);
864 addPlan(encodedName, plan);
865 assign(rsOffline, true, false);
866 } finally {
867 lock.unlock();
868 }
869 }
870 });
871 break;
872
873 case RS_ZK_REGION_OPENING:
874 regionStates.updateRegionState(rt, State.OPENING);
875 break;
876
877 case RS_ZK_REGION_OPENED:
878
879
880
881 regionStates.updateRegionState(rt, State.OPEN);
882 new OpenedRegionHandler(server, this, regionInfo, coordination, ord).process();
883 break;
884 case RS_ZK_REQUEST_REGION_SPLIT:
885 case RS_ZK_REGION_SPLITTING:
886 case RS_ZK_REGION_SPLIT:
887
888
889
890 regionStates.regionOnline(regionInfo, sn);
891 regionStates.updateRegionState(rt, State.SPLITTING);
892 if (!handleRegionSplitting(
893 rt, encodedName, prettyPrintedRegionName, sn)) {
894 deleteSplittingNode(encodedName, sn);
895 }
896 break;
897 case RS_ZK_REQUEST_REGION_MERGE:
898 case RS_ZK_REGION_MERGING:
899 case RS_ZK_REGION_MERGED:
900 if (!handleRegionMerging(
901 rt, encodedName, prettyPrintedRegionName, sn)) {
902 deleteMergingNode(encodedName, sn);
903 }
904 break;
905 default:
906 throw new IllegalStateException("Received region in state:" + et + " is not valid.");
907 }
908 LOG.info("Processed region " + prettyPrintedRegionName + " in state "
909 + et + ", on " + (serverManager.isServerOnline(sn) ? "" : "dead ")
910 + "server: " + sn);
911 return true;
912 }
913
914
915
916
917
918 public void removeClosedRegion(HRegionInfo hri) {
919 if (regionsToReopen.remove(hri.getEncodedName()) != null) {
920 LOG.debug("Removed region from reopening regions because it was closed");
921 }
922 }
923
924
925
926
927
928
929
930
931
932
933
934
935 void handleRegion(final RegionTransition rt, OpenRegionCoordination coordination,
936 OpenRegionCoordination.OpenRegionDetails ord) {
937 if (rt == null) {
938 LOG.warn("Unexpected NULL input for RegionTransition rt");
939 return;
940 }
941 final ServerName sn = rt.getServerName();
942
943 if (sn.equals(HBCK_CODE_SERVERNAME)) {
944 handleHBCK(rt);
945 return;
946 }
947 final long createTime = rt.getCreateTime();
948 final byte[] regionName = rt.getRegionName();
949 String encodedName = HRegionInfo.encodeRegionName(regionName);
950 String prettyPrintedRegionName = HRegionInfo.prettyPrint(encodedName);
951
952 if (!serverManager.isServerOnline(sn)
953 && !ignoreStatesRSOffline.contains(rt.getEventType())) {
954 LOG.warn("Attempted to handle region transition for server but " +
955 "it is not online: " + prettyPrintedRegionName + ", " + rt);
956 return;
957 }
958
959 RegionState regionState =
960 regionStates.getRegionState(encodedName);
961 long startTime = System.currentTimeMillis();
962 if (LOG.isDebugEnabled()) {
963 boolean lateEvent = createTime < (startTime - 15000);
964 LOG.debug("Handling " + rt.getEventType() +
965 ", server=" + sn + ", region=" +
966 (prettyPrintedRegionName == null ? "null" : prettyPrintedRegionName) +
967 (lateEvent ? ", which is more than 15 seconds late" : "") +
968 ", current_state=" + regionState);
969 }
970
971
972 if (rt.getEventType() == EventType.M_ZK_REGION_OFFLINE) {
973 return;
974 }
975
976
977 Lock lock = locker.acquireLock(encodedName);
978 try {
979 RegionState latestState =
980 regionStates.getRegionState(encodedName);
981 if ((regionState == null && latestState != null)
982 || (regionState != null && latestState == null)
983 || (regionState != null && latestState != null
984 && latestState.getState() != regionState.getState())) {
985 LOG.warn("Region state changed from " + regionState + " to "
986 + latestState + ", while acquiring lock");
987 }
988 long waitedTime = System.currentTimeMillis() - startTime;
989 if (waitedTime > 5000) {
990 LOG.warn("Took " + waitedTime + "ms to acquire the lock");
991 }
992 regionState = latestState;
993 switch (rt.getEventType()) {
994 case RS_ZK_REQUEST_REGION_SPLIT:
995 case RS_ZK_REGION_SPLITTING:
996 case RS_ZK_REGION_SPLIT:
997 if (!handleRegionSplitting(
998 rt, encodedName, prettyPrintedRegionName, sn)) {
999 deleteSplittingNode(encodedName, sn);
1000 }
1001 break;
1002
1003 case RS_ZK_REQUEST_REGION_MERGE:
1004 case RS_ZK_REGION_MERGING:
1005 case RS_ZK_REGION_MERGED:
1006
1007
1008 if (!handleRegionMerging(
1009 rt, encodedName, prettyPrintedRegionName, sn)) {
1010 deleteMergingNode(encodedName, sn);
1011 }
1012 break;
1013
1014 case M_ZK_REGION_CLOSING:
1015
1016
1017 if (regionState == null
1018 || !regionState.isPendingCloseOrClosingOnServer(sn)) {
1019 LOG.warn("Received CLOSING for " + prettyPrintedRegionName
1020 + " from " + sn + " but the region isn't PENDING_CLOSE/CLOSING here: "
1021 + regionStates.getRegionState(encodedName));
1022 return;
1023 }
1024
1025 regionStates.updateRegionState(rt, State.CLOSING);
1026 break;
1027
1028 case RS_ZK_REGION_CLOSED:
1029
1030 if (regionState == null
1031 || !regionState.isPendingCloseOrClosingOnServer(sn)) {
1032 LOG.warn("Received CLOSED for " + prettyPrintedRegionName
1033 + " from " + sn + " but the region isn't PENDING_CLOSE/CLOSING here: "
1034 + regionStates.getRegionState(encodedName));
1035 return;
1036 }
1037
1038
1039
1040 new ClosedRegionHandler(server, this, regionState.getRegion()).process();
1041 updateClosedRegionHandlerTracker(regionState.getRegion());
1042 break;
1043
1044 case RS_ZK_REGION_FAILED_OPEN:
1045 if (regionState == null
1046 || !regionState.isPendingOpenOrOpeningOnServer(sn)) {
1047 LOG.warn("Received FAILED_OPEN for " + prettyPrintedRegionName
1048 + " from " + sn + " but the region isn't PENDING_OPEN/OPENING here: "
1049 + regionStates.getRegionState(encodedName));
1050 return;
1051 }
1052 AtomicInteger failedOpenCount = failedOpenTracker.get(encodedName);
1053 if (failedOpenCount == null) {
1054 failedOpenCount = new AtomicInteger();
1055
1056
1057
1058 failedOpenTracker.put(encodedName, failedOpenCount);
1059 }
1060 if (failedOpenCount.incrementAndGet() >= maximumAttempts) {
1061 regionStates.updateRegionState(rt, State.FAILED_OPEN);
1062
1063
1064 failedOpenTracker.remove(encodedName);
1065 } else {
1066
1067 regionState = regionStates.setRegionStateTOCLOSED(rt.getRegionName(), sn);
1068 if (regionState != null) {
1069
1070
1071 try {
1072 getRegionPlan(regionState.getRegion(), sn, true);
1073 new ClosedRegionHandler(server, this, regionState.getRegion()).process();
1074 } catch (HBaseIOException e) {
1075 LOG.warn("Failed to get region plan", e);
1076 }
1077 }
1078 }
1079 break;
1080
1081 case RS_ZK_REGION_OPENING:
1082
1083
1084 if (regionState == null
1085 || !regionState.isPendingOpenOrOpeningOnServer(sn)) {
1086 LOG.warn("Received OPENING for " + prettyPrintedRegionName
1087 + " from " + sn + " but the region isn't PENDING_OPEN/OPENING here: "
1088 + regionStates.getRegionState(encodedName));
1089 return;
1090 }
1091
1092 regionStates.updateRegionState(rt, State.OPENING);
1093 break;
1094
1095 case RS_ZK_REGION_OPENED:
1096
1097 if (regionState == null
1098 || !regionState.isPendingOpenOrOpeningOnServer(sn)) {
1099 LOG.warn("Received OPENED for " + prettyPrintedRegionName
1100 + " from " + sn + " but the region isn't PENDING_OPEN/OPENING here: "
1101 + regionStates.getRegionState(encodedName));
1102
1103 if (regionState != null) {
1104 if(regionState.isOpened() && regionState.getServerName().equals(sn)) {
1105
1106
1107 failedOpenTracker.remove(encodedName);
1108 new OpenedRegionHandler(
1109 server, this, regionState.getRegion(), coordination, ord).process();
1110 updateOpenedRegionHandlerTracker(regionState.getRegion());
1111 } else {
1112
1113
1114
1115 unassign(regionState.getRegion(), null, -1, null, false, sn);
1116 }
1117 }
1118 return;
1119 }
1120
1121 regionState =
1122 regionStates.transitionOpenFromPendingOpenOrOpeningOnServer(rt,regionState, sn);
1123 if (regionState != null) {
1124 failedOpenTracker.remove(encodedName);
1125 new OpenedRegionHandler(
1126 server, this, regionState.getRegion(), coordination, ord).process();
1127 updateOpenedRegionHandlerTracker(regionState.getRegion());
1128 }
1129 break;
1130
1131 default:
1132 throw new IllegalStateException("Received event is not valid.");
1133 }
1134 } finally {
1135 lock.unlock();
1136 }
1137 }
1138
1139
1140 boolean wasClosedHandlerCalled(HRegionInfo hri) {
1141 AtomicBoolean b = closedRegionHandlerCalled.get(hri);
1142
1143
1144
1145 return b == null ? false : b.compareAndSet(true, false);
1146 }
1147
1148
1149 boolean wasOpenedHandlerCalled(HRegionInfo hri) {
1150 AtomicBoolean b = openedRegionHandlerCalled.get(hri);
1151
1152
1153
1154 return b == null ? false : b.compareAndSet(true, false);
1155 }
1156
1157
1158 void initializeHandlerTrackers() {
1159 closedRegionHandlerCalled = new HashMap<HRegionInfo, AtomicBoolean>();
1160 openedRegionHandlerCalled = new HashMap<HRegionInfo, AtomicBoolean>();
1161 }
1162
1163 void updateClosedRegionHandlerTracker(HRegionInfo hri) {
1164 if (closedRegionHandlerCalled != null) {
1165 closedRegionHandlerCalled.put(hri, new AtomicBoolean(true));
1166 }
1167 }
1168
1169 void updateOpenedRegionHandlerTracker(HRegionInfo hri) {
1170 if (openedRegionHandlerCalled != null) {
1171 openedRegionHandlerCalled.put(hri, new AtomicBoolean(true));
1172 }
1173 }
1174
1175
1176
1177
1178
1179
1180 void processFavoredNodes(List<HRegionInfo> regions) throws IOException {
1181 if (!shouldAssignRegionsWithFavoredNodes) return;
1182
1183
1184 Map<HRegionInfo, List<ServerName>> regionToFavoredNodes =
1185 new HashMap<HRegionInfo, List<ServerName>>();
1186 for (HRegionInfo region : regions) {
1187 regionToFavoredNodes.put(region,
1188 ((FavoredNodeLoadBalancer)this.balancer).getFavoredNodes(region));
1189 }
1190 FavoredNodeAssignmentHelper.updateMetaWithFavoredNodesInfo(regionToFavoredNodes,
1191 this.server.getConnection());
1192 }
1193
1194
1195
1196
1197
1198
1199
1200 @SuppressWarnings("deprecation")
1201 private void handleHBCK(RegionTransition rt) {
1202 String encodedName = HRegionInfo.encodeRegionName(rt.getRegionName());
1203 LOG.info("Handling HBCK triggered transition=" + rt.getEventType() +
1204 ", server=" + rt.getServerName() + ", region=" +
1205 HRegionInfo.prettyPrint(encodedName));
1206 RegionState regionState = regionStates.getRegionTransitionState(encodedName);
1207 switch (rt.getEventType()) {
1208 case M_ZK_REGION_OFFLINE:
1209 HRegionInfo regionInfo;
1210 if (regionState != null) {
1211 regionInfo = regionState.getRegion();
1212 } else {
1213 try {
1214 byte [] name = rt.getRegionName();
1215 Pair<HRegionInfo, ServerName> p = MetaTableAccessor.getRegion(
1216 this.server.getConnection(), name);
1217 regionInfo = p.getFirst();
1218 } catch (IOException e) {
1219 LOG.info("Exception reading hbase:meta doing HBCK repair operation", e);
1220 return;
1221 }
1222 }
1223 LOG.info("HBCK repair is triggering assignment of region=" +
1224 regionInfo.getRegionNameAsString());
1225
1226 assign(regionInfo, false);
1227 break;
1228
1229 default:
1230 LOG.warn("Received unexpected region state from HBCK: " + rt.toString());
1231 break;
1232 }
1233
1234 }
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250 @Override
1251 public void nodeCreated(String path) {
1252 handleAssignmentEvent(path);
1253 }
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267 @Override
1268 public void nodeDataChanged(String path) {
1269 handleAssignmentEvent(path);
1270 }
1271
1272
1273
1274
1275
1276 private final Set<String> regionsInProgress = new HashSet<String>();
1277
1278
1279 private final LinkedHashMultimap <String, RegionRunnable>
1280 zkEventWorkerWaitingList = LinkedHashMultimap.create();
1281
1282
1283
1284
1285 private interface RegionRunnable extends Runnable{
1286
1287
1288
1289 String getRegionName();
1290 }
1291
1292
1293
1294
1295
1296 protected void zkEventWorkersSubmit(final RegionRunnable regRunnable) {
1297
1298 synchronized (regionsInProgress) {
1299
1300
1301 if (regionsInProgress.contains(regRunnable.getRegionName())) {
1302 synchronized (zkEventWorkerWaitingList){
1303 zkEventWorkerWaitingList.put(regRunnable.getRegionName(), regRunnable);
1304 }
1305 return;
1306 }
1307
1308
1309 regionsInProgress.add(regRunnable.getRegionName());
1310 zkEventWorkers.submit(new Runnable() {
1311 @Override
1312 public void run() {
1313 try {
1314 regRunnable.run();
1315 } finally {
1316
1317
1318 synchronized (regionsInProgress) {
1319 regionsInProgress.remove(regRunnable.getRegionName());
1320 synchronized (zkEventWorkerWaitingList) {
1321 java.util.Set<RegionRunnable> waiting = zkEventWorkerWaitingList.get(
1322 regRunnable.getRegionName());
1323 if (!waiting.isEmpty()) {
1324
1325 RegionRunnable toSubmit = waiting.iterator().next();
1326 zkEventWorkerWaitingList.remove(toSubmit.getRegionName(), toSubmit);
1327 zkEventWorkersSubmit(toSubmit);
1328 }
1329 }
1330 }
1331 }
1332 }
1333 });
1334 }
1335 }
1336
1337 @Override
1338 public void nodeDeleted(final String path) {
1339 if (path.startsWith(watcher.assignmentZNode)) {
1340 final String regionName = ZKAssign.getRegionName(watcher, path);
1341 zkEventWorkersSubmit(new RegionRunnable() {
1342 @Override
1343 public String getRegionName() {
1344 return regionName;
1345 }
1346
1347 @Override
1348 public void run() {
1349 Lock lock = locker.acquireLock(regionName);
1350 try {
1351 RegionState rs = regionStates.getRegionTransitionState(regionName);
1352 if (rs == null) {
1353 rs = regionStates.getRegionState(regionName);
1354 if (rs == null || !rs.isMergingNew()) {
1355
1356 return;
1357 }
1358 }
1359
1360 HRegionInfo regionInfo = rs.getRegion();
1361 String regionNameStr = regionInfo.getRegionNameAsString();
1362 LOG.debug("Znode " + regionNameStr + " deleted, state: " + rs);
1363
1364 boolean disabled = getTableStateManager().isTableState(regionInfo.getTable(),
1365 ZooKeeperProtos.Table.State.DISABLED, ZooKeeperProtos.Table.State.DISABLING);
1366
1367 ServerName serverName = rs.getServerName();
1368 if (serverManager.isServerOnline(serverName)) {
1369 if (rs.isOnServer(serverName) && (rs.isOpened() || rs.isSplitting())) {
1370 synchronized (regionStates) {
1371 regionOnline(regionInfo, serverName);
1372 if (rs.isSplitting() && splitRegions.containsKey(regionInfo)) {
1373
1374
1375 HRegionInfo hri_a = splitRegions.get(regionInfo).getFirst();
1376 HRegionInfo hri_b = splitRegions.get(regionInfo).getSecond();
1377 if (!regionStates.isRegionInTransition(hri_a.getEncodedName())) {
1378 LOG.warn("Split daughter region not in transition " + hri_a);
1379 }
1380 if (!regionStates.isRegionInTransition(hri_b.getEncodedName())) {
1381 LOG.warn("Split daughter region not in transition" + hri_b);
1382 }
1383 regionOffline(hri_a);
1384 regionOffline(hri_b);
1385 splitRegions.remove(regionInfo);
1386 }
1387 if (disabled) {
1388
1389 LOG.info("Opened " + regionNameStr
1390 + "but this table is disabled, triggering close of region");
1391 unassign(regionInfo);
1392 }
1393 }
1394 } else if (rs.isMergingNew()) {
1395 synchronized (regionStates) {
1396 String p = regionInfo.getEncodedName();
1397 PairOfSameType<HRegionInfo> regions = mergingRegions.get(p);
1398 if (regions != null) {
1399 onlineMergingRegion(disabled, regions.getFirst(), serverName);
1400 onlineMergingRegion(disabled, regions.getSecond(), serverName);
1401 }
1402 }
1403 }
1404 }
1405 } finally {
1406 lock.unlock();
1407 }
1408 }
1409
1410 private void onlineMergingRegion(boolean disabled,
1411 final HRegionInfo hri, final ServerName serverName) {
1412 RegionState regionState = regionStates.getRegionState(hri);
1413 if (regionState != null && regionState.isMerging()
1414 && regionState.isOnServer(serverName)) {
1415 regionOnline(regionState.getRegion(), serverName);
1416 if (disabled) {
1417 unassign(hri);
1418 }
1419 }
1420 }
1421 });
1422 }
1423 }
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437 @Override
1438 public void nodeChildrenChanged(String path) {
1439 if (path.equals(watcher.assignmentZNode)) {
1440 zkEventWorkers.submit(new Runnable() {
1441 @Override
1442 public void run() {
1443 try {
1444
1445 List<String> children =
1446 ZKUtil.listChildrenAndWatchForNewChildren(
1447 watcher, watcher.assignmentZNode);
1448 if (children != null) {
1449 Stat stat = new Stat();
1450 for (String child : children) {
1451
1452
1453
1454 if (!regionStates.isRegionInTransition(child)) {
1455 ZKAssign.getDataAndWatch(watcher, child, stat);
1456 }
1457 }
1458 }
1459 } catch (KeeperException e) {
1460 server.abort("Unexpected ZK exception reading unassigned children", e);
1461 }
1462 }
1463 });
1464 }
1465 }
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476 void regionOnline(HRegionInfo regionInfo, ServerName sn) {
1477 regionOnline(regionInfo, sn, HConstants.NO_SEQNUM);
1478 }
1479
1480 void regionOnline(HRegionInfo regionInfo, ServerName sn, long openSeqNum) {
1481 numRegionsOpened.incrementAndGet();
1482 regionStates.regionOnline(regionInfo, sn, openSeqNum);
1483
1484
1485 clearRegionPlan(regionInfo);
1486 balancer.regionOnline(regionInfo, sn);
1487
1488
1489 sendRegionOpenedNotification(regionInfo, sn);
1490 }
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500 private void handleAssignmentEvent(final String path) {
1501 if (path.startsWith(watcher.assignmentZNode)) {
1502 final String regionName = ZKAssign.getRegionName(watcher, path);
1503
1504 zkEventWorkersSubmit(new RegionRunnable() {
1505 @Override
1506 public String getRegionName() {
1507 return regionName;
1508 }
1509
1510 @Override
1511 public void run() {
1512 try {
1513 Stat stat = new Stat();
1514 byte [] data = ZKAssign.getDataAndWatch(watcher, path, stat);
1515 if (data == null) return;
1516
1517 RegionTransition rt = RegionTransition.parseFrom(data);
1518
1519
1520
1521 BaseCoordinatedStateManager csm =
1522 (BaseCoordinatedStateManager) server.getCoordinatedStateManager();
1523 OpenRegionCoordination openRegionCoordination = csm.getOpenRegionCoordination();
1524
1525 ZkOpenRegionCoordination.ZkOpenRegionDetails zkOrd =
1526 new ZkOpenRegionCoordination.ZkOpenRegionDetails();
1527 zkOrd.setVersion(stat.getVersion());
1528 zkOrd.setServerName(csm.getServer().getServerName());
1529
1530 handleRegion(rt, openRegionCoordination, zkOrd);
1531 } catch (KeeperException e) {
1532 server.abort("Unexpected ZK exception reading unassigned node data", e);
1533 } catch (DeserializationException e) {
1534 server.abort("Unexpected exception deserializing node data", e);
1535 }
1536 }
1537 });
1538 }
1539 }
1540
1541
1542
1543
1544
1545
1546
1547
1548 public void regionOffline(final HRegionInfo regionInfo) {
1549 regionOffline(regionInfo, null);
1550 }
1551
1552 public void offlineDisabledRegion(HRegionInfo regionInfo) {
1553 if (useZKForAssignment) {
1554
1555 LOG.debug("Table being disabled so deleting ZK node and removing from " +
1556 "regions in transition, skipping assignment of region " +
1557 regionInfo.getRegionNameAsString());
1558 String encodedName = regionInfo.getEncodedName();
1559 deleteNodeInStates(encodedName, "closed", null,
1560 EventType.RS_ZK_REGION_CLOSED, EventType.M_ZK_REGION_OFFLINE);
1561 }
1562 replicasToClose.remove(regionInfo);
1563 regionOffline(regionInfo);
1564 }
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586 public void assign(HRegionInfo region, boolean setOfflineInZK) {
1587 assign(region, setOfflineInZK, false);
1588 }
1589
1590
1591
1592
1593 @VisibleForTesting
1594 public void assign(HRegionInfo region,
1595 boolean setOfflineInZK, boolean forceNewPlan) {
1596 if (isDisabledorDisablingRegionInRIT(region)) {
1597 return;
1598 }
1599 String encodedName = region.getEncodedName();
1600 Lock lock = locker.acquireLock(encodedName);
1601 try {
1602 RegionState state = forceRegionStateToOffline(region, forceNewPlan);
1603 if (state != null) {
1604 if (regionStates.wasRegionOnDeadServer(encodedName)) {
1605 LOG.info("Skip assigning " + region.getRegionNameAsString()
1606 + ", it's host " + regionStates.getLastRegionServerOfRegion(encodedName)
1607 + " is dead but not processed yet");
1608 return;
1609 }
1610 assign(state, setOfflineInZK && useZKForAssignment, forceNewPlan);
1611 }
1612 } finally {
1613 lock.unlock();
1614 }
1615 }
1616
1617
1618
1619
1620
1621
1622
1623 boolean assign(final ServerName destination, final List<HRegionInfo> regions)
1624 throws InterruptedException {
1625 long startTime = EnvironmentEdgeManager.currentTime();
1626 try {
1627 int regionCount = regions.size();
1628 if (regionCount == 0) {
1629 return true;
1630 }
1631 LOG.info("Assigning " + regionCount + " region(s) to " + destination.toString());
1632 Set<String> encodedNames = new HashSet<String>(regionCount);
1633 for (HRegionInfo region : regions) {
1634 encodedNames.add(region.getEncodedName());
1635 }
1636
1637 List<HRegionInfo> failedToOpenRegions = new ArrayList<HRegionInfo>();
1638 Map<String, Lock> locks = locker.acquireLocks(encodedNames);
1639 try {
1640 AtomicInteger counter = new AtomicInteger(0);
1641 Map<String, Integer> offlineNodesVersions = new ConcurrentHashMap<String, Integer>();
1642 OfflineCallback cb = new OfflineCallback(
1643 watcher, destination, counter, offlineNodesVersions);
1644 Map<String, RegionPlan> plans = new HashMap<String, RegionPlan>(regions.size());
1645 List<RegionState> states = new ArrayList<RegionState>(regions.size());
1646 for (HRegionInfo region : regions) {
1647 String encodedName = region.getEncodedName();
1648 if (!isDisabledorDisablingRegionInRIT(region)) {
1649 RegionState state = forceRegionStateToOffline(region, false);
1650 boolean onDeadServer = false;
1651 if (state != null) {
1652 if (regionStates.wasRegionOnDeadServer(encodedName)) {
1653 LOG.info("Skip assigning " + region.getRegionNameAsString()
1654 + ", it's host " + regionStates.getLastRegionServerOfRegion(encodedName)
1655 + " is dead but not processed yet");
1656 onDeadServer = true;
1657 } else if (!useZKForAssignment
1658 || asyncSetOfflineInZooKeeper(state, cb, destination)) {
1659 RegionPlan plan = new RegionPlan(region, state.getServerName(), destination);
1660 plans.put(encodedName, plan);
1661 states.add(state);
1662 continue;
1663 }
1664 }
1665
1666 if (!onDeadServer) {
1667 LOG.info("failed to force region state to offline or "
1668 + "failed to set it offline in ZK, will reassign later: " + region);
1669 failedToOpenRegions.add(region);
1670 }
1671 }
1672
1673
1674 Lock lock = locks.remove(encodedName);
1675 lock.unlock();
1676 }
1677
1678 if (useZKForAssignment) {
1679
1680 int total = states.size();
1681 for (int oldCounter = 0; !server.isStopped();) {
1682 int count = counter.get();
1683 if (oldCounter != count) {
1684 LOG.debug(destination.toString() + " unassigned znodes=" + count +
1685 " of total=" + total + "; oldCounter=" + oldCounter);
1686 oldCounter = count;
1687 }
1688 if (count >= total) break;
1689 Thread.sleep(5);
1690 }
1691 }
1692
1693 if (server.isStopped()) {
1694 return false;
1695 }
1696
1697
1698
1699 this.addPlans(plans);
1700
1701 List<Triple<HRegionInfo, Integer, List<ServerName>>> regionOpenInfos =
1702 new ArrayList<Triple<HRegionInfo, Integer, List<ServerName>>>(states.size());
1703 for (RegionState state: states) {
1704 HRegionInfo region = state.getRegion();
1705 String encodedRegionName = region.getEncodedName();
1706 Integer nodeVersion = offlineNodesVersions.get(encodedRegionName);
1707 if (useZKForAssignment && (nodeVersion == null || nodeVersion == -1)) {
1708 LOG.warn("failed to offline in zookeeper: " + region);
1709 failedToOpenRegions.add(region);
1710 Lock lock = locks.remove(encodedRegionName);
1711 lock.unlock();
1712 } else {
1713 regionStates.updateRegionState(
1714 region, State.PENDING_OPEN, destination);
1715 List<ServerName> favoredNodes = ServerName.EMPTY_SERVER_LIST;
1716 if (this.shouldAssignRegionsWithFavoredNodes) {
1717 favoredNodes = ((FavoredNodeLoadBalancer)this.balancer).getFavoredNodes(region);
1718 }
1719 regionOpenInfos.add(new Triple<HRegionInfo, Integer, List<ServerName>>(
1720 region, nodeVersion, favoredNodes));
1721 }
1722 }
1723
1724
1725 try {
1726
1727
1728 long maxWaitTime = System.currentTimeMillis() +
1729 this.server.getConfiguration().
1730 getLong("hbase.regionserver.rpc.startup.waittime", 60000);
1731 for (int i = 1; i <= maximumAttempts && !server.isStopped(); i++) {
1732 try {
1733
1734 if (regionOpenInfos.isEmpty()) {
1735 break;
1736 }
1737 List<RegionOpeningState> regionOpeningStateList = serverManager
1738 .sendRegionOpen(destination, regionOpenInfos);
1739 if (regionOpeningStateList == null) {
1740
1741 return false;
1742 }
1743 for (int k = 0, n = regionOpeningStateList.size(); k < n; k++) {
1744 RegionOpeningState openingState = regionOpeningStateList.get(k);
1745 if (openingState != RegionOpeningState.OPENED) {
1746 HRegionInfo region = regionOpenInfos.get(k).getFirst();
1747 if (openingState == RegionOpeningState.ALREADY_OPENED) {
1748 processAlreadyOpenedRegion(region, destination);
1749 } else if (openingState == RegionOpeningState.FAILED_OPENING) {
1750
1751 failedToOpenRegions.add(region);
1752 } else {
1753 LOG.warn("THIS SHOULD NOT HAPPEN: unknown opening state "
1754 + openingState + " in assigning region " + region);
1755 }
1756 }
1757 }
1758 break;
1759 } catch (IOException e) {
1760 if (e instanceof RemoteException) {
1761 e = ((RemoteException)e).unwrapRemoteException();
1762 }
1763 if (e instanceof RegionServerStoppedException) {
1764 LOG.warn("The region server was shut down, ", e);
1765
1766 return false;
1767 } else if (e instanceof ServerNotRunningYetException) {
1768 long now = System.currentTimeMillis();
1769 if (now < maxWaitTime) {
1770 LOG.debug("Server is not yet up; waiting up to " +
1771 (maxWaitTime - now) + "ms", e);
1772 Thread.sleep(100);
1773 i--;
1774 continue;
1775 }
1776 } else if (e instanceof java.net.SocketTimeoutException
1777 && this.serverManager.isServerOnline(destination)) {
1778
1779
1780
1781
1782 if (LOG.isDebugEnabled()) {
1783 LOG.debug("Bulk assigner openRegion() to " + destination
1784 + " has timed out, but the regions might"
1785 + " already be opened on it.", e);
1786 }
1787
1788 Thread.sleep(100);
1789 i--;
1790 continue;
1791 }
1792 throw e;
1793 }
1794 }
1795 } catch (IOException e) {
1796
1797 LOG.info("Unable to communicate with " + destination
1798 + " in order to assign regions, ", e);
1799 return false;
1800 }
1801 } finally {
1802 for (Lock lock : locks.values()) {
1803 lock.unlock();
1804 }
1805 }
1806
1807 if (!failedToOpenRegions.isEmpty()) {
1808 for (HRegionInfo region : failedToOpenRegions) {
1809 if (!regionStates.isRegionOnline(region)) {
1810 invokeAssign(region);
1811 }
1812 }
1813 }
1814
1815
1816 ArrayList<HRegionInfo> userRegionSet = new ArrayList<HRegionInfo>(regions.size());
1817 for (HRegionInfo region: regions) {
1818 if (!region.getTable().isSystemTable()) {
1819 userRegionSet.add(region);
1820 }
1821 }
1822 if (!waitForAssignment(userRegionSet, true, userRegionSet.size(),
1823 System.currentTimeMillis())) {
1824 LOG.debug("some user regions are still in transition: " + userRegionSet);
1825 }
1826 LOG.debug("Bulk assigning done for " + destination);
1827 return true;
1828 } finally {
1829 metricsAssignmentManager.updateBulkAssignTime(EnvironmentEdgeManager.currentTime() - startTime);
1830 }
1831 }
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843 private void unassign(final HRegionInfo region,
1844 final RegionState state, final int versionOfClosingNode,
1845 final ServerName dest, final boolean transitionInZK,
1846 final ServerName src) {
1847 ServerName server = src;
1848 if (state != null) {
1849 server = state.getServerName();
1850 }
1851 long maxWaitTime = -1;
1852 for (int i = 1; i <= this.maximumAttempts; i++) {
1853 if (this.server.isStopped() || this.server.isAborted()) {
1854 LOG.debug("Server stopped/aborted; skipping unassign of " + region);
1855 return;
1856 }
1857
1858 if (!serverManager.isServerOnline(server)) {
1859 LOG.debug("Offline " + region.getRegionNameAsString()
1860 + ", no need to unassign since it's on a dead server: " + server);
1861 if (transitionInZK) {
1862
1863 deleteClosingOrClosedNode(region, server);
1864 }
1865 if (state != null) {
1866 regionOffline(region);
1867 }
1868 return;
1869 }
1870 try {
1871
1872 if (serverManager.sendRegionClose(server, region,
1873 versionOfClosingNode, dest, transitionInZK)) {
1874 LOG.debug("Sent CLOSE to " + server + " for region " +
1875 region.getRegionNameAsString());
1876 if (useZKForAssignment && !transitionInZK && state != null) {
1877
1878
1879 unassign(region, state, versionOfClosingNode,
1880 dest, transitionInZK, src);
1881 }
1882 return;
1883 }
1884
1885
1886 LOG.warn("Server " + server + " region CLOSE RPC returned false for " +
1887 region.getRegionNameAsString());
1888 } catch (Throwable t) {
1889 long sleepTime = 0;
1890 Configuration conf = this.server.getConfiguration();
1891 if (t instanceof RemoteException) {
1892 t = ((RemoteException)t).unwrapRemoteException();
1893 }
1894 boolean logRetries = true;
1895 if (t instanceof RegionServerAbortedException
1896 || t instanceof RegionServerStoppedException
1897 || t instanceof ServerNotRunningYetException) {
1898
1899
1900 sleepTime = 1 + conf.getInt(RpcClient.FAILED_SERVER_EXPIRY_KEY,
1901 RpcClient.FAILED_SERVER_EXPIRY_DEFAULT);
1902
1903 } else if (t instanceof NotServingRegionException) {
1904 LOG.debug("Offline " + region.getRegionNameAsString()
1905 + ", it's not any more on " + server, t);
1906 if (transitionInZK) {
1907 deleteClosingOrClosedNode(region, server);
1908 }
1909 if (state != null) {
1910 regionOffline(region);
1911 }
1912 return;
1913 } else if ((t instanceof FailedServerException) || (state != null &&
1914 t instanceof RegionAlreadyInTransitionException)) {
1915 if(t instanceof FailedServerException) {
1916 sleepTime = 1 + conf.getInt(RpcClient.FAILED_SERVER_EXPIRY_KEY,
1917 RpcClient.FAILED_SERVER_EXPIRY_DEFAULT);
1918 } else {
1919
1920 LOG.debug("update " + state + " the timestamp.");
1921 state.updateTimestampToNow();
1922 if (maxWaitTime < 0) {
1923 maxWaitTime =
1924 EnvironmentEdgeManager.currentTime()
1925 + conf.getLong(ALREADY_IN_TRANSITION_WAITTIME,
1926 DEFAULT_ALREADY_IN_TRANSITION_WAITTIME);
1927 }
1928 long now = EnvironmentEdgeManager.currentTime();
1929 if (now < maxWaitTime) {
1930 LOG.debug("Region is already in transition; "
1931 + "waiting up to " + (maxWaitTime - now) + "ms", t);
1932 sleepTime = 100;
1933 i--;
1934 logRetries = false;
1935 }
1936 }
1937 }
1938
1939 try {
1940 if (sleepTime > 0) {
1941 Thread.sleep(sleepTime);
1942 }
1943 } catch (InterruptedException ie) {
1944 LOG.warn("Failed to unassign "
1945 + region.getRegionNameAsString() + " since interrupted", ie);
1946 Thread.currentThread().interrupt();
1947 if (state != null) {
1948 regionStates.updateRegionState(region, State.FAILED_CLOSE);
1949 }
1950 return;
1951 }
1952
1953 if (logRetries) {
1954 LOG.info("Server " + server + " returned " + t + " for "
1955 + region.getRegionNameAsString() + ", try=" + i
1956 + " of " + this.maximumAttempts, t);
1957
1958 }
1959 }
1960 }
1961
1962 if (state != null) {
1963 regionStates.updateRegionState(region, State.FAILED_CLOSE);
1964 }
1965 }
1966
1967
1968
1969
1970 private RegionState forceRegionStateToOffline(
1971 final HRegionInfo region, final boolean forceNewPlan) {
1972 RegionState state = regionStates.getRegionState(region);
1973 if (state == null) {
1974 LOG.warn("Assigning but not in region states: " + region);
1975 state = regionStates.createRegionState(region);
1976 }
1977
1978 ServerName sn = state.getServerName();
1979 if (forceNewPlan && LOG.isDebugEnabled()) {
1980 LOG.debug("Force region state offline " + state);
1981 }
1982
1983 switch (state.getState()) {
1984 case OPEN:
1985 case OPENING:
1986 case PENDING_OPEN:
1987 case CLOSING:
1988 case PENDING_CLOSE:
1989 if (!forceNewPlan) {
1990 LOG.debug("Skip assigning " +
1991 region + ", it is already " + state);
1992 return null;
1993 }
1994 case FAILED_CLOSE:
1995 case FAILED_OPEN:
1996 unassign(region, state, -1, null, false, null);
1997 state = regionStates.getRegionState(region);
1998 if (state.isFailedClose()) {
1999
2000
2001 LOG.info("Skip assigning " +
2002 region + ", we couldn't close it: " + state);
2003 return null;
2004 }
2005 case OFFLINE:
2006
2007
2008
2009
2010
2011
2012
2013
2014 if (useZKForAssignment
2015 && regionStates.isServerDeadAndNotProcessed(sn)
2016 && wasRegionOnDeadServerByMeta(region, sn)) {
2017 if (!regionStates.isRegionInTransition(region)) {
2018 LOG.info("Updating the state to " + State.OFFLINE + " to allow to be reassigned by SSH");
2019 regionStates.updateRegionState(region, State.OFFLINE);
2020 }
2021 LOG.info("Skip assigning " + region.getRegionNameAsString()
2022 + ", it is on a dead but not processed yet server: " + sn);
2023 return null;
2024 }
2025 case CLOSED:
2026 break;
2027 default:
2028 LOG.error("Trying to assign region " + region
2029 + ", which is " + state);
2030 return null;
2031 }
2032 return state;
2033 }
2034
2035 @SuppressWarnings("deprecation")
2036 protected boolean wasRegionOnDeadServerByMeta(
2037 final HRegionInfo region, final ServerName sn) {
2038 try {
2039 if (region.isMetaRegion()) {
2040 ServerName server = this.server.getMetaTableLocator().
2041 getMetaRegionLocation(this.server.getZooKeeper());
2042 return regionStates.isServerDeadAndNotProcessed(server);
2043 }
2044 while (!server.isStopped()) {
2045 try {
2046 this.server.getMetaTableLocator().waitMetaRegionLocation(server.getZooKeeper());
2047 Result r = MetaTableAccessor.getRegionResult(server.getConnection(),
2048 region.getRegionName());
2049 if (r == null || r.isEmpty()) return false;
2050 ServerName server = HRegionInfo.getServerName(r);
2051 return regionStates.isServerDeadAndNotProcessed(server);
2052 } catch (IOException ioe) {
2053 LOG.info("Received exception accessing hbase:meta during force assign "
2054 + region.getRegionNameAsString() + ", retrying", ioe);
2055 }
2056 }
2057 } catch (InterruptedException e) {
2058 Thread.currentThread().interrupt();
2059 LOG.info("Interrupted accessing hbase:meta", e);
2060 }
2061
2062 return regionStates.isServerDeadAndNotProcessed(sn);
2063 }
2064
2065
2066
2067
2068
2069
2070
2071 public void assign(RegionState state,
2072 boolean setOfflineInZK, final boolean forceNewPlan) {
2073 long startTime = EnvironmentEdgeManager.currentTime();
2074 try {
2075 Configuration conf = server.getConfiguration();
2076 RegionState currentState = state;
2077 int versionOfOfflineNode = -1;
2078 RegionPlan plan = null;
2079 long maxWaitTime = -1;
2080 HRegionInfo region = state.getRegion();
2081 RegionOpeningState regionOpenState;
2082 Throwable previousException = null;
2083 for (int i = 1; i <= maximumAttempts; i++) {
2084 if (server.isStopped() || server.isAborted()) {
2085 LOG.info("Skip assigning " + region.getRegionNameAsString()
2086 + ", the server is stopped/aborted");
2087 return;
2088 }
2089
2090 if (plan == null) {
2091 try {
2092 plan = getRegionPlan(region, forceNewPlan);
2093 } catch (HBaseIOException e) {
2094 LOG.warn("Failed to get region plan", e);
2095 }
2096 }
2097
2098 if (plan == null) {
2099 LOG.warn("Unable to determine a plan to assign " + region);
2100
2101
2102 if (region.isMetaRegion()) {
2103 if (i == maximumAttempts) {
2104 i = 0;
2105
2106 LOG.warn("Unable to determine a plan to assign a hbase:meta region " + region +
2107 " after maximumAttempts (" + this.maximumAttempts +
2108 "). Reset attempts count and continue retrying.");
2109 }
2110 waitForRetryingMetaAssignment();
2111 continue;
2112 }
2113
2114 regionStates.updateRegionState(region, State.FAILED_OPEN);
2115 return;
2116 }
2117 if (setOfflineInZK && versionOfOfflineNode == -1) {
2118 LOG.info("Setting node as OFFLINED in ZooKeeper for region " + region);
2119
2120
2121 versionOfOfflineNode = setOfflineInZooKeeper(currentState, plan.getDestination());
2122 if (versionOfOfflineNode != -1) {
2123 if (isDisabledorDisablingRegionInRIT(region)) {
2124 return;
2125 }
2126
2127
2128
2129
2130
2131
2132 TableName tableName = region.getTable();
2133 if (!tableStateManager.isTableState(tableName,
2134 ZooKeeperProtos.Table.State.ENABLED, ZooKeeperProtos.Table.State.ENABLING)) {
2135 LOG.debug("Setting table " + tableName + " to ENABLED state.");
2136 setEnabledTable(tableName);
2137 }
2138 }
2139 }
2140 if (setOfflineInZK && versionOfOfflineNode == -1) {
2141 LOG.info("Unable to set offline in ZooKeeper to assign " + region);
2142
2143
2144
2145
2146 if (!server.isAborted()) {
2147 continue;
2148 }
2149 }
2150 LOG.info("Assigning " + region.getRegionNameAsString() +
2151 " to " + plan.getDestination());
2152
2153 currentState = regionStates.updateRegionState(region,
2154 State.PENDING_OPEN, plan.getDestination());
2155
2156 boolean needNewPlan;
2157 final String assignMsg = "Failed assignment of " + region.getRegionNameAsString() +
2158 " to " + plan.getDestination();
2159 try {
2160 List<ServerName> favoredNodes = ServerName.EMPTY_SERVER_LIST;
2161 if (this.shouldAssignRegionsWithFavoredNodes) {
2162 favoredNodes = ((FavoredNodeLoadBalancer)this.balancer).getFavoredNodes(region);
2163 }
2164 regionOpenState = serverManager.sendRegionOpen(
2165 plan.getDestination(), region, versionOfOfflineNode, favoredNodes);
2166
2167 if (regionOpenState == RegionOpeningState.FAILED_OPENING) {
2168
2169 needNewPlan = true;
2170 LOG.warn(assignMsg + ", regionserver says 'FAILED_OPENING', " +
2171 " trying to assign elsewhere instead; " +
2172 "try=" + i + " of " + this.maximumAttempts);
2173 } else {
2174
2175 if (regionOpenState == RegionOpeningState.ALREADY_OPENED) {
2176 processAlreadyOpenedRegion(region, plan.getDestination());
2177 }
2178 return;
2179 }
2180
2181 } catch (Throwable t) {
2182 if (t instanceof RemoteException) {
2183 t = ((RemoteException) t).unwrapRemoteException();
2184 }
2185 previousException = t;
2186
2187
2188
2189
2190 boolean hold = (t instanceof RegionAlreadyInTransitionException ||
2191 t instanceof ServerNotRunningYetException);
2192
2193
2194
2195
2196
2197
2198 boolean retry = !hold && (t instanceof java.net.SocketTimeoutException
2199 && this.serverManager.isServerOnline(plan.getDestination()));
2200
2201
2202 if (hold) {
2203 LOG.warn(assignMsg + ", waiting a little before trying on the same region server " +
2204 "try=" + i + " of " + this.maximumAttempts, t);
2205
2206 if (maxWaitTime < 0) {
2207 if (t instanceof RegionAlreadyInTransitionException) {
2208 maxWaitTime = EnvironmentEdgeManager.currentTime()
2209 + this.server.getConfiguration().getLong(ALREADY_IN_TRANSITION_WAITTIME,
2210 DEFAULT_ALREADY_IN_TRANSITION_WAITTIME);
2211 } else {
2212 maxWaitTime = EnvironmentEdgeManager.currentTime()
2213 + this.server.getConfiguration().getLong(
2214 "hbase.regionserver.rpc.startup.waittime", 60000);
2215 }
2216 }
2217 try {
2218 needNewPlan = false;
2219 long now = EnvironmentEdgeManager.currentTime();
2220 if (now < maxWaitTime) {
2221 LOG.debug("Server is not yet up or region is already in transition; "
2222 + "waiting up to " + (maxWaitTime - now) + "ms", t);
2223 Thread.sleep(100);
2224 i--;
2225 } else if (!(t instanceof RegionAlreadyInTransitionException)) {
2226 LOG.debug("Server is not up for a while; try a new one", t);
2227 needNewPlan = true;
2228 }
2229 } catch (InterruptedException ie) {
2230 LOG.warn("Failed to assign "
2231 + region.getRegionNameAsString() + " since interrupted", ie);
2232 regionStates.updateRegionState(region, State.FAILED_OPEN);
2233 Thread.currentThread().interrupt();
2234 return;
2235 }
2236 } else if (retry) {
2237 needNewPlan = false;
2238 i--;
2239 LOG.warn(assignMsg + ", trying to assign to the same region server due ", t);
2240 } else {
2241 needNewPlan = true;
2242 LOG.warn(assignMsg + ", trying to assign elsewhere instead;" +
2243 " try=" + i + " of " + this.maximumAttempts, t);
2244 }
2245 }
2246
2247 if (i == this.maximumAttempts) {
2248
2249 if (region.isMetaRegion()) {
2250 i = 0;
2251 LOG.warn(assignMsg +
2252 ", trying to assign a hbase:meta region reached to maximumAttempts (" +
2253 this.maximumAttempts + "). Reset attempt counts and continue retrying.");
2254 waitForRetryingMetaAssignment();
2255 }
2256 else {
2257
2258
2259 continue;
2260 }
2261 }
2262
2263
2264
2265
2266 if (needNewPlan) {
2267
2268
2269
2270
2271 RegionPlan newPlan = null;
2272 try {
2273 newPlan = getRegionPlan(region, true);
2274 } catch (HBaseIOException e) {
2275 LOG.warn("Failed to get region plan", e);
2276 }
2277 if (newPlan == null) {
2278 regionStates.updateRegionState(region, State.FAILED_OPEN);
2279 LOG.warn("Unable to find a viable location to assign region " +
2280 region.getRegionNameAsString());
2281 return;
2282 }
2283
2284 if (plan != newPlan && !plan.getDestination().equals(newPlan.getDestination())) {
2285
2286
2287
2288 LOG.info("Region assignment plan changed from " + plan.getDestination() + " to "
2289 + newPlan.getDestination() + " server.");
2290 currentState = regionStates.updateRegionState(region, State.OFFLINE);
2291 versionOfOfflineNode = -1;
2292 if (useZKForAssignment) {
2293 setOfflineInZK = true;
2294 }
2295 plan = newPlan;
2296 } else if(plan.getDestination().equals(newPlan.getDestination()) &&
2297 previousException instanceof FailedServerException) {
2298 try {
2299 LOG.info("Trying to re-assign " + region.getRegionNameAsString() +
2300 " to the same failed server.");
2301 Thread.sleep(1 + conf.getInt(RpcClient.FAILED_SERVER_EXPIRY_KEY,
2302 RpcClient.FAILED_SERVER_EXPIRY_DEFAULT));
2303 } catch (InterruptedException ie) {
2304 LOG.warn("Failed to assign "
2305 + region.getRegionNameAsString() + " since interrupted", ie);
2306 regionStates.updateRegionState(region, State.FAILED_OPEN);
2307 Thread.currentThread().interrupt();
2308 return;
2309 }
2310 }
2311 }
2312 }
2313
2314 regionStates.updateRegionState(region, State.FAILED_OPEN);
2315 } finally {
2316 metricsAssignmentManager.updateAssignmentTime(EnvironmentEdgeManager.currentTime() - startTime);
2317 }
2318 }
2319
2320 private void processAlreadyOpenedRegion(HRegionInfo region, ServerName sn) {
2321
2322
2323
2324 LOG.debug("ALREADY_OPENED " + region.getRegionNameAsString()
2325 + " to " + sn);
2326 String encodedName = region.getEncodedName();
2327
2328
2329
2330 if (useZKForAssignment) {
2331 String node = ZKAssign.getNodeName(watcher, encodedName);
2332 Stat stat = new Stat();
2333 try {
2334 byte[] existingBytes = ZKUtil.getDataNoWatch(watcher, node, stat);
2335 if (existingBytes != null) {
2336 RegionTransition rt= RegionTransition.parseFrom(existingBytes);
2337 EventType et = rt.getEventType();
2338 if (et.equals(EventType.RS_ZK_REGION_OPENED)) {
2339 LOG.debug("ALREADY_OPENED " + region.getRegionNameAsString()
2340 + " and node in " + et + " state");
2341 return;
2342 }
2343 }
2344 } catch (KeeperException ke) {
2345 LOG.warn("Unexpected ZK exception getData " + node
2346 + " node for the region " + encodedName, ke);
2347 } catch (DeserializationException e) {
2348 LOG.warn("Get RegionTransition from zk deserialization failed! ", e);
2349 }
2350 deleteNodeInStates(encodedName, "offline", sn, EventType.M_ZK_REGION_OFFLINE);
2351 }
2352
2353 regionStates.regionOnline(region, sn);
2354 }
2355
2356 private boolean isDisabledorDisablingRegionInRIT(final HRegionInfo region) {
2357 if (this.tableStateManager.isTableState(region.getTable(),
2358 ZooKeeperProtos.Table.State.DISABLED,
2359 ZooKeeperProtos.Table.State.DISABLING) || replicasToClose.contains(region)) {
2360 LOG.info("Table " + region.getTable() + " is disabled or disabling;"
2361 + " skipping assign of " + region.getRegionNameAsString());
2362 offlineDisabledRegion(region);
2363 return true;
2364 }
2365 return false;
2366 }
2367
2368
2369
2370
2371
2372
2373
2374
2375 private int setOfflineInZooKeeper(final RegionState state, final ServerName destination) {
2376 if (!state.isClosed() && !state.isOffline()) {
2377 String msg = "Unexpected state : " + state + " .. Cannot transit it to OFFLINE.";
2378 this.server.abort(msg, new IllegalStateException(msg));
2379 return -1;
2380 }
2381 regionStates.updateRegionState(state.getRegion(), State.OFFLINE);
2382 int versionOfOfflineNode;
2383 try {
2384
2385 versionOfOfflineNode = ZKAssign.createOrForceNodeOffline(watcher,
2386 state.getRegion(), destination);
2387 if (versionOfOfflineNode == -1) {
2388 LOG.warn("Attempted to create/force node into OFFLINE state before "
2389 + "completing assignment but failed to do so for " + state);
2390 return -1;
2391 }
2392 } catch (KeeperException e) {
2393 server.abort("Unexpected ZK exception creating/setting node OFFLINE", e);
2394 return -1;
2395 }
2396 return versionOfOfflineNode;
2397 }
2398
2399
2400
2401
2402
2403
2404 private RegionPlan getRegionPlan(final HRegionInfo region,
2405 final boolean forceNewPlan) throws HBaseIOException {
2406 return getRegionPlan(region, null, forceNewPlan);
2407 }
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418 private RegionPlan getRegionPlan(final HRegionInfo region,
2419 final ServerName serverToExclude, final boolean forceNewPlan) throws HBaseIOException {
2420
2421 final String encodedName = region.getEncodedName();
2422 final List<ServerName> destServers =
2423 serverManager.createDestinationServersList(serverToExclude);
2424
2425 if (destServers.isEmpty()){
2426 LOG.warn("Can't move " + encodedName +
2427 ", there is no destination server available.");
2428 return null;
2429 }
2430
2431 RegionPlan randomPlan = null;
2432 boolean newPlan = false;
2433 RegionPlan existingPlan;
2434
2435 synchronized (this.regionPlans) {
2436 existingPlan = this.regionPlans.get(encodedName);
2437
2438 if (existingPlan != null && existingPlan.getDestination() != null) {
2439 LOG.debug("Found an existing plan for " + region.getRegionNameAsString()
2440 + " destination server is " + existingPlan.getDestination() +
2441 " accepted as a dest server = " + destServers.contains(existingPlan.getDestination()));
2442 }
2443
2444 if (forceNewPlan
2445 || existingPlan == null
2446 || existingPlan.getDestination() == null
2447 || !destServers.contains(existingPlan.getDestination())) {
2448 newPlan = true;
2449 }
2450 }
2451
2452 if (newPlan) {
2453 ServerName destination = null;
2454 try {
2455 destination = balancer.randomAssignment(region, destServers);
2456 } catch (IOException ex) {
2457 LOG.warn("Failed to create new plan.",ex);
2458 return null;
2459 }
2460 if (destination == null) {
2461 LOG.warn("Can't find a destination for " + encodedName);
2462 return null;
2463 }
2464 synchronized (this.regionPlans) {
2465 randomPlan = new RegionPlan(region, null, destination);
2466 if (!region.isMetaTable() && shouldAssignRegionsWithFavoredNodes) {
2467 List<HRegionInfo> regions = new ArrayList<HRegionInfo>(1);
2468 regions.add(region);
2469 try {
2470 processFavoredNodes(regions);
2471 } catch (IOException ie) {
2472 LOG.warn("Ignoring exception in processFavoredNodes " + ie);
2473 }
2474 }
2475 this.regionPlans.put(encodedName, randomPlan);
2476 }
2477 LOG.debug("No previous transition plan found (or ignoring " + "an existing plan) for "
2478 + region.getRegionNameAsString() + "; generated random plan=" + randomPlan + "; "
2479 + destServers.size() + " (online=" + serverManager.getOnlineServers().size()
2480 + ") available servers, forceNewPlan=" + forceNewPlan);
2481 return randomPlan;
2482 }
2483 LOG.debug("Using pre-existing plan for " +
2484 region.getRegionNameAsString() + "; plan=" + existingPlan);
2485 return existingPlan;
2486 }
2487
2488
2489
2490
2491 private void waitForRetryingMetaAssignment() {
2492 try {
2493 Thread.sleep(this.sleepTimeBeforeRetryingMetaAssignment);
2494 } catch (InterruptedException e) {
2495 LOG.error("Got exception while waiting for hbase:meta assignment");
2496 Thread.currentThread().interrupt();
2497 }
2498 }
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513 public void unassign(HRegionInfo region) {
2514 unassign(region, false);
2515 }
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532 public void unassign(HRegionInfo region, boolean force, ServerName dest) {
2533
2534 LOG.debug("Starting unassign of " + region.getRegionNameAsString()
2535 + " (offlining), current state: " + regionStates.getRegionState(region));
2536
2537 String encodedName = region.getEncodedName();
2538
2539 int versionOfClosingNode = -1;
2540
2541
2542 ReentrantLock lock = locker.acquireLock(encodedName);
2543 RegionState state = regionStates.getRegionTransitionState(encodedName);
2544 boolean reassign = true;
2545 try {
2546 if (state == null) {
2547
2548
2549 state = regionStates.getRegionState(encodedName);
2550 if (state != null && state.isUnassignable()) {
2551 LOG.info("Attempting to unassign " + state + ", ignored");
2552
2553 return;
2554 }
2555
2556 try {
2557 if (state == null || state.getServerName() == null) {
2558
2559
2560 LOG.warn("Attempting to unassign a region not in RegionStates "
2561 + region.getRegionNameAsString() + ", offlined");
2562 regionOffline(region);
2563 return;
2564 }
2565 if (useZKForAssignment) {
2566 versionOfClosingNode = ZKAssign.createNodeClosing(
2567 watcher, region, state.getServerName());
2568 if (versionOfClosingNode == -1) {
2569 LOG.info("Attempting to unassign " +
2570 region.getRegionNameAsString() + " but ZK closing node "
2571 + "can't be created.");
2572 reassign = false;
2573 return;
2574 }
2575 }
2576 } catch (KeeperException e) {
2577 if (e instanceof NodeExistsException) {
2578
2579
2580
2581
2582 NodeExistsException nee = (NodeExistsException)e;
2583 String path = nee.getPath();
2584 try {
2585 if (isSplitOrSplittingOrMergedOrMerging(path)) {
2586 LOG.debug(path + " is SPLIT or SPLITTING or MERGED or MERGING; " +
2587 "skipping unassign because region no longer exists -- its split or merge");
2588 reassign = false;
2589 return;
2590 }
2591 } catch (KeeperException.NoNodeException ke) {
2592 LOG.warn("Failed getData on SPLITTING/SPLIT at " + path +
2593 "; presuming split and that the region to unassign, " +
2594 encodedName + ", no longer exists -- confirm", ke);
2595 return;
2596 } catch (KeeperException ke) {
2597 LOG.error("Unexpected zk state", ke);
2598 } catch (DeserializationException de) {
2599 LOG.error("Failed parse", de);
2600 }
2601 }
2602
2603 server.abort("Unexpected ZK exception creating node CLOSING", e);
2604 reassign = false;
2605 return;
2606 }
2607 state = regionStates.updateRegionState(region, State.PENDING_CLOSE);
2608 } else if (state.isFailedOpen()) {
2609
2610 regionOffline(region);
2611 return;
2612 } else if (force && state.isPendingCloseOrClosing()) {
2613 LOG.debug("Attempting to unassign " + region.getRegionNameAsString() +
2614 " which is already " + state.getState() +
2615 " but forcing to send a CLOSE RPC again ");
2616 if (state.isFailedClose()) {
2617 state = regionStates.updateRegionState(region, State.PENDING_CLOSE);
2618 }
2619 state.updateTimestampToNow();
2620 } else {
2621 LOG.debug("Attempting to unassign " +
2622 region.getRegionNameAsString() + " but it is " +
2623 "already in transition (" + state.getState() + ", force=" + force + ")");
2624 return;
2625 }
2626
2627 unassign(region, state, versionOfClosingNode, dest, useZKForAssignment, null);
2628 } finally {
2629 lock.unlock();
2630
2631
2632 if (!replicasToClose.contains(region) && reassign && regionStates.isRegionOffline(region)) {
2633 assign(region, true);
2634 }
2635 }
2636 }
2637
2638 public void unassign(HRegionInfo region, boolean force){
2639 unassign(region, force, null);
2640 }
2641
2642
2643
2644
2645 public void deleteClosingOrClosedNode(HRegionInfo region, ServerName sn) {
2646 String encodedName = region.getEncodedName();
2647 deleteNodeInStates(encodedName, "closing", sn, EventType.M_ZK_REGION_CLOSING,
2648 EventType.RS_ZK_REGION_CLOSED);
2649 }
2650
2651
2652
2653
2654
2655
2656
2657 private boolean isSplitOrSplittingOrMergedOrMerging(final String path)
2658 throws KeeperException, DeserializationException {
2659 boolean result = false;
2660
2661
2662 byte [] data = ZKAssign.getData(watcher, path);
2663 if (data == null) {
2664 LOG.info("Node " + path + " is gone");
2665 return false;
2666 }
2667 RegionTransition rt = RegionTransition.parseFrom(data);
2668 switch (rt.getEventType()) {
2669 case RS_ZK_REQUEST_REGION_SPLIT:
2670 case RS_ZK_REGION_SPLIT:
2671 case RS_ZK_REGION_SPLITTING:
2672 case RS_ZK_REQUEST_REGION_MERGE:
2673 case RS_ZK_REGION_MERGED:
2674 case RS_ZK_REGION_MERGING:
2675 result = true;
2676 break;
2677 default:
2678 LOG.info("Node " + path + " is in " + rt.getEventType());
2679 break;
2680 }
2681 return result;
2682 }
2683
2684
2685
2686
2687
2688
2689 public int getNumRegionsOpened() {
2690 return numRegionsOpened.get();
2691 }
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702 public boolean waitForAssignment(HRegionInfo regionInfo)
2703 throws InterruptedException {
2704 ArrayList<HRegionInfo> regionSet = new ArrayList<HRegionInfo>(1);
2705 regionSet.add(regionInfo);
2706 return waitForAssignment(regionSet, true, Long.MAX_VALUE);
2707 }
2708
2709
2710
2711
2712 protected boolean waitForAssignment(final Collection<HRegionInfo> regionSet,
2713 final boolean waitTillAllAssigned, final int reassigningRegions,
2714 final long minEndTime) throws InterruptedException {
2715 long deadline = minEndTime + bulkPerRegionOpenTimeGuesstimate * (reassigningRegions + 1);
2716 if (deadline < 0) {
2717 deadline = Long.MAX_VALUE;
2718 }
2719 return waitForAssignment(regionSet, waitTillAllAssigned, deadline);
2720 }
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730 protected boolean waitForAssignment(final Collection<HRegionInfo> regionSet,
2731 final boolean waitTillAllAssigned, final long deadline) throws InterruptedException {
2732
2733 while (!regionSet.isEmpty() && !server.isStopped() && deadline > System.currentTimeMillis()) {
2734 int failedOpenCount = 0;
2735 Iterator<HRegionInfo> regionInfoIterator = regionSet.iterator();
2736 while (regionInfoIterator.hasNext()) {
2737 HRegionInfo hri = regionInfoIterator.next();
2738 if (regionStates.isRegionOnline(hri) || regionStates.isRegionInState(hri,
2739 State.SPLITTING, State.SPLIT, State.MERGING, State.MERGED)) {
2740 regionInfoIterator.remove();
2741 } else if (regionStates.isRegionInState(hri, State.FAILED_OPEN)) {
2742 failedOpenCount++;
2743 }
2744 }
2745 if (!waitTillAllAssigned) {
2746
2747 break;
2748 }
2749 if (!regionSet.isEmpty()) {
2750 if (failedOpenCount == regionSet.size()) {
2751
2752 break;
2753 }
2754 regionStates.waitForUpdate(100);
2755 }
2756 }
2757 return regionSet.isEmpty();
2758 }
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771 public void assignMeta(HRegionInfo hri) throws KeeperException {
2772 this.server.getMetaTableLocator().deleteMetaLocation(this.watcher, hri.getReplicaId());
2773 assign(hri, true);
2774 }
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784 public void assign(Map<HRegionInfo, ServerName> regions)
2785 throws IOException, InterruptedException {
2786 if (regions == null || regions.isEmpty()) {
2787 return;
2788 }
2789 List<ServerName> servers = serverManager.createDestinationServersList();
2790 if (servers == null || servers.isEmpty()) {
2791 throw new IOException("Found no destination server to assign region(s)");
2792 }
2793
2794
2795 Map<ServerName, List<HRegionInfo>> bulkPlan =
2796 balancer.retainAssignment(regions, servers);
2797 if (bulkPlan == null) {
2798 throw new IOException("Unable to determine a plan to assign region(s)");
2799 }
2800
2801 processBogusAssignments(bulkPlan);
2802
2803 assign(regions.size(), servers.size(),
2804 "retainAssignment=true", bulkPlan);
2805 }
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815 public void assign(List<HRegionInfo> regions)
2816 throws IOException, InterruptedException {
2817 if (regions == null || regions.isEmpty()) {
2818 return;
2819 }
2820
2821 List<ServerName> servers = serverManager.createDestinationServersList();
2822 if (servers == null || servers.isEmpty()) {
2823 throw new IOException("Found no destination server to assign region(s)");
2824 }
2825
2826
2827 Map<ServerName, List<HRegionInfo>> bulkPlan
2828 = balancer.roundRobinAssignment(regions, servers);
2829 if (bulkPlan == null) {
2830 throw new IOException("Unable to determine a plan to assign region(s)");
2831 }
2832
2833 processBogusAssignments(bulkPlan);
2834
2835 processFavoredNodes(regions);
2836 assign(regions.size(), servers.size(),
2837 "round-robin=true", bulkPlan);
2838 }
2839
2840 private void assign(int regions, int totalServers,
2841 String message, Map<ServerName, List<HRegionInfo>> bulkPlan)
2842 throws InterruptedException, IOException {
2843
2844 int servers = bulkPlan.size();
2845 if (servers == 1 || (regions < bulkAssignThresholdRegions
2846 && servers < bulkAssignThresholdServers)) {
2847
2848
2849
2850 if (LOG.isTraceEnabled()) {
2851 LOG.trace("Not using bulk assignment since we are assigning only " + regions +
2852 " region(s) to " + servers + " server(s)");
2853 }
2854
2855
2856 ArrayList<HRegionInfo> userRegionSet = new ArrayList<HRegionInfo>(regions);
2857 for (Map.Entry<ServerName, List<HRegionInfo>> plan: bulkPlan.entrySet()) {
2858 if (!assign(plan.getKey(), plan.getValue())) {
2859 for (HRegionInfo region: plan.getValue()) {
2860 if (!regionStates.isRegionOnline(region)) {
2861 invokeAssign(region);
2862 if (!region.getTable().isSystemTable()) {
2863 userRegionSet.add(region);
2864 }
2865 }
2866 }
2867 }
2868 }
2869
2870
2871 if (!waitForAssignment(userRegionSet, true, userRegionSet.size(),
2872 System.currentTimeMillis())) {
2873 LOG.debug("some user regions are still in transition: " + userRegionSet);
2874 }
2875 } else {
2876 LOG.info("Bulk assigning " + regions + " region(s) across "
2877 + totalServers + " server(s), " + message);
2878
2879
2880 BulkAssigner ba = new GeneralBulkAssigner(
2881 this.server, bulkPlan, this, bulkAssignWaitTillAllAssigned);
2882 ba.bulkAssign();
2883 LOG.info("Bulk assigning done");
2884 }
2885 }
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896 private void assignAllUserRegions(Map<HRegionInfo, ServerName> allRegions)
2897 throws IOException, InterruptedException {
2898 if (allRegions == null || allRegions.isEmpty()) return;
2899
2900
2901 boolean retainAssignment = server.getConfiguration().
2902 getBoolean("hbase.master.startup.retainassign", true);
2903
2904 Set<HRegionInfo> regionsFromMetaScan = allRegions.keySet();
2905 if (retainAssignment) {
2906 assign(allRegions);
2907 } else {
2908 List<HRegionInfo> regions = new ArrayList<HRegionInfo>(regionsFromMetaScan);
2909 assign(regions);
2910 }
2911
2912 for (HRegionInfo hri : regionsFromMetaScan) {
2913 TableName tableName = hri.getTable();
2914 if (!tableStateManager.isTableState(tableName,
2915 ZooKeeperProtos.Table.State.ENABLED)) {
2916 setEnabledTable(tableName);
2917 }
2918 }
2919
2920 assign(replicaRegionsNotRecordedInMeta(regionsFromMetaScan, (MasterServices)server));
2921 }
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934 public static List<HRegionInfo> replicaRegionsNotRecordedInMeta(
2935 Set<HRegionInfo> regionsRecordedInMeta, MasterServices master)throws IOException {
2936 List<HRegionInfo> regionsNotRecordedInMeta = new ArrayList<HRegionInfo>();
2937 for (HRegionInfo hri : regionsRecordedInMeta) {
2938 TableName table = hri.getTable();
2939 HTableDescriptor htd = master.getTableDescriptors().get(table);
2940
2941 int desiredRegionReplication = htd.getRegionReplication();
2942 for (int i = 0; i < desiredRegionReplication; i++) {
2943 HRegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
2944 if (regionsRecordedInMeta.contains(replica)) continue;
2945 regionsNotRecordedInMeta.add(replica);
2946 }
2947 }
2948 return regionsNotRecordedInMeta;
2949 }
2950
2951
2952
2953
2954
2955
2956
2957 boolean waitUntilNoRegionsInTransition(final long timeout)
2958 throws InterruptedException {
2959
2960
2961
2962
2963
2964
2965 final long endTime = System.currentTimeMillis() + timeout;
2966
2967 while (!this.server.isStopped() && regionStates.isRegionsInTransition()
2968 && endTime > System.currentTimeMillis()) {
2969 regionStates.waitForUpdate(100);
2970 }
2971
2972 return !regionStates.isRegionsInTransition();
2973 }
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983 Set<ServerName> rebuildUserRegions() throws
2984 IOException, KeeperException, CoordinatedStateException {
2985 Set<TableName> disabledOrEnablingTables = tableStateManager.getTablesInStates(
2986 ZooKeeperProtos.Table.State.DISABLED, ZooKeeperProtos.Table.State.ENABLING);
2987
2988 Set<TableName> disabledOrDisablingOrEnabling = tableStateManager.getTablesInStates(
2989 ZooKeeperProtos.Table.State.DISABLED,
2990 ZooKeeperProtos.Table.State.DISABLING,
2991 ZooKeeperProtos.Table.State.ENABLING);
2992
2993
2994 List<Result> results = MetaTableAccessor.fullScanOfMeta(server.getConnection());
2995
2996 Set<ServerName> onlineServers = serverManager.getOnlineServers().keySet();
2997
2998 Set<ServerName> offlineServers = new HashSet<ServerName>();
2999
3000 for (Result result : results) {
3001 if (result == null && LOG.isDebugEnabled()){
3002 LOG.debug("null result from meta - ignoring but this is strange.");
3003 continue;
3004 }
3005
3006
3007
3008 PairOfSameType<HRegionInfo> p = MetaTableAccessor.getMergeRegions(result);
3009 if (p.getFirst() != null && p.getSecond() != null) {
3010 int numReplicas = ((MasterServices)server).getTableDescriptors().get(p.getFirst().
3011 getTable()).getRegionReplication();
3012 for (HRegionInfo merge : p) {
3013 for (int i = 1; i < numReplicas; i++) {
3014 replicasToClose.add(RegionReplicaUtil.getRegionInfoForReplica(merge, i));
3015 }
3016 }
3017 }
3018 RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
3019 if (rl == null) continue;
3020 HRegionLocation[] locations = rl.getRegionLocations();
3021 if (locations == null) continue;
3022 for (HRegionLocation hrl : locations) {
3023 if (hrl == null) continue;
3024 HRegionInfo regionInfo = hrl.getRegionInfo();
3025 if (regionInfo == null) continue;
3026 int replicaId = regionInfo.getReplicaId();
3027 State state = RegionStateStore.getRegionState(result, replicaId);
3028
3029
3030
3031 if (replicaId == 0 && state.equals(State.SPLIT)) {
3032 for (HRegionLocation h : locations) {
3033 replicasToClose.add(h.getRegionInfo());
3034 }
3035 }
3036 ServerName lastHost = hrl.getServerName();
3037 ServerName regionLocation = RegionStateStore.getRegionServer(result, replicaId);
3038 if (tableStateManager.isTableState(regionInfo.getTable(),
3039 ZooKeeperProtos.Table.State.DISABLED)) {
3040
3041
3042 lastHost = null;
3043 regionLocation = null;
3044 }
3045 regionStates.createRegionState(regionInfo, state, regionLocation, lastHost);
3046 if (!regionStates.isRegionInState(regionInfo, State.OPEN)) {
3047
3048 continue;
3049 }
3050 TableName tableName = regionInfo.getTable();
3051 if (!onlineServers.contains(regionLocation)) {
3052
3053 offlineServers.add(regionLocation);
3054 if (useZKForAssignment) {
3055 regionStates.regionOffline(regionInfo);
3056 }
3057 } else if (!disabledOrEnablingTables.contains(tableName)) {
3058
3059
3060 regionStates.regionOnline(regionInfo, regionLocation);
3061 balancer.regionOnline(regionInfo, regionLocation);
3062 } else if (useZKForAssignment) {
3063 regionStates.regionOffline(regionInfo);
3064 }
3065
3066
3067 if (!disabledOrDisablingOrEnabling.contains(tableName)
3068 && !getTableStateManager().isTableState(tableName,
3069 ZooKeeperProtos.Table.State.ENABLED)) {
3070 setEnabledTable(tableName);
3071 }
3072 }
3073 }
3074 return offlineServers;
3075 }
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085 private void recoverTableInDisablingState()
3086 throws KeeperException, IOException, CoordinatedStateException {
3087 Set<TableName> disablingTables =
3088 tableStateManager.getTablesInStates(ZooKeeperProtos.Table.State.DISABLING);
3089 if (disablingTables.size() != 0) {
3090 for (TableName tableName : disablingTables) {
3091
3092 LOG.info("The table " + tableName
3093 + " is in DISABLING state. Hence recovering by moving the table"
3094 + " to DISABLED state.");
3095 new DisableTableHandler(this.server, tableName,
3096 this, tableLockManager, true).prepare().process();
3097 }
3098 }
3099 }
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109 private void recoverTableInEnablingState()
3110 throws KeeperException, IOException, CoordinatedStateException {
3111 Set<TableName> enablingTables = tableStateManager.
3112 getTablesInStates(ZooKeeperProtos.Table.State.ENABLING);
3113 if (enablingTables.size() != 0) {
3114 for (TableName tableName : enablingTables) {
3115
3116 LOG.info("The table " + tableName
3117 + " is in ENABLING state. Hence recovering by moving the table"
3118 + " to ENABLED state.");
3119
3120
3121 EnableTableHandler eth = new EnableTableHandler(this.server, tableName,
3122 this, tableLockManager, true);
3123 try {
3124 eth.prepare();
3125 } catch (TableNotFoundException e) {
3126 LOG.warn("Table " + tableName + " not found in hbase:meta to recover.");
3127 continue;
3128 }
3129 eth.process();
3130 }
3131 }
3132 }
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149 private void processDeadServersAndRecoverLostRegions(
3150 Set<ServerName> deadServers) throws IOException, KeeperException {
3151 if (deadServers != null && !deadServers.isEmpty()) {
3152 for (ServerName serverName: deadServers) {
3153 if (!serverManager.isServerDead(serverName)) {
3154 serverManager.expireServer(serverName);
3155 }
3156 }
3157 }
3158
3159 List<String> nodes = useZKForAssignment ?
3160 ZKUtil.listChildrenAndWatchForNewChildren(watcher, watcher.assignmentZNode)
3161 : ZKUtil.listChildrenNoWatch(watcher, watcher.assignmentZNode);
3162 if (nodes != null && !nodes.isEmpty()) {
3163 for (String encodedRegionName : nodes) {
3164 processRegionInTransition(encodedRegionName, null);
3165 }
3166 } else if (!useZKForAssignment) {
3167 processRegionInTransitionZkLess();
3168 }
3169 }
3170
3171 void processRegionInTransitionZkLess() {
3172
3173
3174
3175
3176
3177 Map<String, RegionState> rits = regionStates.getRegionsInTransition();
3178 for (RegionState regionState : rits.values()) {
3179 LOG.info("Processing " + regionState);
3180 ServerName serverName = regionState.getServerName();
3181
3182
3183 if (serverName != null
3184 && !serverManager.getOnlineServers().containsKey(serverName)) {
3185 LOG.info("Server " + serverName + " isn't online. SSH will handle this");
3186 continue;
3187 }
3188 HRegionInfo regionInfo = regionState.getRegion();
3189 State state = regionState.getState();
3190
3191 switch (state) {
3192 case CLOSED:
3193 invokeAssign(regionInfo);
3194 break;
3195 case PENDING_OPEN:
3196 retrySendRegionOpen(regionState);
3197 break;
3198 case PENDING_CLOSE:
3199 retrySendRegionClose(regionState);
3200 break;
3201 case FAILED_CLOSE:
3202 case FAILED_OPEN:
3203 invokeUnAssign(regionInfo);
3204 break;
3205 default:
3206
3207 }
3208 }
3209 }
3210
3211
3212
3213
3214
3215 private void retrySendRegionOpen(final RegionState regionState) {
3216 this.executorService.submit(
3217 new EventHandler(server, EventType.M_MASTER_RECOVERY) {
3218 @Override
3219 public void process() throws IOException {
3220 HRegionInfo hri = regionState.getRegion();
3221 ServerName serverName = regionState.getServerName();
3222 ReentrantLock lock = locker.acquireLock(hri.getEncodedName());
3223 try {
3224 for (int i = 1; i <= maximumAttempts; i++) {
3225 if (!serverManager.isServerOnline(serverName)
3226 || server.isStopped() || server.isAborted()) {
3227 return;
3228 }
3229 try {
3230 if (!regionState.equals(regionStates.getRegionState(hri))) {
3231 return;
3232 }
3233 List<ServerName> favoredNodes = ServerName.EMPTY_SERVER_LIST;
3234 if (shouldAssignRegionsWithFavoredNodes) {
3235 favoredNodes = ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(hri);
3236 }
3237 RegionOpeningState regionOpenState = serverManager.sendRegionOpen(
3238 serverName, hri, -1, favoredNodes);
3239
3240 if (regionOpenState == RegionOpeningState.FAILED_OPENING) {
3241
3242
3243 LOG.debug("Got failed_opening in retry sendRegionOpen for "
3244 + regionState + ", re-assign it");
3245 invokeAssign(hri, true);
3246 }
3247 return;
3248 } catch (Throwable t) {
3249 if (t instanceof RemoteException) {
3250 t = ((RemoteException) t).unwrapRemoteException();
3251 }
3252
3253 if (t instanceof java.net.SocketTimeoutException
3254 || t instanceof FailedServerException) {
3255 Threads.sleep(100);
3256 continue;
3257 }
3258
3259 LOG.debug("Got exception in retry sendRegionOpen for "
3260 + regionState + ", re-assign it", t);
3261 invokeAssign(hri);
3262 return;
3263 }
3264 }
3265 } finally {
3266 lock.unlock();
3267 }
3268 }
3269 });
3270 }
3271
3272
3273
3274
3275
3276 private void retrySendRegionClose(final RegionState regionState) {
3277 this.executorService.submit(
3278 new EventHandler(server, EventType.M_MASTER_RECOVERY) {
3279 @Override
3280 public void process() throws IOException {
3281 HRegionInfo hri = regionState.getRegion();
3282 ServerName serverName = regionState.getServerName();
3283 ReentrantLock lock = locker.acquireLock(hri.getEncodedName());
3284 try {
3285 for (int i = 1; i <= maximumAttempts; i++) {
3286 if (!serverManager.isServerOnline(serverName)
3287 || server.isStopped() || server.isAborted()) {
3288 return;
3289 }
3290 try {
3291 if (!regionState.equals(regionStates.getRegionState(hri))) {
3292 return;
3293 }
3294 if (!serverManager.sendRegionClose(serverName, hri, -1, null, false)) {
3295
3296 LOG.debug("Got false in retry sendRegionClose for "
3297 + regionState + ", re-close it");
3298 invokeUnAssign(hri);
3299 }
3300 return;
3301 } catch (Throwable t) {
3302 if (t instanceof RemoteException) {
3303 t = ((RemoteException) t).unwrapRemoteException();
3304 }
3305
3306 if (t instanceof java.net.SocketTimeoutException
3307 || t instanceof FailedServerException) {
3308 Threads.sleep(100);
3309 continue;
3310 }
3311 if (!(t instanceof NotServingRegionException
3312 || t instanceof RegionAlreadyInTransitionException)) {
3313
3314
3315
3316 LOG.debug("Got exception in retry sendRegionClose for "
3317 + regionState + ", re-close it", t);
3318 invokeUnAssign(hri);
3319 }
3320 return;
3321 }
3322 }
3323 } finally {
3324 lock.unlock();
3325 }
3326 }
3327 });
3328 }
3329
3330
3331
3332
3333
3334
3335
3336
3337 public void updateRegionsInTransitionMetrics() {
3338 long currentTime = System.currentTimeMillis();
3339 int totalRITs = 0;
3340 int totalRITsOverThreshold = 0;
3341 long oldestRITTime = 0;
3342 int ritThreshold = this.server.getConfiguration().
3343 getInt(HConstants.METRICS_RIT_STUCK_WARNING_THRESHOLD, 60000);
3344 for (RegionState state: regionStates.getRegionsInTransition().values()) {
3345 totalRITs++;
3346 long ritTime = currentTime - state.getStamp();
3347 if (ritTime > ritThreshold) {
3348 totalRITsOverThreshold++;
3349 }
3350 if (oldestRITTime < ritTime) {
3351 oldestRITTime = ritTime;
3352 }
3353 }
3354 if (this.metricsAssignmentManager != null) {
3355 this.metricsAssignmentManager.updateRITOldestAge(oldestRITTime);
3356 this.metricsAssignmentManager.updateRITCount(totalRITs);
3357 this.metricsAssignmentManager.updateRITCountOverThreshold(totalRITsOverThreshold);
3358 }
3359 }
3360
3361
3362
3363
3364 void clearRegionPlan(final HRegionInfo region) {
3365 synchronized (this.regionPlans) {
3366 this.regionPlans.remove(region.getEncodedName());
3367 }
3368 }
3369
3370
3371
3372
3373
3374
3375 public void waitOnRegionToClearRegionsInTransition(final HRegionInfo hri)
3376 throws IOException, InterruptedException {
3377 waitOnRegionToClearRegionsInTransition(hri, -1L);
3378 }
3379
3380
3381
3382
3383
3384
3385
3386
3387 public boolean waitOnRegionToClearRegionsInTransition(final HRegionInfo hri, long timeOut)
3388 throws InterruptedException {
3389 if (!regionStates.isRegionInTransition(hri)) return true;
3390 long end = (timeOut <= 0) ? Long.MAX_VALUE : EnvironmentEdgeManager.currentTime()
3391 + timeOut;
3392
3393
3394 LOG.info("Waiting for " + hri.getEncodedName() +
3395 " to leave regions-in-transition, timeOut=" + timeOut + " ms.");
3396 while (!this.server.isStopped() && regionStates.isRegionInTransition(hri)) {
3397 regionStates.waitForUpdate(100);
3398 if (EnvironmentEdgeManager.currentTime() > end) {
3399 LOG.info("Timed out on waiting for " + hri.getEncodedName() + " to be assigned.");
3400 return false;
3401 }
3402 }
3403 if (this.server.isStopped()) {
3404 LOG.info("Giving up wait on regions in transition because stoppable.isStopped is set");
3405 return false;
3406 }
3407 return true;
3408 }
3409
3410 void invokeAssign(HRegionInfo regionInfo) {
3411 invokeAssign(regionInfo, true);
3412 }
3413
3414 void invokeAssign(HRegionInfo regionInfo, boolean newPlan) {
3415 threadPoolExecutorService.submit(new AssignCallable(this, regionInfo, newPlan));
3416 }
3417
3418 void invokeUnAssign(HRegionInfo regionInfo) {
3419 threadPoolExecutorService.submit(new UnAssignCallable(this, regionInfo));
3420 }
3421
3422 public ServerHostRegion isCarryingMeta(ServerName serverName) {
3423 return isCarryingRegion(serverName, HRegionInfo.FIRST_META_REGIONINFO);
3424 }
3425
3426 public ServerHostRegion isCarryingMetaReplica(ServerName serverName, int replicaId) {
3427 return isCarryingRegion(serverName,
3428 RegionReplicaUtil.getRegionInfoForReplica(HRegionInfo.FIRST_META_REGIONINFO, replicaId));
3429 }
3430
3431 public ServerHostRegion isCarryingMetaReplica(ServerName serverName, HRegionInfo metaHri) {
3432 return isCarryingRegion(serverName, metaHri);
3433 }
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445 private ServerHostRegion isCarryingRegion(ServerName serverName, HRegionInfo hri) {
3446 RegionTransition rt = null;
3447 try {
3448 byte [] data = ZKAssign.getData(watcher, hri.getEncodedName());
3449
3450 rt = data == null? null: RegionTransition.parseFrom(data);
3451 } catch (KeeperException e) {
3452 server.abort("Exception reading unassigned node for region=" + hri.getEncodedName(), e);
3453 } catch (DeserializationException e) {
3454 server.abort("Exception parsing unassigned node for region=" + hri.getEncodedName(), e);
3455 }
3456
3457 ServerName addressFromZK = rt != null? rt.getServerName(): null;
3458 if (addressFromZK != null) {
3459
3460 boolean matchZK = addressFromZK.equals(serverName);
3461 LOG.debug("Checking region=" + hri.getRegionNameAsString() + ", zk server=" + addressFromZK +
3462 " current=" + serverName + ", matches=" + matchZK);
3463 return matchZK ? ServerHostRegion.HOSTING_REGION : ServerHostRegion.NOT_HOSTING_REGION;
3464 }
3465
3466 ServerName addressFromAM = regionStates.getRegionServerOfRegion(hri);
3467 if (addressFromAM != null) {
3468 boolean matchAM = addressFromAM.equals(serverName);
3469 LOG.debug("based on AM, current region=" + hri.getRegionNameAsString() +
3470 " is on server=" + (addressFromAM != null ? addressFromAM : "null") +
3471 " server being checked: " + serverName);
3472 return matchAM ? ServerHostRegion.HOSTING_REGION : ServerHostRegion.NOT_HOSTING_REGION;
3473 }
3474
3475 if (hri.isMetaRegion() && RegionReplicaUtil.isDefaultReplica(hri)) {
3476
3477 final ServerName serverNameInZK =
3478 server.getMetaTableLocator().getMetaRegionLocation(this.server.getZooKeeper());
3479 LOG.debug("Based on MetaTableLocator, the META region is on server="
3480 + (serverNameInZK == null ? "null" : serverNameInZK)
3481 + " server being checked: " + serverName);
3482 if (serverNameInZK != null) {
3483 return serverNameInZK.equals(serverName) ?
3484 ServerHostRegion.HOSTING_REGION : ServerHostRegion.NOT_HOSTING_REGION;
3485 }
3486 }
3487
3488
3489 return ServerHostRegion.UNKNOWN;
3490 }
3491
3492
3493
3494
3495
3496
3497 public List<HRegionInfo> processServerShutdown(final ServerName sn) {
3498
3499 synchronized (this.regionPlans) {
3500 for (Iterator <Map.Entry<String, RegionPlan>> i =
3501 this.regionPlans.entrySet().iterator(); i.hasNext();) {
3502 Map.Entry<String, RegionPlan> e = i.next();
3503 ServerName otherSn = e.getValue().getDestination();
3504
3505 if (otherSn != null && otherSn.equals(sn)) {
3506
3507 i.remove();
3508 }
3509 }
3510 }
3511 List<HRegionInfo> regions = regionStates.serverOffline(watcher, sn);
3512 for (Iterator<HRegionInfo> it = regions.iterator(); it.hasNext(); ) {
3513 HRegionInfo hri = it.next();
3514 String encodedName = hri.getEncodedName();
3515
3516
3517 Lock lock = locker.acquireLock(encodedName);
3518 try {
3519 RegionState regionState =
3520 regionStates.getRegionTransitionState(encodedName);
3521 if (regionState == null
3522 || (regionState.getServerName() != null && !regionState.isOnServer(sn))
3523 || !(regionState.isFailedClose() || regionState.isOffline()
3524 || regionState.isPendingOpenOrOpening())) {
3525 LOG.info("Skip " + regionState + " since it is not opening/failed_close"
3526 + " on the dead server any more: " + sn);
3527 it.remove();
3528 } else {
3529 try {
3530
3531 ZKAssign.deleteNodeFailSilent(watcher, hri);
3532 } catch (KeeperException ke) {
3533 server.abort("Unexpected ZK exception deleting node " + hri, ke);
3534 }
3535 if (tableStateManager.isTableState(hri.getTable(),
3536 ZooKeeperProtos.Table.State.DISABLED, ZooKeeperProtos.Table.State.DISABLING)) {
3537 regionStates.regionOffline(hri);
3538 it.remove();
3539 continue;
3540 }
3541
3542 regionStates.updateRegionState(hri, State.OFFLINE);
3543 }
3544 } finally {
3545 lock.unlock();
3546 }
3547 }
3548 return regions;
3549 }
3550
3551
3552
3553
3554 public void balance(final RegionPlan plan) {
3555
3556 HRegionInfo hri = plan.getRegionInfo();
3557 TableName tableName = hri.getTable();
3558 if (tableStateManager.isTableState(tableName,
3559 ZooKeeperProtos.Table.State.DISABLED, ZooKeeperProtos.Table.State.DISABLING)) {
3560 LOG.info("Ignored moving region of disabling/disabled table "
3561 + tableName);
3562 return;
3563 }
3564
3565
3566 String encodedName = hri.getEncodedName();
3567 ReentrantLock lock = locker.acquireLock(encodedName);
3568 try {
3569 if (!regionStates.isRegionOnline(hri)) {
3570 RegionState state = regionStates.getRegionState(encodedName);
3571 LOG.info("Ignored moving region not assigned: " + hri + ", "
3572 + (state == null ? "not in region states" : state));
3573 return;
3574 }
3575 synchronized (this.regionPlans) {
3576 this.regionPlans.put(plan.getRegionName(), plan);
3577 }
3578 unassign(hri, false, plan.getDestination());
3579 } finally {
3580 lock.unlock();
3581 }
3582 }
3583
3584 public void stop() {
3585 shutdown();
3586 }
3587
3588
3589
3590
3591 public void shutdown() {
3592
3593 synchronized (zkEventWorkerWaitingList){
3594 zkEventWorkerWaitingList.clear();
3595 }
3596
3597
3598 threadPoolExecutorService.shutdownNow();
3599 zkEventWorkers.shutdownNow();
3600 regionStateStore.stop();
3601 }
3602
3603 protected void setEnabledTable(TableName tableName) {
3604 try {
3605 this.tableStateManager.setTableState(tableName,
3606 ZooKeeperProtos.Table.State.ENABLED);
3607 } catch (CoordinatedStateException e) {
3608
3609 String errorMsg = "Unable to ensure that the table " + tableName
3610 + " will be" + " enabled because of a ZooKeeper issue";
3611 LOG.error(errorMsg);
3612 this.server.abort(errorMsg, e);
3613 }
3614 }
3615
3616
3617
3618
3619
3620
3621
3622 private boolean asyncSetOfflineInZooKeeper(final RegionState state,
3623 final AsyncCallback.StringCallback cb, final ServerName destination) {
3624 if (!state.isClosed() && !state.isOffline()) {
3625 this.server.abort("Unexpected state trying to OFFLINE; " + state,
3626 new IllegalStateException());
3627 return false;
3628 }
3629 regionStates.updateRegionState(state.getRegion(), State.OFFLINE);
3630 try {
3631 ZKAssign.asyncCreateNodeOffline(watcher, state.getRegion(),
3632 destination, cb, state);
3633 } catch (KeeperException e) {
3634 if (e instanceof NodeExistsException) {
3635 LOG.warn("Node for " + state.getRegion() + " already exists");
3636 } else {
3637 server.abort("Unexpected ZK exception creating/setting node OFFLINE", e);
3638 }
3639 return false;
3640 }
3641 return true;
3642 }
3643
3644 private boolean deleteNodeInStates(String encodedName,
3645 String desc, ServerName sn, EventType... types) {
3646 try {
3647 for (EventType et: types) {
3648 if (ZKAssign.deleteNode(watcher, encodedName, et, sn)) {
3649 return true;
3650 }
3651 }
3652 LOG.info("Failed to delete the " + desc + " node for "
3653 + encodedName + ". The node type may not match");
3654 } catch (NoNodeException e) {
3655 if (LOG.isDebugEnabled()) {
3656 LOG.debug("The " + desc + " node for " + encodedName + " already deleted");
3657 }
3658 } catch (KeeperException ke) {
3659 server.abort("Unexpected ZK exception deleting " + desc
3660 + " node for the region " + encodedName, ke);
3661 }
3662 return false;
3663 }
3664
3665 private void deleteMergingNode(String encodedName, ServerName sn) {
3666 deleteNodeInStates(encodedName, "merging", sn, EventType.RS_ZK_REGION_MERGING,
3667 EventType.RS_ZK_REQUEST_REGION_MERGE, EventType.RS_ZK_REGION_MERGED);
3668 }
3669
3670 private void deleteSplittingNode(String encodedName, ServerName sn) {
3671 deleteNodeInStates(encodedName, "splitting", sn, EventType.RS_ZK_REGION_SPLITTING,
3672 EventType.RS_ZK_REQUEST_REGION_SPLIT, EventType.RS_ZK_REGION_SPLIT);
3673 }
3674
3675 private void onRegionFailedOpen(
3676 final HRegionInfo hri, final ServerName sn) {
3677 String encodedName = hri.getEncodedName();
3678 AtomicInteger failedOpenCount = failedOpenTracker.get(encodedName);
3679 if (failedOpenCount == null) {
3680 failedOpenCount = new AtomicInteger();
3681
3682
3683
3684 failedOpenTracker.put(encodedName, failedOpenCount);
3685 }
3686 if (failedOpenCount.incrementAndGet() >= maximumAttempts && !hri.isMetaRegion()) {
3687 regionStates.updateRegionState(hri, State.FAILED_OPEN);
3688
3689
3690 failedOpenTracker.remove(encodedName);
3691 } else {
3692 if (hri.isMetaRegion() && failedOpenCount.get() >= maximumAttempts) {
3693
3694
3695 LOG.warn("Failed to open the hbase:meta region " +
3696 hri.getRegionNameAsString() + " after" +
3697 failedOpenCount.get() + " retries. Continue retrying.");
3698 }
3699
3700
3701 RegionState regionState = regionStates.updateRegionState(hri, State.CLOSED);
3702 if (regionState != null) {
3703
3704
3705 if (getTableStateManager().isTableState(hri.getTable(),
3706 ZooKeeperProtos.Table.State.DISABLED, ZooKeeperProtos.Table.State.DISABLING) ||
3707 replicasToClose.contains(hri)) {
3708 offlineDisabledRegion(hri);
3709 return;
3710 }
3711
3712 regionStates.updateRegionState(hri, RegionState.State.CLOSED);
3713
3714 removeClosedRegion(hri);
3715 try {
3716 getRegionPlan(hri, sn, true);
3717 } catch (HBaseIOException e) {
3718 LOG.warn("Failed to get region plan", e);
3719 }
3720 invokeAssign(hri, false);
3721 }
3722 }
3723 }
3724
3725 private void onRegionOpen(
3726 final HRegionInfo hri, final ServerName sn, long openSeqNum) {
3727 regionOnline(hri, sn, openSeqNum);
3728 if (useZKForAssignment) {
3729 try {
3730
3731 ZKAssign.deleteNodeFailSilent(watcher, hri);
3732 } catch (KeeperException ke) {
3733 server.abort("Unexpected ZK exception deleting node " + hri, ke);
3734 }
3735 }
3736
3737
3738 failedOpenTracker.remove(hri.getEncodedName());
3739 if (getTableStateManager().isTableState(hri.getTable(),
3740 ZooKeeperProtos.Table.State.DISABLED, ZooKeeperProtos.Table.State.DISABLING)) {
3741 invokeUnAssign(hri);
3742 }
3743 }
3744
3745 private void onRegionClosed(final HRegionInfo hri) {
3746 if (getTableStateManager().isTableState(hri.getTable(),
3747 ZooKeeperProtos.Table.State.DISABLED, ZooKeeperProtos.Table.State.DISABLING) ||
3748 replicasToClose.contains(hri)) {
3749 offlineDisabledRegion(hri);
3750 return;
3751 }
3752 regionStates.updateRegionState(hri, RegionState.State.CLOSED);
3753 sendRegionClosedNotification(hri);
3754
3755 removeClosedRegion(hri);
3756 invokeAssign(hri, false);
3757 }
3758
3759 private String onRegionSplit(ServerName sn, TransitionCode code,
3760 final HRegionInfo p, final HRegionInfo a, final HRegionInfo b) {
3761 final RegionState rs_p = regionStates.getRegionState(p);
3762 RegionState rs_a = regionStates.getRegionState(a);
3763 RegionState rs_b = regionStates.getRegionState(b);
3764 if (!(rs_p.isOpenOrSplittingOnServer(sn)
3765 && (rs_a == null || rs_a.isOpenOrSplittingNewOnServer(sn))
3766 && (rs_b == null || rs_b.isOpenOrSplittingNewOnServer(sn)))) {
3767 return "Not in state good for split";
3768 }
3769 regionStates.updateRegionState(a, State.SPLITTING_NEW, sn);
3770 regionStates.updateRegionState(b, State.SPLITTING_NEW, sn);
3771 regionStates.updateRegionState(p, State.SPLITTING);
3772
3773 if (code == TransitionCode.SPLIT) {
3774 if (TEST_SKIP_SPLIT_HANDLING) {
3775 return "Skipping split message, TEST_SKIP_SPLIT_HANDLING is set";
3776 }
3777 regionOffline(p, State.SPLIT);
3778 regionOnline(a, sn, 1);
3779 regionOnline(b, sn, 1);
3780
3781
3782 if (getTableStateManager().isTableState(p.getTable(),
3783 ZooKeeperProtos.Table.State.DISABLED, ZooKeeperProtos.Table.State.DISABLING)) {
3784 invokeUnAssign(a);
3785 invokeUnAssign(b);
3786 } else {
3787 Callable<Object> splitReplicasCallable = new Callable<Object>() {
3788 @Override
3789 public Object call() {
3790 doSplittingOfReplicas(p, a, b);
3791 return null;
3792 }
3793 };
3794 threadPoolExecutorService.submit(splitReplicasCallable);
3795 }
3796 } else if (code == TransitionCode.SPLIT_PONR) {
3797 try {
3798 regionStates.splitRegion(p, a, b, sn);
3799 } catch (IOException ioe) {
3800 LOG.info("Failed to record split region " + p.getShortNameToLog());
3801 return "Failed to record the splitting in meta";
3802 }
3803 } else if (code == TransitionCode.SPLIT_REVERTED) {
3804
3805
3806 regionOnline(p, sn);
3807
3808
3809 RegionState regionStateA = regionStates.getRegionState(a);
3810 RegionState regionStateB = regionStates.getRegionState(b);
3811 if (regionStateA != null) {
3812 regionOffline(a);
3813 }
3814 if (regionStateB != null) {
3815 regionOffline(b);
3816 }
3817
3818 if (getTableStateManager().isTableState(p.getTable(),
3819 ZooKeeperProtos.Table.State.DISABLED, ZooKeeperProtos.Table.State.DISABLING)) {
3820 invokeUnAssign(p);
3821 }
3822 }
3823 return null;
3824 }
3825
3826 private String onRegionMerge(ServerName sn, TransitionCode code,
3827 final HRegionInfo p, final HRegionInfo a, final HRegionInfo b) {
3828 RegionState rs_p = regionStates.getRegionState(p);
3829 RegionState rs_a = regionStates.getRegionState(a);
3830 RegionState rs_b = regionStates.getRegionState(b);
3831 if (!(rs_a.isOpenOrMergingOnServer(sn) && rs_b.isOpenOrMergingOnServer(sn)
3832 && (rs_p == null || rs_p.isOpenOrMergingNewOnServer(sn)))) {
3833 return "Not in state good for merge";
3834 }
3835 regionStates.updateRegionState(a, State.MERGING);
3836 regionStates.updateRegionState(b, State.MERGING);
3837 regionStates.updateRegionState(p, State.MERGING_NEW, sn);
3838
3839 String encodedName = p.getEncodedName();
3840 if (code == TransitionCode.READY_TO_MERGE) {
3841 mergingRegions.put(encodedName,
3842 new PairOfSameType<HRegionInfo>(a, b));
3843 } else if (code == TransitionCode.MERGED) {
3844 mergingRegions.remove(encodedName);
3845 regionOffline(a, State.MERGED);
3846 regionOffline(b, State.MERGED);
3847 regionOnline(p, sn, 1);
3848
3849
3850 if (getTableStateManager().isTableState(p.getTable(),
3851 ZooKeeperProtos.Table.State.DISABLED, ZooKeeperProtos.Table.State.DISABLING)) {
3852 invokeUnAssign(p);
3853 } else {
3854 Callable<Object> mergeReplicasCallable = new Callable<Object>() {
3855 @Override
3856 public Object call() {
3857 doMergingOfReplicas(p, a, b);
3858 return null;
3859 }
3860 };
3861 threadPoolExecutorService.submit(mergeReplicasCallable);
3862 }
3863 } else if (code == TransitionCode.MERGE_PONR) {
3864 try {
3865 regionStates.mergeRegions(p, a, b, sn);
3866 } catch (IOException ioe) {
3867 LOG.info("Failed to record merged region " + p.getShortNameToLog());
3868 return "Failed to record the merging in meta";
3869 }
3870 }
3871 return null;
3872 }
3873
3874 private String onRegionMergeReverted(ServerName sn, TransitionCode code,
3875 final HRegionInfo p, final HRegionInfo a, final HRegionInfo b) {
3876 RegionState rs_p = regionStates.getRegionState(p);
3877 String encodedName = p.getEncodedName();
3878 mergingRegions.remove(encodedName);
3879
3880
3881
3882 regionOnline(a, sn);
3883 regionOnline(b, sn);
3884
3885
3886 if (rs_p != null) {
3887 regionOffline(p);
3888 }
3889
3890 if (getTableStateManager().isTableState(p.getTable(),
3891 ZooKeeperProtos.Table.State.DISABLED, ZooKeeperProtos.Table.State.DISABLING)) {
3892 invokeUnAssign(a);
3893 invokeUnAssign(b);
3894 }
3895
3896 return null;
3897 }
3898
3899
3900
3901
3902
3903 private boolean handleRegionMerging(final RegionTransition rt, final String encodedName,
3904 final String prettyPrintedRegionName, final ServerName sn) {
3905 if (!serverManager.isServerOnline(sn)) {
3906 LOG.warn("Dropped merging! ServerName=" + sn + " unknown.");
3907 return false;
3908 }
3909 byte [] payloadOfMerging = rt.getPayload();
3910 List<HRegionInfo> mergingRegions;
3911 try {
3912 mergingRegions = HRegionInfo.parseDelimitedFrom(
3913 payloadOfMerging, 0, payloadOfMerging.length);
3914 } catch (IOException e) {
3915 LOG.error("Dropped merging! Failed reading " + rt.getEventType()
3916 + " payload for " + prettyPrintedRegionName);
3917 return false;
3918 }
3919 assert mergingRegions.size() == 3;
3920 HRegionInfo p = mergingRegions.get(0);
3921 HRegionInfo hri_a = mergingRegions.get(1);
3922 HRegionInfo hri_b = mergingRegions.get(2);
3923
3924 RegionState rs_p = regionStates.getRegionState(p);
3925 RegionState rs_a = regionStates.getRegionState(hri_a);
3926 RegionState rs_b = regionStates.getRegionState(hri_b);
3927
3928 if (!((rs_a == null || rs_a.isOpenOrMergingOnServer(sn))
3929 && (rs_b == null || rs_b.isOpenOrMergingOnServer(sn))
3930 && (rs_p == null || rs_p.isOpenOrMergingNewOnServer(sn)))) {
3931 LOG.warn("Dropped merging! Not in state good for MERGING; rs_p="
3932 + rs_p + ", rs_a=" + rs_a + ", rs_b=" + rs_b);
3933 return false;
3934 }
3935
3936 EventType et = rt.getEventType();
3937 if (et == EventType.RS_ZK_REQUEST_REGION_MERGE) {
3938 try {
3939 RegionMergeCoordination.RegionMergeDetails std =
3940 ((BaseCoordinatedStateManager) server.getCoordinatedStateManager())
3941 .getRegionMergeCoordination().getDefaultDetails();
3942 ((BaseCoordinatedStateManager) server.getCoordinatedStateManager())
3943 .getRegionMergeCoordination().processRegionMergeRequest(p, hri_a, hri_b, sn, std);
3944 if (((ZkRegionMergeCoordination.ZkRegionMergeDetails) std).getZnodeVersion() == -1) {
3945 byte[] data = ZKAssign.getData(watcher, encodedName);
3946 EventType currentType = null;
3947 if (data != null) {
3948 RegionTransition newRt = RegionTransition.parseFrom(data);
3949 currentType = newRt.getEventType();
3950 }
3951 if (currentType == null || (currentType != EventType.RS_ZK_REGION_MERGED
3952 && currentType != EventType.RS_ZK_REGION_MERGING)) {
3953 LOG.warn("Failed to transition pending_merge node "
3954 + encodedName + " to merging, it's now " + currentType);
3955 return false;
3956 }
3957 }
3958 } catch (Exception e) {
3959 LOG.warn("Failed to transition pending_merge node "
3960 + encodedName + " to merging", e);
3961 return false;
3962 }
3963 }
3964
3965 synchronized (regionStates) {
3966 regionStates.updateRegionState(hri_a, State.MERGING);
3967 regionStates.updateRegionState(hri_b, State.MERGING);
3968 regionStates.updateRegionState(p, State.MERGING_NEW, sn);
3969
3970 if (et != EventType.RS_ZK_REGION_MERGED) {
3971 this.mergingRegions.put(encodedName,
3972 new PairOfSameType<HRegionInfo>(hri_a, hri_b));
3973 } else {
3974 this.mergingRegions.remove(encodedName);
3975 regionOffline(hri_a, State.MERGED);
3976 regionOffline(hri_b, State.MERGED);
3977 regionOnline(p, sn);
3978 }
3979 }
3980
3981 if (et == EventType.RS_ZK_REGION_MERGED) {
3982 doMergingOfReplicas(p, hri_a, hri_b);
3983 LOG.debug("Handling MERGED event for " + encodedName + "; deleting node");
3984
3985 try {
3986 boolean successful = false;
3987 while (!successful) {
3988
3989
3990 successful = ZKAssign.deleteNode(watcher, encodedName,
3991 EventType.RS_ZK_REGION_MERGED, sn);
3992 }
3993 } catch (KeeperException e) {
3994 if (e instanceof NoNodeException) {
3995 String znodePath = ZKUtil.joinZNode(watcher.splitLogZNode, encodedName);
3996 LOG.debug("The znode " + znodePath + " does not exist. May be deleted already.");
3997 } else {
3998 server.abort("Error deleting MERGED node " + encodedName, e);
3999 }
4000 }
4001 LOG.info("Handled MERGED event; merged=" + p.getRegionNameAsString()
4002 + ", region_a=" + hri_a.getRegionNameAsString() + ", region_b="
4003 + hri_b.getRegionNameAsString() + ", on " + sn);
4004
4005
4006 if (tableStateManager.isTableState(p.getTable(),
4007 ZooKeeperProtos.Table.State.DISABLED, ZooKeeperProtos.Table.State.DISABLING)) {
4008 unassign(p);
4009 }
4010 }
4011 return true;
4012 }
4013
4014
4015
4016
4017 private boolean handleRegionSplitting(final RegionTransition rt, final String encodedName,
4018 final String prettyPrintedRegionName, final ServerName sn) {
4019 if (!serverManager.isServerOnline(sn)) {
4020 LOG.warn("Dropped splitting! ServerName=" + sn + " unknown.");
4021 return false;
4022 }
4023 byte [] payloadOfSplitting = rt.getPayload();
4024 List<HRegionInfo> splittingRegions;
4025 try {
4026 splittingRegions = HRegionInfo.parseDelimitedFrom(
4027 payloadOfSplitting, 0, payloadOfSplitting.length);
4028 } catch (IOException e) {
4029 LOG.error("Dropped splitting! Failed reading " + rt.getEventType()
4030 + " payload for " + prettyPrintedRegionName);
4031 return false;
4032 }
4033 assert splittingRegions.size() == 2;
4034 HRegionInfo hri_a = splittingRegions.get(0);
4035 HRegionInfo hri_b = splittingRegions.get(1);
4036
4037 RegionState rs_p = regionStates.getRegionState(encodedName);
4038 RegionState rs_a = regionStates.getRegionState(hri_a);
4039 RegionState rs_b = regionStates.getRegionState(hri_b);
4040
4041 if (!((rs_p == null || rs_p.isOpenOrSplittingOnServer(sn))
4042 && (rs_a == null || rs_a.isOpenOrSplittingNewOnServer(sn))
4043 && (rs_b == null || rs_b.isOpenOrSplittingNewOnServer(sn)))) {
4044 LOG.warn("Dropped splitting! Not in state good for SPLITTING; rs_p="
4045 + rs_p + ", rs_a=" + rs_a + ", rs_b=" + rs_b);
4046 return false;
4047 }
4048
4049 if (rs_p == null) {
4050
4051 rs_p = regionStates.updateRegionState(rt, State.OPEN);
4052 if (rs_p == null) {
4053 LOG.warn("Received splitting for region " + prettyPrintedRegionName
4054 + " from server " + sn + " but it doesn't exist anymore,"
4055 + " probably already processed its split");
4056 return false;
4057 }
4058 regionStates.regionOnline(rs_p.getRegion(), sn);
4059 }
4060
4061 HRegionInfo p = rs_p.getRegion();
4062 EventType et = rt.getEventType();
4063 if (et == EventType.RS_ZK_REQUEST_REGION_SPLIT) {
4064 try {
4065 SplitTransactionDetails std =
4066 ((BaseCoordinatedStateManager) server.getCoordinatedStateManager())
4067 .getSplitTransactionCoordination().getDefaultDetails();
4068 if (((BaseCoordinatedStateManager) server.getCoordinatedStateManager())
4069 .getSplitTransactionCoordination().processTransition(p, hri_a, hri_b, sn, std) == -1) {
4070 byte[] data = ZKAssign.getData(watcher, encodedName);
4071 EventType currentType = null;
4072 if (data != null) {
4073 RegionTransition newRt = RegionTransition.parseFrom(data);
4074 currentType = newRt.getEventType();
4075 }
4076 if (currentType == null
4077 || (currentType != EventType.RS_ZK_REGION_SPLIT && currentType != EventType.RS_ZK_REGION_SPLITTING)) {
4078 LOG.warn("Failed to transition pending_split node " + encodedName
4079 + " to splitting, it's now " + currentType);
4080 return false;
4081 }
4082 }
4083 } catch (Exception e) {
4084 LOG.warn("Failed to transition pending_split node " + encodedName + " to splitting", e);
4085 return false;
4086 }
4087 }
4088
4089 synchronized (regionStates) {
4090 splitRegions.put(p, new PairOfSameType<HRegionInfo>(hri_a, hri_b));
4091 regionStates.updateRegionState(hri_a, State.SPLITTING_NEW, sn);
4092 regionStates.updateRegionState(hri_b, State.SPLITTING_NEW, sn);
4093 regionStates.updateRegionState(rt, State.SPLITTING);
4094
4095
4096
4097 if (TEST_SKIP_SPLIT_HANDLING) {
4098 LOG.warn("Skipping split message, TEST_SKIP_SPLIT_HANDLING is set");
4099 return true;
4100 }
4101
4102 if (et == EventType.RS_ZK_REGION_SPLIT) {
4103 regionOffline(p, State.SPLIT);
4104 regionOnline(hri_a, sn);
4105 regionOnline(hri_b, sn);
4106 splitRegions.remove(p);
4107 }
4108 }
4109
4110 if (et == EventType.RS_ZK_REGION_SPLIT) {
4111
4112 doSplittingOfReplicas(rs_p.getRegion(), hri_a, hri_b);
4113 LOG.debug("Handling SPLIT event for " + encodedName + "; deleting node");
4114
4115 try {
4116 boolean successful = false;
4117 while (!successful) {
4118
4119
4120 successful = ZKAssign.deleteNode(watcher, encodedName,
4121 EventType.RS_ZK_REGION_SPLIT, sn);
4122 }
4123 } catch (KeeperException e) {
4124 if (e instanceof NoNodeException) {
4125 String znodePath = ZKUtil.joinZNode(watcher.splitLogZNode, encodedName);
4126 LOG.debug("The znode " + znodePath + " does not exist. May be deleted already.");
4127 } else {
4128 server.abort("Error deleting SPLIT node " + encodedName, e);
4129 }
4130 }
4131 LOG.info("Handled SPLIT event; parent=" + p.getRegionNameAsString()
4132 + ", daughter a=" + hri_a.getRegionNameAsString() + ", daughter b="
4133 + hri_b.getRegionNameAsString() + ", on " + sn);
4134
4135
4136 if (tableStateManager.isTableState(p.getTable(),
4137 ZooKeeperProtos.Table.State.DISABLED, ZooKeeperProtos.Table.State.DISABLING)) {
4138 unassign(hri_a);
4139 unassign(hri_b);
4140 }
4141 }
4142 return true;
4143 }
4144
4145 private void doMergingOfReplicas(HRegionInfo mergedHri, final HRegionInfo hri_a,
4146 final HRegionInfo hri_b) {
4147
4148
4149 List<HRegionInfo> unmergedRegions = new ArrayList<HRegionInfo>();
4150 unmergedRegions.add(hri_a);
4151 unmergedRegions.add(hri_b);
4152 Map<ServerName, List<HRegionInfo>> map = regionStates.getRegionAssignments(unmergedRegions);
4153 Collection<List<HRegionInfo>> c = map.values();
4154 for (List<HRegionInfo> l : c) {
4155 for (HRegionInfo h : l) {
4156 if (!RegionReplicaUtil.isDefaultReplica(h)) {
4157 LOG.debug("Unassigning un-merged replica " + h);
4158 unassign(h);
4159 }
4160 }
4161 }
4162 int numReplicas = 1;
4163 try {
4164 numReplicas = ((MasterServices)server).getTableDescriptors().get(mergedHri.getTable()).
4165 getRegionReplication();
4166 } catch (IOException e) {
4167 LOG.warn("Couldn't get the replication attribute of the table " + mergedHri.getTable() +
4168 " due to " + e.getMessage() + ". The assignment of replicas for the merged region " +
4169 "will not be done");
4170 }
4171 List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
4172 for (int i = 1; i < numReplicas; i++) {
4173 regions.add(RegionReplicaUtil.getRegionInfoForReplica(mergedHri, i));
4174 }
4175 try {
4176 assign(regions);
4177 } catch (IOException ioe) {
4178 LOG.warn("Couldn't assign all replica(s) of region " + mergedHri + " because of " +
4179 ioe.getMessage());
4180 } catch (InterruptedException ie) {
4181 LOG.warn("Couldn't assign all replica(s) of region " + mergedHri+ " because of " +
4182 ie.getMessage());
4183 }
4184 }
4185
4186 private void doSplittingOfReplicas(final HRegionInfo parentHri, final HRegionInfo hri_a,
4187 final HRegionInfo hri_b) {
4188
4189
4190
4191 int numReplicas = 1;
4192 try {
4193 numReplicas = ((MasterServices)server).getTableDescriptors().get(parentHri.getTable()).
4194 getRegionReplication();
4195 } catch (IOException e) {
4196 LOG.warn("Couldn't get the replication attribute of the table " + parentHri.getTable() +
4197 " due to " + e.getMessage() + ". The assignment of daughter replicas " +
4198 "replicas will not be done");
4199 }
4200
4201 List<HRegionInfo> parentRegion = new ArrayList<HRegionInfo>();
4202 parentRegion.add(parentHri);
4203 Map<ServerName, List<HRegionInfo>> currentAssign =
4204 regionStates.getRegionAssignments(parentRegion);
4205 Collection<List<HRegionInfo>> c = currentAssign.values();
4206 for (List<HRegionInfo> l : c) {
4207 for (HRegionInfo h : l) {
4208 if (!RegionReplicaUtil.isDefaultReplica(h)) {
4209 LOG.debug("Unassigning parent's replica " + h);
4210 unassign(h);
4211 }
4212 }
4213 }
4214
4215 Map<HRegionInfo, ServerName> map = new HashMap<HRegionInfo, ServerName>();
4216 for (int i = 1; i < numReplicas; i++) {
4217 prepareDaughterReplicaForAssignment(hri_a, parentHri, i, map);
4218 prepareDaughterReplicaForAssignment(hri_b, parentHri, i, map);
4219 }
4220 try {
4221 assign(map);
4222 } catch (IOException e) {
4223 LOG.warn("Caught exception " + e + " while trying to assign replica(s) of daughter(s)");
4224 } catch (InterruptedException e) {
4225 LOG.warn("Caught exception " + e + " while trying to assign replica(s) of daughter(s)");
4226 }
4227 }
4228
4229 private void prepareDaughterReplicaForAssignment(HRegionInfo daughterHri, HRegionInfo parentHri,
4230 int replicaId, Map<HRegionInfo, ServerName> map) {
4231 HRegionInfo parentReplica = RegionReplicaUtil.getRegionInfoForReplica(parentHri, replicaId);
4232 HRegionInfo daughterReplica = RegionReplicaUtil.getRegionInfoForReplica(daughterHri,
4233 replicaId);
4234 LOG.debug("Created replica region for daughter " + daughterReplica);
4235 ServerName sn;
4236 if ((sn = regionStates.getRegionServerOfRegion(parentReplica)) != null) {
4237 map.put(daughterReplica, sn);
4238 } else {
4239 List<ServerName> servers = serverManager.getOnlineServersList();
4240 sn = servers.get((new Random(System.currentTimeMillis())).nextInt(servers.size()));
4241 map.put(daughterReplica, sn);
4242 }
4243 }
4244
4245 public Set<HRegionInfo> getReplicasToClose() {
4246 return replicasToClose;
4247 }
4248
4249
4250
4251
4252
4253
4254 private void regionOffline(final HRegionInfo regionInfo, final State state) {
4255 regionStates.regionOffline(regionInfo, state);
4256 removeClosedRegion(regionInfo);
4257
4258 clearRegionPlan(regionInfo);
4259 balancer.regionOffline(regionInfo);
4260
4261
4262 sendRegionClosedNotification(regionInfo);
4263
4264 if (state != null && state.equals(State.SPLIT)) {
4265 Collection<HRegionInfo> c = new ArrayList<HRegionInfo>(1);
4266 c.add(regionInfo);
4267 Map<ServerName, List<HRegionInfo>> map = regionStates.getRegionAssignments(c);
4268 Collection<List<HRegionInfo>> allReplicas = map.values();
4269 for (List<HRegionInfo> list : allReplicas) {
4270 replicasToClose.addAll(list);
4271 }
4272 }
4273 else if (state != null && state.equals(State.MERGED)) {
4274 Collection<HRegionInfo> c = new ArrayList<HRegionInfo>(1);
4275 c.add(regionInfo);
4276 Map<ServerName, List<HRegionInfo>> map = regionStates.getRegionAssignments(c);
4277 Collection<List<HRegionInfo>> allReplicas = map.values();
4278 for (List<HRegionInfo> list : allReplicas) {
4279 replicasToClose.addAll(list);
4280 }
4281 }
4282 }
4283
4284 private void sendRegionOpenedNotification(final HRegionInfo regionInfo,
4285 final ServerName serverName) {
4286 if (!this.listeners.isEmpty()) {
4287 for (AssignmentListener listener : this.listeners) {
4288 listener.regionOpened(regionInfo, serverName);
4289 }
4290 }
4291 }
4292
4293 private void sendRegionClosedNotification(final HRegionInfo regionInfo) {
4294 if (!this.listeners.isEmpty()) {
4295 for (AssignmentListener listener : this.listeners) {
4296 listener.regionClosed(regionInfo);
4297 }
4298 }
4299 }
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345 protected String onRegionTransition(final ServerName serverName,
4346 final RegionStateTransition transition) {
4347 TransitionCode code = transition.getTransitionCode();
4348 HRegionInfo hri = HRegionInfo.convert(transition.getRegionInfo(0));
4349 RegionState current = regionStates.getRegionState(hri);
4350 if (LOG.isDebugEnabled()) {
4351 LOG.debug("Got transition " + code + " for "
4352 + (current != null ? current.toString() : hri.getShortNameToLog())
4353 + " from " + serverName);
4354 }
4355 String errorMsg = null;
4356 switch (code) {
4357 case OPENED:
4358 if (current != null && current.isOpened() && current.isOnServer(serverName)) {
4359 LOG.info("Region " + hri.getShortNameToLog() + " is already " + current.getState() + " on "
4360 + serverName);
4361 break;
4362 }
4363 case FAILED_OPEN:
4364 if (current == null
4365 || !current.isPendingOpenOrOpeningOnServer(serverName)) {
4366 errorMsg = hri.getShortNameToLog()
4367 + " is not pending open on " + serverName;
4368 } else if (code == TransitionCode.FAILED_OPEN) {
4369 onRegionFailedOpen(hri, serverName);
4370 } else {
4371 long openSeqNum = HConstants.NO_SEQNUM;
4372 if (transition.hasOpenSeqNum()) {
4373 openSeqNum = transition.getOpenSeqNum();
4374 }
4375 if (openSeqNum < 0) {
4376 errorMsg = "Newly opened region has invalid open seq num " + openSeqNum;
4377 } else {
4378 onRegionOpen(hri, serverName, openSeqNum);
4379 }
4380 }
4381 break;
4382
4383 case CLOSED:
4384 if (current == null
4385 || !current.isPendingCloseOrClosingOnServer(serverName)) {
4386 errorMsg = hri.getShortNameToLog()
4387 + " is not pending close on " + serverName;
4388 } else {
4389 onRegionClosed(hri);
4390 }
4391 break;
4392
4393 case READY_TO_SPLIT:
4394 try {
4395 regionStateListener.onRegionSplit(hri);
4396 if (!((HMaster)server).getSplitOrMergeTracker().isSplitOrMergeEnabled(
4397 Admin.MasterSwitchType.SPLIT)) {
4398 errorMsg = "split switch is off!";
4399 }
4400 } catch (IOException exp) {
4401 errorMsg = StringUtils.stringifyException(exp);
4402 }
4403 break;
4404 case SPLIT_PONR:
4405 case SPLIT:
4406 case SPLIT_REVERTED:
4407 errorMsg =
4408 onRegionSplit(serverName, code, hri, HRegionInfo.convert(transition.getRegionInfo(1)),
4409 HRegionInfo.convert(transition.getRegionInfo(2)));
4410 if (org.apache.commons.lang.StringUtils.isEmpty(errorMsg)) {
4411 try {
4412 regionStateListener.onRegionSplitReverted(hri);
4413 } catch (IOException exp) {
4414 LOG.warn(StringUtils.stringifyException(exp));
4415 }
4416 }
4417 break;
4418 case READY_TO_MERGE:
4419 if (!((HMaster)server).getSplitOrMergeTracker().isSplitOrMergeEnabled(
4420 Admin.MasterSwitchType.MERGE)) {
4421 errorMsg = "merge switch is off!";
4422 }
4423 break;
4424 case MERGE_PONR:
4425 case MERGED:
4426 errorMsg = onRegionMerge(serverName, code, hri,
4427 HRegionInfo.convert(transition.getRegionInfo(1)),
4428 HRegionInfo.convert(transition.getRegionInfo(2)));
4429 if (code == TransitionCode.MERGED && org.apache.commons.lang.StringUtils.isEmpty(errorMsg)) {
4430 try {
4431 regionStateListener.onRegionMerged(hri);
4432 } catch (IOException exp) {
4433 errorMsg = StringUtils.stringifyException(exp);
4434 }
4435 }
4436 break;
4437 case MERGE_REVERTED:
4438 errorMsg = onRegionMergeReverted(serverName, code, hri,
4439 HRegionInfo.convert(transition.getRegionInfo(1)),
4440 HRegionInfo.convert(transition.getRegionInfo(2)));
4441 break;
4442
4443 default:
4444 errorMsg = "Unexpected transition code " + code;
4445 }
4446 if (errorMsg != null) {
4447 LOG.error("Failed to transtion region from " + current + " to "
4448 + code + " by " + serverName + ": " + errorMsg);
4449 }
4450 return errorMsg;
4451 }
4452
4453 private void processBogusAssignments(Map<ServerName, List<HRegionInfo>> bulkPlan) {
4454 if (bulkPlan.containsKey(LoadBalancer.BOGUS_SERVER_NAME)) {
4455
4456 for (HRegionInfo hri : bulkPlan.get(LoadBalancer.BOGUS_SERVER_NAME)) {
4457 regionStates.updateRegionState(hri, State.FAILED_OPEN);
4458 }
4459 bulkPlan.remove(LoadBalancer.BOGUS_SERVER_NAME);
4460 }
4461 }
4462
4463
4464
4465
4466 public LoadBalancer getBalancer() {
4467 return this.balancer;
4468 }
4469
4470 public Map<ServerName, List<HRegionInfo>>
4471 getSnapShotOfAssignment(Collection<HRegionInfo> infos) {
4472 return getRegionStates().getRegionAssignments(infos);
4473 }
4474
4475 void setRegionStateListener(RegionStateListener listener) {
4476 this.regionStateListener = listener;
4477 }
4478 }