1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import java.io.IOException;
22 import java.io.InterruptedIOException;
23 import java.lang.Thread.UncaughtExceptionHandler;
24 import java.lang.management.ManagementFactory;
25 import java.lang.management.MemoryUsage;
26 import java.lang.reflect.Constructor;
27 import java.net.BindException;
28 import java.net.InetAddress;
29 import java.net.InetSocketAddress;
30 import java.util.ArrayList;
31 import java.util.Collection;
32 import java.util.Collections;
33 import java.util.Comparator;
34 import java.util.HashMap;
35 import java.util.HashSet;
36 import java.util.Iterator;
37 import java.util.List;
38 import java.util.Map;
39 import java.util.Objects;
40 import java.util.Map.Entry;
41 import java.util.Set;
42 import java.util.SortedMap;
43 import java.util.TreeMap;
44 import java.util.TreeSet;
45 import java.util.concurrent.ConcurrentHashMap;
46 import java.util.concurrent.ConcurrentMap;
47 import java.util.concurrent.ConcurrentSkipListMap;
48 import java.util.concurrent.CountDownLatch;
49 import java.util.concurrent.TimeUnit;
50 import java.util.concurrent.atomic.AtomicBoolean;
51 import java.util.concurrent.atomic.AtomicReference;
52 import java.util.concurrent.locks.ReentrantReadWriteLock;
53
54 import javax.management.MalformedObjectNameException;
55 import javax.management.ObjectName;
56 import javax.servlet.http.HttpServlet;
57
58 import org.apache.commons.lang.math.RandomUtils;
59 import org.apache.commons.logging.Log;
60 import org.apache.commons.logging.LogFactory;
61 import org.apache.hadoop.conf.Configuration;
62 import org.apache.hadoop.fs.FileSystem;
63 import org.apache.hadoop.fs.Path;
64 import org.apache.hadoop.hbase.ChoreService;
65 import org.apache.hadoop.hbase.ClockOutOfSyncException;
66 import org.apache.hadoop.hbase.CoordinatedStateManager;
67 import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
68 import org.apache.hadoop.hbase.DoNotRetryIOException;
69 import org.apache.hadoop.hbase.HBaseConfiguration;
70 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
71 import org.apache.hadoop.hbase.HConstants;
72 import org.apache.hadoop.hbase.HRegionInfo;
73 import org.apache.hadoop.hbase.HealthCheckChore;
74 import org.apache.hadoop.hbase.MetaTableAccessor;
75 import org.apache.hadoop.hbase.NotServingRegionException;
76 import org.apache.hadoop.hbase.PleaseHoldException;
77 import org.apache.hadoop.hbase.RemoteExceptionHandler;
78 import org.apache.hadoop.hbase.ScheduledChore;
79 import org.apache.hadoop.hbase.ServerName;
80 import org.apache.hadoop.hbase.Stoppable;
81 import org.apache.hadoop.hbase.TableDescriptors;
82 import org.apache.hadoop.hbase.TableName;
83 import org.apache.hadoop.hbase.YouAreDeadException;
84 import org.apache.hadoop.hbase.ZNodeClearer;
85 import org.apache.hadoop.hbase.backup.impl.BackupManager;
86 import org.apache.hadoop.hbase.classification.InterfaceAudience;
87 import org.apache.hadoop.hbase.client.ClusterConnection;
88 import org.apache.hadoop.hbase.client.ConnectionFactory;
89 import org.apache.hadoop.hbase.client.ConnectionUtils;
90 import org.apache.hadoop.hbase.client.RpcRetryingCallerFactory;
91 import org.apache.hadoop.hbase.conf.ConfigurationManager;
92 import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
93 import org.apache.hadoop.hbase.coordination.CloseRegionCoordination;
94 import org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination;
95 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
96 import org.apache.hadoop.hbase.exceptions.RegionMovedException;
97 import org.apache.hadoop.hbase.exceptions.RegionOpeningException;
98 import org.apache.hadoop.hbase.exceptions.UnknownProtocolException;
99 import org.apache.hadoop.hbase.executor.ExecutorService;
100 import org.apache.hadoop.hbase.executor.ExecutorType;
101 import org.apache.hadoop.hbase.fs.HFileSystem;
102 import org.apache.hadoop.hbase.http.HttpServer;
103 import org.apache.hadoop.hbase.http.InfoServer;
104 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
105 import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils;
106 import org.apache.hadoop.hbase.ipc.RpcClient;
107 import org.apache.hadoop.hbase.ipc.RpcClientFactory;
108 import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
109 import org.apache.hadoop.hbase.ipc.RpcServerInterface;
110 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
111 import org.apache.hadoop.hbase.ipc.ServerRpcController;
112 import org.apache.hadoop.hbase.master.HMaster;
113 import org.apache.hadoop.hbase.master.RegionState.State;
114 import org.apache.hadoop.hbase.master.TableLockManager;
115 import org.apache.hadoop.hbase.mob.MobCacheConfig;
116 import org.apache.hadoop.hbase.procedure.RegionServerProcedureManagerHost;
117 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
118 import org.apache.hadoop.hbase.protobuf.RequestConverter;
119 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
120 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceCall;
121 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
122 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
123 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos;
124 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad;
125 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionStoreSequenceIds;
126 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.Coprocessor;
127 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.Coprocessor.Builder;
128 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
129 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo;
130 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier;
131 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType;
132 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdRequest;
133 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdResponse;
134 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
135 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
136 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupResponse;
137 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStatusService;
138 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionSpaceUse;
139 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportRequest;
140 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition;
141 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
142 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorRequest;
143 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest;
144 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionResponse;
145 import org.apache.hadoop.hbase.quotas.FileSystemUtilizationChore;
146 import org.apache.hadoop.hbase.quotas.QuotaUtil;
147 import org.apache.hadoop.hbase.quotas.RegionServerRpcQuotaManager;
148 import org.apache.hadoop.hbase.quotas.RegionServerSpaceQuotaManager;
149 import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
150 import org.apache.hadoop.hbase.regionserver.handler.CloseMetaHandler;
151 import org.apache.hadoop.hbase.regionserver.handler.CloseRegionHandler;
152 import org.apache.hadoop.hbase.regionserver.handler.RegionReplicaFlushHandler;
153 import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
154 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
155 import org.apache.hadoop.hbase.replication.regionserver.Replication;
156 import org.apache.hadoop.hbase.replication.regionserver.ReplicationLoad;
157 import org.apache.hadoop.hbase.security.Superusers;
158 import org.apache.hadoop.hbase.security.UserProvider;
159 import org.apache.hadoop.hbase.trace.SpanReceiverHost;
160 import org.apache.hadoop.hbase.util.Addressing;
161 import org.apache.hadoop.hbase.util.ByteStringer;
162 import org.apache.hadoop.hbase.util.Bytes;
163 import org.apache.hadoop.hbase.util.CompressionTest;
164 import org.apache.hadoop.hbase.util.ConfigUtil;
165 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
166 import org.apache.hadoop.hbase.util.FSTableDescriptors;
167 import org.apache.hadoop.hbase.util.FSUtils;
168 import org.apache.hadoop.hbase.util.HasThread;
169 import org.apache.hadoop.hbase.util.JSONBean;
170 import org.apache.hadoop.hbase.util.JvmPauseMonitor;
171 import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
172 import org.apache.hadoop.hbase.util.Sleeper;
173 import org.apache.hadoop.hbase.util.Threads;
174 import org.apache.hadoop.hbase.util.VersionInfo;
175 import org.apache.hadoop.hbase.wal.DefaultWALProvider;
176 import org.apache.hadoop.hbase.wal.WAL;
177 import org.apache.hadoop.hbase.wal.WALFactory;
178 import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
179 import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
180 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
181 import org.apache.hadoop.hbase.zookeeper.RecoveringRegionWatcher;
182 import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
183 import org.apache.hadoop.hbase.zookeeper.ZKSplitLog;
184 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
185 import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker;
186 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
187 import org.apache.hadoop.ipc.RemoteException;
188 import org.apache.hadoop.metrics.util.MBeanUtil;
189 import org.apache.hadoop.util.ReflectionUtils;
190 import org.apache.hadoop.util.StringUtils;
191 import org.apache.zookeeper.KeeperException;
192 import org.apache.zookeeper.KeeperException.NoNodeException;
193 import org.apache.zookeeper.data.Stat;
194
195 import com.google.common.annotations.VisibleForTesting;
196 import com.google.common.base.Preconditions;
197 import com.google.common.collect.Maps;
198 import com.google.protobuf.BlockingRpcChannel;
199 import com.google.protobuf.Descriptors;
200 import com.google.protobuf.Message;
201 import com.google.protobuf.RpcCallback;
202 import com.google.protobuf.RpcController;
203 import com.google.protobuf.Service;
204 import com.google.protobuf.ServiceException;
205
206
207
208
209
210 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
211 @SuppressWarnings("deprecation")
212 public class HRegionServer extends HasThread implements
213 RegionServerServices, LastSequenceId {
214
215 public static final Log LOG = LogFactory.getLog(HRegionServer.class);
216
217
218
219
220
221 protected static final String OPEN = "OPEN";
222 protected static final String CLOSE = "CLOSE";
223
224
225
226
227 protected final ConcurrentMap<byte[], Boolean> regionsInTransitionInRS =
228 new ConcurrentSkipListMap<byte[], Boolean>(Bytes.BYTES_COMPARATOR);
229
230
231 protected MemStoreFlusher cacheFlusher;
232
233 protected HeapMemoryManager hMemManager;
234 protected CountDownLatch initLatch = null;
235
236
237
238
239
240
241 protected ClusterConnection clusterConnection;
242
243
244
245
246
247
248
249 protected MetaTableLocator metaTableLocator;
250
251
252 @SuppressWarnings("unused")
253 private RecoveringRegionWatcher recoveringRegionWatcher;
254
255
256
257
258 protected TableDescriptors tableDescriptors;
259
260
261 protected ReplicationSourceService replicationSourceHandler;
262 protected ReplicationSinkService replicationSinkHandler;
263
264
265 public CompactSplitThread compactSplitThread;
266
267
268
269
270
271 protected final Map<String, Region> onlineRegions = new ConcurrentHashMap<String, Region>();
272
273
274
275
276
277
278
279
280
281
282 protected final Map<String, InetSocketAddress[]> regionFavoredNodesMap =
283 new ConcurrentHashMap<String, InetSocketAddress[]>();
284
285
286
287
288
289 protected final Map<String, Region> recoveringRegions = Collections
290 .synchronizedMap(new HashMap<String, Region>());
291
292
293 protected Leases leases;
294
295
296 protected ExecutorService service;
297
298
299 protected volatile boolean fsOk;
300 protected HFileSystem fs;
301 protected HFileSystem walFs;
302
303
304
305
306 private volatile boolean stopped = false;
307
308
309
310 private volatile boolean abortRequested;
311
312 ConcurrentMap<String, Integer> rowlocks = new ConcurrentHashMap<String, Integer>();
313
314
315
316 private boolean stopping = false;
317
318 private volatile boolean killed = false;
319
320 protected final Configuration conf;
321
322 private Path rootDir;
323 private Path walRootDir;
324
325 protected final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
326
327 final int numRetries;
328 protected final int threadWakeFrequency;
329 protected final int msgInterval;
330
331 protected final int numRegionsToReport;
332
333
334 private volatile RegionServerStatusService.BlockingInterface rssStub;
335
336 RpcClient rpcClient;
337
338 private RpcRetryingCallerFactory rpcRetryingCallerFactory;
339 private RpcControllerFactory rpcControllerFactory;
340
341 private UncaughtExceptionHandler uncaughtExceptionHandler;
342
343
344
345
346 protected InfoServer infoServer;
347 private JvmPauseMonitor pauseMonitor;
348
349
350 public static final String REGIONSERVER = "regionserver";
351
352 MetricsRegionServer metricsRegionServer;
353 MetricsTable metricsTable;
354 private SpanReceiverHost spanReceiverHost;
355
356
357
358
359 private final ChoreService choreService;
360
361
362
363
364 ScheduledChore compactionChecker;
365
366
367
368
369 ScheduledChore periodicFlusher;
370
371 protected volatile WALFactory walFactory;
372
373
374
375 final LogRoller walRoller;
376
377 final AtomicReference<LogRoller> metawalRoller = new AtomicReference<LogRoller>();
378
379
380 final AtomicBoolean online = new AtomicBoolean(false);
381
382
383 protected ZooKeeperWatcher zooKeeper;
384
385
386 private MasterAddressTracker masterAddressTracker;
387
388
389 protected ClusterStatusTracker clusterStatusTracker;
390
391
392 private SplitLogWorker splitLogWorker;
393
394
395 protected final Sleeper sleeper;
396
397 private final int operationTimeout;
398 private final int shortOperationTimeout;
399
400 private final RegionServerAccounting regionServerAccounting;
401
402
403 protected CacheConfig cacheConfig;
404
405 final MobCacheConfig mobCacheConfig;
406
407
408 private HealthCheckChore healthCheckChore;
409
410
411 private ScheduledChore nonceManagerChore;
412
413 private Map<String, Service> coprocessorServiceHandlers = Maps.newHashMap();
414
415
416
417
418
419
420 protected ServerName serverName;
421
422
423
424
425 private String useThisHostnameInstead;
426
427
428
429
430 final static String RS_HOSTNAME_KEY = "hbase.regionserver.hostname";
431
432 final static String MASTER_HOSTNAME_KEY = "hbase.master.hostname";
433
434
435
436 final static String RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY =
437 "hbase.regionserver.hostname.disable.master.reversedns";
438
439
440
441
442 protected final long startcode;
443
444
445
446
447 private String clusterId;
448
449
450
451
452 private ObjectName mxBean = null;
453
454
455
456
457 private MovedRegionsCleaner movedRegionsCleaner;
458
459
460 private StorefileRefresherChore storefileRefresher;
461
462 private RegionServerCoprocessorHost rsHost;
463
464 private RegionServerProcedureManagerHost rspmHost;
465
466 private RegionServerRpcQuotaManager rsQuotaManager;
467 private RegionServerSpaceQuotaManager rsSpaceQuotaManager;
468
469
470 protected TableLockManager tableLockManager;
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490 final ServerNonceManager nonceManager;
491
492 private UserProvider userProvider;
493
494 protected final RSRpcServices rpcServices;
495
496 protected BaseCoordinatedStateManager csm;
497
498 private final boolean useZKForAssignment;
499
500
501
502
503
504 protected final ConfigurationManager configurationManager;
505
506 protected FileSystemUtilizationChore fsUtilizationChore;
507
508
509
510
511
512
513
514 public HRegionServer(Configuration conf) throws IOException, InterruptedException {
515 this(conf, CoordinatedStateManagerFactory.getCoordinatedStateManager(conf));
516 }
517
518
519
520
521
522
523
524
525 public HRegionServer(Configuration conf, CoordinatedStateManager csm)
526 throws IOException, InterruptedException {
527 this.fsOk = true;
528 this.conf = conf;
529 checkCodecs(this.conf);
530 this.userProvider = UserProvider.instantiate(conf);
531 FSUtils.setupShortCircuitRead(this.conf);
532
533 Replication.decorateRegionServerConfiguration(this.conf);
534
535
536 this.conf.setBoolean(HConstants.USE_META_REPLICAS, false);
537 BackupManager.decorateRSConfiguration(conf);
538
539 this.numRetries = this.conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
540 HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
541 this.threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000);
542 this.msgInterval = conf.getInt("hbase.regionserver.msginterval", 3 * 1000);
543
544 this.sleeper = new Sleeper(this.msgInterval, this);
545
546 boolean isNoncesEnabled = conf.getBoolean(HConstants.HBASE_RS_NONCES_ENABLED, true);
547 this.nonceManager = isNoncesEnabled ? new ServerNonceManager(this.conf) : null;
548
549 this.numRegionsToReport = conf.getInt(
550 "hbase.regionserver.numregionstoreport", 10);
551
552 this.operationTimeout = conf.getInt(
553 HConstants.HBASE_CLIENT_OPERATION_TIMEOUT,
554 HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT);
555
556 this.shortOperationTimeout = conf.getInt(
557 HConstants.HBASE_RPC_SHORTOPERATION_TIMEOUT_KEY,
558 HConstants.DEFAULT_HBASE_RPC_SHORTOPERATION_TIMEOUT);
559
560 this.abortRequested = false;
561 this.stopped = false;
562
563 rpcServices = createRpcServices();
564 this.startcode = System.currentTimeMillis();
565 if (this instanceof HMaster) {
566 useThisHostnameInstead = conf.get(MASTER_HOSTNAME_KEY);
567 } else {
568 useThisHostnameInstead = conf.get(RS_HOSTNAME_KEY);
569 if (conf.getBoolean(RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY, false)) {
570 if (shouldUseThisHostnameInstead()) {
571 String msg = RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY + " and " + RS_HOSTNAME_KEY +
572 " are mutually exclusive. Do not set " + RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY +
573 " to true while " + RS_HOSTNAME_KEY + " is used";
574 throw new IOException(msg);
575 } else {
576 useThisHostnameInstead = rpcServices.isa.getHostName();
577 }
578 }
579 }
580 String hostName = shouldUseThisHostnameInstead() ? useThisHostnameInstead :
581 rpcServices.isa.getHostName();
582 serverName = ServerName.valueOf(hostName, rpcServices.isa.getPort(), startcode);
583
584 rpcControllerFactory = RpcControllerFactory.instantiate(this.conf);
585 rpcRetryingCallerFactory = RpcRetryingCallerFactory.instantiate(this.conf);
586
587
588 ZKUtil.loginClient(this.conf, "hbase.zookeeper.client.keytab.file",
589 "hbase.zookeeper.client.kerberos.principal", hostName);
590
591 login(userProvider, hostName);
592
593
594 Superusers.initialize(conf);
595
596 regionServerAccounting = new RegionServerAccounting();
597 cacheConfig = new CacheConfig(conf);
598 mobCacheConfig = new MobCacheConfig(conf);
599 uncaughtExceptionHandler = new UncaughtExceptionHandler() {
600 @Override
601 public void uncaughtException(Thread t, Throwable e) {
602 abort("Uncaught exception in service thread " + t.getName(), e);
603 }
604 };
605
606 useZKForAssignment = ConfigUtil.useZKForAssignment(conf);
607
608 initializeFileSystem();
609
610 service = new ExecutorService(getServerName().toShortString());
611 spanReceiverHost = SpanReceiverHost.getInstance(getConfiguration());
612
613
614 if (!conf.getBoolean("hbase.testing.nocluster", false)) {
615
616 zooKeeper = new ZooKeeperWatcher(conf, getProcessName() + ":" +
617 rpcServices.isa.getPort(), this, canCreateBaseZNode());
618
619 this.csm = (BaseCoordinatedStateManager) csm;
620 this.csm.initialize(this);
621 this.csm.start();
622
623 tableLockManager = TableLockManager.createTableLockManager(
624 conf, zooKeeper, serverName);
625
626 masterAddressTracker = new MasterAddressTracker(getZooKeeper(), this);
627 masterAddressTracker.start();
628
629 clusterStatusTracker = new ClusterStatusTracker(zooKeeper, this);
630 clusterStatusTracker.start();
631 }
632 this.configurationManager = new ConfigurationManager();
633
634 rpcServices.start();
635 putUpWebUI();
636 this.walRoller = new LogRoller(this, this);
637 this.choreService = new ChoreService(getServerName().toString());
638 }
639
640 protected void setInitLatch(CountDownLatch latch) {
641 this.initLatch = latch;
642 }
643
644 private void initializeFileSystem() throws IOException {
645
646
647 boolean useHBaseChecksum = conf.getBoolean(HConstants.HBASE_CHECKSUM_VERIFICATION, true);
648 FSUtils.setFsDefault(this.conf, FSUtils.getWALRootDir(this.conf));
649 this.walFs = new HFileSystem(this.conf, useHBaseChecksum);
650 this.walRootDir = FSUtils.getWALRootDir(this.conf);
651
652
653
654 FSUtils.setFsDefault(this.conf, FSUtils.getRootDir(this.conf));
655 this.fs = new HFileSystem(this.conf, useHBaseChecksum);
656 this.rootDir = FSUtils.getRootDir(this.conf);
657 this.tableDescriptors = new FSTableDescriptors(this.conf, this.fs, this.rootDir,
658 !canUpdateTableDescriptor(), false);
659 }
660
661
662
663
664 protected boolean shouldUseThisHostnameInstead() {
665 return useThisHostnameInstead != null && !useThisHostnameInstead.isEmpty();
666 }
667
668 protected void login(UserProvider user, String host) throws IOException {
669 user.login("hbase.regionserver.keytab.file",
670 "hbase.regionserver.kerberos.principal", host);
671 }
672
673 protected void waitForMasterActive(){
674 }
675
676 protected String getProcessName() {
677 return REGIONSERVER;
678 }
679
680 protected boolean canCreateBaseZNode() {
681 return false;
682 }
683
684 protected boolean canUpdateTableDescriptor() {
685 return false;
686 }
687
688 protected RSRpcServices createRpcServices() throws IOException {
689 return new RSRpcServices(this);
690 }
691
692 protected void configureInfoServer() {
693 infoServer.addServlet("rs-status", "/rs-status", RSStatusServlet.class);
694 infoServer.setAttribute(REGIONSERVER, this);
695 }
696
697 protected Class<? extends HttpServlet> getDumpServlet() {
698 return RSDumpServlet.class;
699 }
700
701 @Override
702 public boolean registerService(Service instance) {
703
704
705
706 Descriptors.ServiceDescriptor serviceDesc = instance.getDescriptorForType();
707 String serviceName = CoprocessorRpcUtils.getServiceName(serviceDesc);
708 if (coprocessorServiceHandlers.containsKey(serviceName)) {
709 LOG.error("Coprocessor service " + serviceName
710 + " already registered, rejecting request from " + instance);
711 return false;
712 }
713
714 coprocessorServiceHandlers.put(serviceName, instance);
715 if (LOG.isDebugEnabled()) {
716 LOG.debug("Registered regionserver coprocessor service: service=" + serviceName);
717 }
718 return true;
719 }
720
721
722
723
724
725
726
727 @VisibleForTesting
728 protected ClusterConnection createClusterConnection() throws IOException {
729
730
731
732 return ConnectionUtils.createShortCircuitHConnection(
733 ConnectionFactory.createConnection(conf), serverName, rpcServices, rpcServices);
734 }
735
736
737
738
739
740
741 private static void checkCodecs(final Configuration c) throws IOException {
742
743 String [] codecs = c.getStrings("hbase.regionserver.codecs", (String[])null);
744 if (codecs == null) return;
745 for (String codec : codecs) {
746 if (!CompressionTest.testCompression(codec)) {
747 throw new IOException("Compression codec " + codec +
748 " not supported, aborting RS construction");
749 }
750 }
751 }
752
753 public String getClusterId() {
754 return this.clusterId;
755 }
756
757
758
759
760
761 protected synchronized void setupClusterConnection() throws IOException {
762 if (clusterConnection == null) {
763 clusterConnection = createClusterConnection();
764 metaTableLocator = new MetaTableLocator();
765 }
766 }
767
768
769
770
771
772
773
774 private void preRegistrationInitialization(){
775 try {
776 setupClusterConnection();
777
778
779 if (isHealthCheckerConfigured()) {
780 int sleepTime = this.conf.getInt(HConstants.HEALTH_CHORE_WAKE_FREQ,
781 HConstants.DEFAULT_THREAD_WAKE_FREQUENCY);
782 healthCheckChore = new HealthCheckChore(sleepTime, this, getConfiguration());
783 }
784
785 initializeZooKeeper();
786 if (!isStopped() && !isAborted()) {
787 initializeThreads();
788 }
789 } catch (Throwable t) {
790
791
792 this.rpcServices.stop();
793 abort("Initialization of RS failed. Hence aborting RS.", t);
794 }
795 }
796
797
798
799
800
801
802
803
804
805 private void initializeZooKeeper() throws IOException, InterruptedException {
806
807
808
809 blockAndCheckIfStopped(this.masterAddressTracker);
810
811
812
813 blockAndCheckIfStopped(this.clusterStatusTracker);
814
815 if (this.initLatch != null) {
816 this.initLatch.await(20, TimeUnit.SECONDS);
817 }
818
819
820
821 try {
822 clusterId = ZKClusterId.readClusterIdZNode(this.zooKeeper);
823 if (clusterId == null) {
824 this.abort("Cluster ID has not been set");
825 }
826 LOG.info("ClusterId : "+clusterId);
827 } catch (KeeperException e) {
828 this.abort("Failed to retrieve Cluster ID",e);
829 }
830
831
832
833
834
835 waitForMasterActive();
836 if (isStopped() || isAborted()) {
837 return;
838 }
839
840
841 try {
842 rspmHost = new RegionServerProcedureManagerHost();
843 rspmHost.loadProcedures(conf);
844 rspmHost.initialize(this);
845 } catch (IOException e) {
846 this.abort("Failed to reach coordination cluster when creating procedure handler.", e);
847 }
848
849 this.recoveringRegionWatcher = new RecoveringRegionWatcher(this.zooKeeper, this);
850 }
851
852
853
854
855
856
857
858
859 private void blockAndCheckIfStopped(ZooKeeperNodeTracker tracker)
860 throws IOException, InterruptedException {
861 while (tracker.blockUntilAvailable(this.msgInterval, false) == null) {
862 if (this.stopped) {
863 throw new IOException("Received the shutdown message while waiting.");
864 }
865 }
866 }
867
868
869
870
871 private boolean isClusterUp() {
872 return clusterStatusTracker != null && clusterStatusTracker.isClusterUp();
873 }
874
875 private void initializeThreads() throws IOException {
876
877 this.cacheFlusher = new MemStoreFlusher(conf, this);
878
879
880 this.compactSplitThread = new CompactSplitThread(this);
881
882
883
884 this.compactionChecker = new CompactionChecker(this, this.threadWakeFrequency, this);
885 this.periodicFlusher = new PeriodicMemstoreFlusher(this.threadWakeFrequency, this);
886 this.leases = new Leases(this.threadWakeFrequency);
887
888
889 movedRegionsCleaner = MovedRegionsCleaner.create(this);
890
891 if (this.nonceManager != null) {
892
893 nonceManagerChore = this.nonceManager.createCleanupScheduledChore(this);
894 }
895
896
897 rsQuotaManager = new RegionServerRpcQuotaManager(this);
898 rsSpaceQuotaManager = new RegionServerSpaceQuotaManager(this);
899
900 if (QuotaUtil.isQuotaEnabled(conf)) {
901 this.fsUtilizationChore = new FileSystemUtilizationChore(this);
902 }
903
904
905 rpcClient = RpcClientFactory.createClient(conf, clusterId, new InetSocketAddress(
906 rpcServices.isa.getAddress(), 0));
907
908 boolean onlyMetaRefresh = false;
909 int storefileRefreshPeriod = conf.getInt(
910 StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD
911 , StorefileRefresherChore.DEFAULT_REGIONSERVER_STOREFILE_REFRESH_PERIOD);
912 if (storefileRefreshPeriod == 0) {
913 storefileRefreshPeriod = conf.getInt(
914 StorefileRefresherChore.REGIONSERVER_META_STOREFILE_REFRESH_PERIOD,
915 StorefileRefresherChore.DEFAULT_REGIONSERVER_STOREFILE_REFRESH_PERIOD);
916 onlyMetaRefresh = true;
917 }
918 if (storefileRefreshPeriod > 0) {
919 this.storefileRefresher = new StorefileRefresherChore(storefileRefreshPeriod,
920 onlyMetaRefresh, this, this);
921 }
922 registerConfigurationObservers();
923 }
924
925 private void registerConfigurationObservers() {
926
927 configurationManager.registerObserver(this.compactSplitThread);
928 }
929
930
931
932
933 @Override
934 public void run() {
935 try {
936
937 preRegistrationInitialization();
938 } catch (Throwable e) {
939 abort("Fatal exception during initialization", e);
940 }
941
942 try {
943 if (!isStopped() && !isAborted()) {
944 ShutdownHook.install(conf, fs, this, Thread.currentThread());
945
946 createMyEphemeralNode();
947
948
949 this.rsHost = new RegionServerCoprocessorHost(this, this.conf);
950 }
951
952
953
954 while (keepLooping()) {
955 RegionServerStartupResponse w = reportForDuty();
956 if (w == null) {
957 LOG.warn("reportForDuty failed; sleeping and then retrying.");
958 this.sleeper.sleep();
959 } else {
960 handleReportForDutyResponse(w);
961 break;
962 }
963 }
964
965 if (!isStopped() && isHealthy()){
966
967
968 rspmHost.start();
969 }
970
971
972 if (this.rsQuotaManager != null) {
973 rsQuotaManager.start(getRpcServer().getScheduler());
974 rsSpaceQuotaManager.start();
975 }
976
977
978 long lastMsg = System.currentTimeMillis();
979 long oldRequestCount = -1;
980
981 while (!isStopped() && isHealthy()) {
982 if (!isClusterUp()) {
983 if (isOnlineRegionsEmpty()) {
984 stop("Exiting; cluster shutdown set and not carrying any regions");
985 } else if (!this.stopping) {
986 this.stopping = true;
987 LOG.info("Closing user regions");
988 closeUserRegions(this.abortRequested);
989 } else if (this.stopping) {
990 boolean allUserRegionsOffline = areAllUserRegionsOffline();
991 if (allUserRegionsOffline) {
992
993
994
995 if (oldRequestCount == getWriteRequestCount()) {
996 stop("Stopped; only catalog regions remaining online");
997 break;
998 }
999 oldRequestCount = getWriteRequestCount();
1000 } else {
1001
1002
1003
1004 closeUserRegions(this.abortRequested);
1005 }
1006 LOG.debug("Waiting on " + getOnlineRegionsAsPrintableString());
1007 }
1008 }
1009 long now = System.currentTimeMillis();
1010 if ((now - lastMsg) >= msgInterval) {
1011 tryRegionServerReport(lastMsg, now);
1012 lastMsg = System.currentTimeMillis();
1013 }
1014 if (!isStopped() && !isAborted()) {
1015 this.sleeper.sleep();
1016 }
1017 }
1018 } catch (Throwable t) {
1019 if (!rpcServices.checkOOME(t)) {
1020 String prefix = t instanceof YouAreDeadException? "": "Unhandled: ";
1021 abort(prefix + t.getMessage(), t);
1022 }
1023 }
1024
1025 if (mxBean != null) {
1026 MBeanUtil.unregisterMBean(mxBean);
1027 mxBean = null;
1028 }
1029 if (this.leases != null) this.leases.closeAfterLeasesExpire();
1030 if (this.splitLogWorker != null) {
1031 splitLogWorker.stop();
1032 }
1033 if (this.infoServer != null) {
1034 LOG.info("Stopping infoServer");
1035 try {
1036 this.infoServer.stop();
1037 } catch (Exception e) {
1038 LOG.error("Failed to stop infoServer", e);
1039 }
1040 }
1041
1042 if (cacheConfig != null && cacheConfig.isBlockCacheEnabled()) {
1043 cacheConfig.getBlockCache().shutdown();
1044 }
1045 mobCacheConfig.getMobFileCache().shutdown();
1046
1047 if (movedRegionsCleaner != null) {
1048 movedRegionsCleaner.stop("Region Server stopping");
1049 }
1050
1051
1052
1053 if (this.hMemManager != null) this.hMemManager.stop();
1054 if (this.cacheFlusher != null) this.cacheFlusher.interruptIfNecessary();
1055 if (this.compactSplitThread != null) this.compactSplitThread.interruptIfNecessary();
1056 if (this.compactionChecker != null) this.compactionChecker.cancel(true);
1057 if (this.healthCheckChore != null) this.healthCheckChore.cancel(true);
1058 if (this.nonceManagerChore != null) this.nonceManagerChore.cancel(true);
1059 if (this.storefileRefresher != null) this.storefileRefresher.cancel(true);
1060 sendShutdownInterrupt();
1061
1062
1063 if (rsQuotaManager != null) {
1064 rsQuotaManager.stop();
1065 }
1066 if (rsSpaceQuotaManager != null) {
1067 rsSpaceQuotaManager.stop();
1068 }
1069
1070
1071 if (rspmHost != null) {
1072 rspmHost.stop(this.abortRequested || this.killed);
1073 }
1074
1075 if (this.killed) {
1076
1077 } else if (abortRequested) {
1078 if (this.fsOk) {
1079 closeUserRegions(abortRequested);
1080 }
1081 LOG.info("aborting server " + this.serverName);
1082 } else {
1083 closeUserRegions(abortRequested);
1084 LOG.info("stopping server " + this.serverName);
1085 }
1086
1087
1088 if (this.metaTableLocator != null) this.metaTableLocator.stop();
1089 if (this.clusterConnection != null && !clusterConnection.isClosed()) {
1090 try {
1091 this.clusterConnection.close();
1092 } catch (IOException e) {
1093
1094
1095 LOG.warn("Attempt to close server's short circuit HConnection failed.", e);
1096 }
1097 }
1098
1099
1100 if (!this.killed && containsMetaTableRegions()) {
1101 if (!abortRequested || this.fsOk) {
1102 if (this.compactSplitThread != null) {
1103 this.compactSplitThread.join();
1104 this.compactSplitThread = null;
1105 }
1106 closeMetaTableRegions(abortRequested);
1107 }
1108 }
1109
1110 if (!this.killed && this.fsOk) {
1111 waitOnAllRegionsToClose(abortRequested);
1112 LOG.info("stopping server " + this.serverName +
1113 "; all regions closed.");
1114 }
1115
1116
1117 if (this.fsOk) {
1118 shutdownWAL(!abortRequested);
1119 }
1120
1121
1122 if (this.rssStub != null) {
1123 this.rssStub = null;
1124 }
1125 if (this.rpcClient != null) {
1126 this.rpcClient.close();
1127 }
1128 if (this.leases != null) {
1129 this.leases.close();
1130 }
1131 if (this.pauseMonitor != null) {
1132 this.pauseMonitor.stop();
1133 }
1134
1135 if (!killed) {
1136 stopServiceThreads();
1137 }
1138
1139 if (this.rpcServices != null) {
1140 this.rpcServices.stop();
1141 }
1142
1143 try {
1144 deleteMyEphemeralNode();
1145 } catch (KeeperException.NoNodeException nn) {
1146 } catch (KeeperException e) {
1147 LOG.warn("Failed deleting my ephemeral node", e);
1148 }
1149
1150
1151 ZNodeClearer.deleteMyEphemeralNodeOnDisk();
1152
1153 if (this.zooKeeper != null) {
1154 this.zooKeeper.close();
1155 }
1156 LOG.info("stopping server " + this.serverName +
1157 "; zookeeper connection closed.");
1158
1159 LOG.info(Thread.currentThread().getName() + " exiting");
1160 }
1161
1162 private boolean containsMetaTableRegions() {
1163 return onlineRegions.containsKey(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName());
1164 }
1165
1166 private boolean areAllUserRegionsOffline() {
1167 if (getNumberOfOnlineRegions() > 2) return false;
1168 boolean allUserRegionsOffline = true;
1169 for (Map.Entry<String, Region> e: this.onlineRegions.entrySet()) {
1170 if (!e.getValue().getRegionInfo().isMetaTable()) {
1171 allUserRegionsOffline = false;
1172 break;
1173 }
1174 }
1175 return allUserRegionsOffline;
1176 }
1177
1178
1179
1180
1181 private long getWriteRequestCount() {
1182 long writeCount = 0;
1183 for (Map.Entry<String, Region> e: this.onlineRegions.entrySet()) {
1184 writeCount += e.getValue().getWriteRequestsCount();
1185 }
1186 return writeCount;
1187 }
1188
1189 @VisibleForTesting
1190 protected void tryRegionServerReport(long reportStartTime, long reportEndTime)
1191 throws IOException {
1192 RegionServerStatusService.BlockingInterface rss = rssStub;
1193 if (rss == null) {
1194
1195 return;
1196 }
1197 ClusterStatusProtos.ServerLoad sl = buildServerLoad(reportStartTime, reportEndTime);
1198 try {
1199 RegionServerReportRequest.Builder request = RegionServerReportRequest.newBuilder();
1200 ServerName sn = ServerName.parseVersionedServerName(
1201 this.serverName.getVersionedBytes());
1202 request.setServer(ProtobufUtil.toServerName(sn));
1203 request.setLoad(sl);
1204 rss.regionServerReport(null, request.build());
1205 } catch (ServiceException se) {
1206 IOException ioe = ProtobufUtil.getRemoteException(se);
1207 if (ioe instanceof YouAreDeadException) {
1208
1209 throw ioe;
1210 }
1211 if (rssStub == rss) {
1212 rssStub = null;
1213 }
1214
1215
1216 createRegionServerStatusStub();
1217 }
1218 }
1219
1220
1221
1222
1223
1224
1225
1226 public boolean reportRegionSizesForQuotas(final Map<HRegionInfo, Long> onlineRegionSizes) {
1227 RegionServerStatusService.BlockingInterface rss = rssStub;
1228 if (rss == null) {
1229
1230 LOG.trace("Skipping Region size report to HMaster as stub is null");
1231 return true;
1232 }
1233 try {
1234 RegionSpaceUseReportRequest request = buildRegionSpaceUseReportRequest(
1235 Objects.requireNonNull(onlineRegionSizes));
1236 rss.reportRegionSpaceUse(null, request);
1237 } catch (ServiceException se) {
1238 IOException ioe = ProtobufUtil.getRemoteException(se);
1239 if (ioe instanceof PleaseHoldException) {
1240 LOG.trace("Failed to report region sizes to Master because it is initializing."
1241 + " This will be retried.", ioe);
1242
1243 return true;
1244 }
1245 if (rssStub == rss) {
1246 rssStub = null;
1247 }
1248 createRegionServerStatusStub();
1249 if (ioe instanceof DoNotRetryIOException) {
1250 DoNotRetryIOException doNotRetryEx = (DoNotRetryIOException) ioe;
1251 if (doNotRetryEx.getCause() != null) {
1252 Throwable t = doNotRetryEx.getCause();
1253 if (t instanceof UnsupportedOperationException) {
1254 LOG.debug("master doesn't support ReportRegionSpaceUse, pause before retrying");
1255 return false;
1256 }
1257 }
1258 }
1259 LOG.debug("Failed to report region sizes to Master. This will be retried.", ioe);
1260 }
1261 return true;
1262 }
1263
1264
1265
1266
1267
1268
1269
1270 RegionSpaceUseReportRequest buildRegionSpaceUseReportRequest(Map<HRegionInfo,Long> regionSizes) {
1271 RegionSpaceUseReportRequest.Builder request = RegionSpaceUseReportRequest.newBuilder();
1272 for (Entry<HRegionInfo, Long> entry : Objects.requireNonNull(regionSizes).entrySet()) {
1273 request.addSpaceUse(convertRegionSize(entry.getKey(), entry.getValue()));
1274 }
1275 return request.build();
1276 }
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286 RegionSpaceUse convertRegionSize(HRegionInfo regionInfo, Long sizeInBytes) {
1287 return RegionSpaceUse.newBuilder()
1288 .setRegionInfo(HRegionInfo.convert(Objects.requireNonNull(regionInfo)))
1289 .setRegionSize(Objects.requireNonNull(sizeInBytes))
1290 .build();
1291 }
1292
1293 ClusterStatusProtos.ServerLoad buildServerLoad(long reportStartTime, long reportEndTime)
1294 throws IOException {
1295
1296
1297
1298
1299
1300
1301
1302 MetricsRegionServerWrapper regionServerWrapper = metricsRegionServer.getRegionServerWrapper();
1303 Collection<Region> regions = getOnlineRegionsLocalContext();
1304 MemoryUsage memory = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
1305
1306 ClusterStatusProtos.ServerLoad.Builder serverLoad =
1307 ClusterStatusProtos.ServerLoad.newBuilder();
1308 serverLoad.setNumberOfRequests((int) regionServerWrapper.getRequestsPerSecond());
1309 serverLoad.setTotalNumberOfRequests((int) regionServerWrapper.getTotalRequestCount());
1310 serverLoad.setUsedHeapMB((int)(memory.getUsed() / 1024 / 1024));
1311 serverLoad.setMaxHeapMB((int) (memory.getMax() / 1024 / 1024));
1312 Set<String> coprocessors = getWAL(null).getCoprocessorHost().getCoprocessors();
1313 Builder coprocessorBuilder = Coprocessor.newBuilder();
1314 for (String coprocessor : coprocessors) {
1315 serverLoad.addCoprocessors(coprocessorBuilder.setName(coprocessor).build());
1316 }
1317 RegionLoad.Builder regionLoadBldr = RegionLoad.newBuilder();
1318 RegionSpecifier.Builder regionSpecifier = RegionSpecifier.newBuilder();
1319 for (Region region : regions) {
1320 if (region.getCoprocessorHost() != null) {
1321 Set<String> regionCoprocessors = region.getCoprocessorHost().getCoprocessors();
1322 Iterator<String> iterator = regionCoprocessors.iterator();
1323 while (iterator.hasNext()) {
1324 serverLoad.addCoprocessors(coprocessorBuilder.setName(iterator.next()).build());
1325 }
1326 }
1327 serverLoad.addRegionLoads(createRegionLoad(region, regionLoadBldr, regionSpecifier));
1328 for (String coprocessor : getWAL(region.getRegionInfo()).getCoprocessorHost()
1329 .getCoprocessors()) {
1330 serverLoad.addCoprocessors(coprocessorBuilder.setName(coprocessor).build());
1331 }
1332 }
1333 serverLoad.setReportStartTime(reportStartTime);
1334 serverLoad.setReportEndTime(reportEndTime);
1335 if (this.infoServer != null) {
1336 serverLoad.setInfoServerPort(this.infoServer.getPort());
1337 } else {
1338 serverLoad.setInfoServerPort(-1);
1339 }
1340
1341
1342
1343 ReplicationSourceService rsources = getReplicationSourceService();
1344
1345 if (rsources != null) {
1346
1347 ReplicationLoad rLoad = rsources.refreshAndGetReplicationLoad();
1348 if (rLoad != null) {
1349 serverLoad.setReplLoadSink(rLoad.getReplicationLoadSink());
1350 for (ClusterStatusProtos.ReplicationLoadSource rLS : rLoad.getReplicationLoadSourceList()) {
1351 serverLoad.addReplLoadSource(rLS);
1352 }
1353 }
1354 }
1355
1356 return serverLoad.build();
1357 }
1358
1359 String getOnlineRegionsAsPrintableString() {
1360 StringBuilder sb = new StringBuilder();
1361 for (Region r: this.onlineRegions.values()) {
1362 if (sb.length() > 0) sb.append(", ");
1363 sb.append(r.getRegionInfo().getEncodedName());
1364 }
1365 return sb.toString();
1366 }
1367
1368
1369
1370
1371 private void waitOnAllRegionsToClose(final boolean abort) {
1372
1373 int lastCount = -1;
1374 long previousLogTime = 0;
1375 Set<String> closedRegions = new HashSet<String>();
1376 boolean interrupted = false;
1377 try {
1378 while (!isOnlineRegionsEmpty()) {
1379 int count = getNumberOfOnlineRegions();
1380
1381 if (count != lastCount) {
1382
1383 if (System.currentTimeMillis() > (previousLogTime + 1000)) {
1384 previousLogTime = System.currentTimeMillis();
1385 lastCount = count;
1386 LOG.info("Waiting on " + count + " regions to close");
1387
1388
1389 if (count < 10 && LOG.isDebugEnabled()) {
1390 LOG.debug(this.onlineRegions);
1391 }
1392 }
1393 }
1394
1395
1396
1397 for (Map.Entry<String, Region> e : this.onlineRegions.entrySet()) {
1398 HRegionInfo hri = e.getValue().getRegionInfo();
1399 if (!this.regionsInTransitionInRS.containsKey(hri.getEncodedNameAsBytes())
1400 && !closedRegions.contains(hri.getEncodedName())) {
1401 closedRegions.add(hri.getEncodedName());
1402
1403 closeRegionIgnoreErrors(hri, abort);
1404 }
1405 }
1406
1407 if (this.regionsInTransitionInRS.isEmpty()) {
1408 if (!isOnlineRegionsEmpty()) {
1409 LOG.info("We were exiting though online regions are not empty," +
1410 " because some regions failed closing");
1411 }
1412 break;
1413 }
1414 if (sleep(200)) {
1415 interrupted = true;
1416 }
1417 }
1418 } finally {
1419 if (interrupted) {
1420 Thread.currentThread().interrupt();
1421 }
1422 }
1423 }
1424
1425 private boolean sleep(long millis) {
1426 boolean interrupted = false;
1427 try {
1428 Thread.sleep(millis);
1429 } catch (InterruptedException e) {
1430 LOG.warn("Interrupted while sleeping");
1431 interrupted = true;
1432 }
1433 return interrupted;
1434 }
1435
1436 private void shutdownWAL(final boolean close) {
1437 if (this.walFactory != null) {
1438 try {
1439 if (close) {
1440 walFactory.close();
1441 } else {
1442 walFactory.shutdown();
1443 }
1444 } catch (Throwable e) {
1445 e = RemoteExceptionHandler.checkThrowable(e);
1446 LOG.error("Shutdown / close of WAL failed: " + e);
1447 LOG.debug("Shutdown / close exception details:", e);
1448 }
1449 }
1450 }
1451
1452
1453
1454
1455
1456
1457 protected void handleReportForDutyResponse(final RegionServerStartupResponse c)
1458 throws IOException {
1459 try {
1460 boolean updateRootDir = false;
1461 for (NameStringPair e : c.getMapEntriesList()) {
1462 String key = e.getName();
1463
1464 if (key.equals(HConstants.KEY_FOR_HOSTNAME_SEEN_BY_MASTER)) {
1465 String hostnameFromMasterPOV = e.getValue();
1466 this.serverName = ServerName.valueOf(hostnameFromMasterPOV,
1467 rpcServices.isa.getPort(), this.startcode);
1468 if (shouldUseThisHostnameInstead() &&
1469 !hostnameFromMasterPOV.equals(useThisHostnameInstead)) {
1470 String msg = "Master passed us a different hostname to use; was=" +
1471 this.useThisHostnameInstead + ", but now=" + hostnameFromMasterPOV;
1472 LOG.error(msg);
1473 throw new IOException(msg);
1474 }
1475 if (!shouldUseThisHostnameInstead() &&
1476 !hostnameFromMasterPOV.equals(rpcServices.isa.getHostName())) {
1477 String msg = "Master passed us a different hostname to use; was=" +
1478 rpcServices.isa.getHostName() + ", but now=" + hostnameFromMasterPOV;
1479 LOG.error(msg);
1480 }
1481 continue;
1482 }
1483
1484 String value = e.getValue();
1485 if (key.equals(HConstants.HBASE_DIR)) {
1486 if (value != null && !value.equals(conf.get(HConstants.HBASE_DIR))) {
1487 updateRootDir = true;
1488 }
1489 }
1490
1491 if (LOG.isDebugEnabled()) {
1492 LOG.debug("Config from master: " + key + "=" + value);
1493 }
1494 this.conf.set(key, value);
1495 }
1496
1497 if (updateRootDir) {
1498
1499 initializeFileSystem();
1500 }
1501
1502
1503
1504 if (this.conf.get("mapreduce.task.attempt.id") == null) {
1505 this.conf.set("mapreduce.task.attempt.id", "hb_rs_" +
1506 this.serverName.toString());
1507 }
1508
1509
1510 ZNodeClearer.writeMyEphemeralNodeOnDisk(getMyEphemeralNodePath());
1511
1512 this.cacheConfig = new CacheConfig(conf);
1513 this.walFactory = setupWALAndReplication();
1514
1515 this.metricsTable = new MetricsTable(new MetricsTableWrapperAggregateImpl(this));
1516 this.metricsRegionServer = new MetricsRegionServer(getConfiguration(),
1517 new MetricsRegionServerWrapperImpl(this), metricsTable);
1518
1519 this.pauseMonitor = new JvmPauseMonitor(conf, getMetrics().getMetricsSource());
1520 pauseMonitor.start();
1521
1522 startServiceThreads();
1523 startHeapMemoryManager();
1524 LOG.info("Serving as " + this.serverName +
1525 ", RpcServer on " + rpcServices.isa +
1526 ", sessionid=0x" +
1527 Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()));
1528
1529
1530 synchronized (online) {
1531 online.set(true);
1532 online.notifyAll();
1533 }
1534 } catch (Throwable e) {
1535 stop("Failed initialization");
1536 throw convertThrowableToIOE(cleanup(e, "Failed init"),
1537 "Region server startup failed");
1538 } finally {
1539 sleeper.skipSleepCycle();
1540 }
1541 }
1542
1543 private void startHeapMemoryManager() {
1544 this.hMemManager = HeapMemoryManager.create(this.conf, this.cacheFlusher, this);
1545 if (this.hMemManager != null) {
1546 this.hMemManager.start(getChoreService());
1547 }
1548 }
1549
1550 private void createMyEphemeralNode() throws KeeperException, IOException {
1551 RegionServerInfo.Builder rsInfo = RegionServerInfo.newBuilder();
1552 rsInfo.setInfoPort(infoServer != null ? infoServer.getPort() : -1);
1553 byte[] data = ProtobufUtil.prependPBMagic(rsInfo.build().toByteArray());
1554 ZKUtil.createEphemeralNodeAndWatch(this.zooKeeper,
1555 getMyEphemeralNodePath(), data);
1556 }
1557
1558 private void deleteMyEphemeralNode() throws KeeperException {
1559 ZKUtil.deleteNode(this.zooKeeper, getMyEphemeralNodePath());
1560 }
1561
1562 @Override
1563 public RegionServerAccounting getRegionServerAccounting() {
1564 return regionServerAccounting;
1565 }
1566
1567 @Override
1568 public TableLockManager getTableLockManager() {
1569 return tableLockManager;
1570 }
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580 private RegionLoad createRegionLoad(final Region r, RegionLoad.Builder regionLoadBldr,
1581 RegionSpecifier.Builder regionSpecifier) throws IOException {
1582 byte[] name = r.getRegionInfo().getRegionName();
1583 int stores = 0;
1584 int storefiles = 0;
1585 int storeUncompressedSizeMB = 0;
1586 int storefileSizeMB = 0;
1587 int memstoreSizeMB = (int) (r.getMemstoreSize() / 1024 / 1024);
1588 int storefileIndexSizeMB = 0;
1589 int rootIndexSizeKB = 0;
1590 int totalStaticIndexSizeKB = 0;
1591 int totalStaticBloomSizeKB = 0;
1592 long totalCompactingKVs = 0;
1593 long currentCompactedKVs = 0;
1594 List<Store> storeList = r.getStores();
1595 stores += storeList.size();
1596 for (Store store : storeList) {
1597 storefiles += store.getStorefilesCount();
1598 storeUncompressedSizeMB += (int) (store.getStoreSizeUncompressed() / 1024 / 1024);
1599 storefileSizeMB += (int) (store.getStorefilesSize() / 1024 / 1024);
1600 storefileIndexSizeMB += (int) (store.getStorefilesIndexSize() / 1024 / 1024);
1601 CompactionProgress progress = store.getCompactionProgress();
1602 if (progress != null) {
1603 totalCompactingKVs += progress.totalCompactingKVs;
1604 currentCompactedKVs += progress.currentCompactedKVs;
1605 }
1606 rootIndexSizeKB += (int) (store.getStorefilesIndexSize() / 1024);
1607 totalStaticIndexSizeKB += (int) (store.getTotalStaticIndexSize() / 1024);
1608 totalStaticBloomSizeKB += (int) (store.getTotalStaticBloomSize() / 1024);
1609 }
1610
1611 float dataLocality =
1612 r.getHDFSBlocksDistribution().getBlockLocalityIndex(serverName.getHostname());
1613 if (regionLoadBldr == null) {
1614 regionLoadBldr = RegionLoad.newBuilder();
1615 }
1616 if (regionSpecifier == null) {
1617 regionSpecifier = RegionSpecifier.newBuilder();
1618 }
1619 regionSpecifier.setType(RegionSpecifierType.REGION_NAME);
1620 regionSpecifier.setValue(ByteStringer.wrap(name));
1621 regionLoadBldr.setRegionSpecifier(regionSpecifier.build())
1622 .setStores(stores)
1623 .setStorefiles(storefiles)
1624 .setStoreUncompressedSizeMB(storeUncompressedSizeMB)
1625 .setStorefileSizeMB(storefileSizeMB)
1626 .setMemstoreSizeMB(memstoreSizeMB)
1627 .setStorefileIndexSizeMB(storefileIndexSizeMB)
1628 .setRootIndexSizeKB(rootIndexSizeKB)
1629 .setTotalStaticIndexSizeKB(totalStaticIndexSizeKB)
1630 .setTotalStaticBloomSizeKB(totalStaticBloomSizeKB)
1631 .setReadRequestsCount(r.getReadRequestsCount())
1632 .setWriteRequestsCount(r.getWriteRequestsCount())
1633 .setTotalCompactingKVs(totalCompactingKVs)
1634 .setCurrentCompactedKVs(currentCompactedKVs)
1635 .setDataLocality(dataLocality)
1636 .setLastMajorCompactionTs(r.getOldestHfileTs(true));
1637 ((HRegion)r).setCompleteSequenceId(regionLoadBldr);
1638
1639 return regionLoadBldr.build();
1640 }
1641
1642
1643
1644
1645
1646 public RegionLoad createRegionLoad(final String encodedRegionName) throws IOException {
1647 Region r = onlineRegions.get(encodedRegionName);
1648 return r != null ? createRegionLoad(r, null, null) : null;
1649 }
1650
1651
1652
1653
1654 private static class CompactionChecker extends ScheduledChore {
1655 private final HRegionServer instance;
1656 private final int majorCompactPriority;
1657 private final static int DEFAULT_PRIORITY = Integer.MAX_VALUE;
1658 private long iteration = 0;
1659
1660 CompactionChecker(final HRegionServer h, final int sleepTime,
1661 final Stoppable stopper) {
1662 super("CompactionChecker", stopper, sleepTime);
1663 this.instance = h;
1664 LOG.info(this.getName() + " runs every " + StringUtils.formatTime(sleepTime));
1665
1666
1667
1668
1669 this.majorCompactPriority = this.instance.conf.
1670 getInt("hbase.regionserver.compactionChecker.majorCompactPriority",
1671 DEFAULT_PRIORITY);
1672 }
1673
1674 @Override
1675 protected void chore() {
1676 for (Region r : this.instance.onlineRegions.values()) {
1677 if (r == null)
1678 continue;
1679 for (Store s : r.getStores()) {
1680 try {
1681 long multiplier = s.getCompactionCheckMultiplier();
1682 assert multiplier > 0;
1683 if (iteration % multiplier != 0) continue;
1684 if (s.needsCompaction()) {
1685
1686 this.instance.compactSplitThread.requestSystemCompaction(r, s, getName()
1687 + " requests compaction");
1688 } else if (s.isMajorCompaction()) {
1689 if (majorCompactPriority == DEFAULT_PRIORITY
1690 || majorCompactPriority > ((HRegion)r).getCompactPriority()) {
1691 this.instance.compactSplitThread.requestCompaction(r, s, getName()
1692 + " requests major compaction; use default priority", null);
1693 } else {
1694 this.instance.compactSplitThread.requestCompaction(r, s, getName()
1695 + " requests major compaction; use configured priority",
1696 this.majorCompactPriority, null, null);
1697 }
1698 }
1699 } catch (IOException e) {
1700 LOG.warn("Failed major compaction check on " + r, e);
1701 }
1702 }
1703 }
1704 iteration = (iteration == Long.MAX_VALUE) ? 0 : (iteration + 1);
1705 }
1706 }
1707
1708 static class PeriodicMemstoreFlusher extends ScheduledChore {
1709 final HRegionServer server;
1710 final static int RANGE_OF_DELAY = 20000;
1711 final static int MIN_DELAY_TIME = 3000;
1712 public PeriodicMemstoreFlusher(int cacheFlushInterval, final HRegionServer server) {
1713 super(server.getServerName() + "-MemstoreFlusherChore", server, cacheFlushInterval);
1714 this.server = server;
1715 }
1716
1717 @Override
1718 protected void chore() {
1719 for (Region r : this.server.onlineRegions.values()) {
1720 if (r == null)
1721 continue;
1722 if (((HRegion)r).shouldFlush()) {
1723 FlushRequester requester = server.getFlushRequester();
1724 if (requester != null) {
1725 long randomDelay = RandomUtils.nextInt(RANGE_OF_DELAY) + MIN_DELAY_TIME;
1726 LOG.info(getName() + " requesting flush for region " +
1727 r.getRegionInfo().getRegionNameAsString() + " after a delay of " + randomDelay);
1728
1729
1730
1731 requester.requestDelayedFlush(r, randomDelay, false);
1732 }
1733 }
1734 }
1735 }
1736 }
1737
1738
1739
1740
1741
1742
1743
1744
1745 public boolean isOnline() {
1746 return online.get();
1747 }
1748
1749
1750
1751
1752
1753
1754
1755 private WALFactory setupWALAndReplication() throws IOException {
1756
1757 final Path oldLogDir = new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
1758 final String logName = DefaultWALProvider.getWALDirectoryName(this.serverName.toString());
1759
1760 Path logDir = new Path(walRootDir, logName);
1761 if (LOG.isDebugEnabled()) LOG.debug("logDir=" + logDir);
1762 if (this.walFs.exists(logDir)) {
1763 throw new RegionServerRunningException("Region server has already " +
1764 "created directory at " + this.serverName.toString());
1765 }
1766
1767
1768
1769 createNewReplicationInstance(conf, this, this.walFs, logDir, oldLogDir);
1770
1771
1772 final List<WALActionsListener> listeners = new ArrayList<WALActionsListener>();
1773 listeners.add(new MetricsWAL());
1774
1775
1776 if (this.replicationSourceHandler != null &&
1777 this.replicationSourceHandler.getWALActionsListener() != null) {
1778
1779 listeners.add(this.replicationSourceHandler.getWALActionsListener());
1780 }
1781
1782 return new WALFactory(conf, listeners, serverName.toString());
1783 }
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793 protected LogRoller ensureMetaWALRoller() {
1794
1795
1796 LogRoller roller = metawalRoller.get();
1797 if (null == roller) {
1798 LogRoller tmpLogRoller = new LogRoller(this, this);
1799 String n = Thread.currentThread().getName();
1800 Threads.setDaemonThreadRunning(tmpLogRoller.getThread(),
1801 n + "-MetaLogRoller", uncaughtExceptionHandler);
1802 if (metawalRoller.compareAndSet(null, tmpLogRoller)) {
1803 roller = tmpLogRoller;
1804 } else {
1805
1806 Threads.shutdown(tmpLogRoller.getThread());
1807 roller = metawalRoller.get();
1808 }
1809 }
1810 return roller;
1811 }
1812
1813 public MetricsRegionServer getRegionServerMetrics() {
1814 return this.metricsRegionServer;
1815 }
1816
1817
1818
1819
1820 public MasterAddressTracker getMasterAddressTracker() {
1821 return this.masterAddressTracker;
1822 }
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836 private void startServiceThreads() throws IOException {
1837
1838 this.service.startExecutorService(ExecutorType.RS_OPEN_REGION,
1839 conf.getInt("hbase.regionserver.executor.openregion.threads", 3));
1840 this.service.startExecutorService(ExecutorType.RS_OPEN_META,
1841 conf.getInt("hbase.regionserver.executor.openmeta.threads", 1));
1842 this.service.startExecutorService(ExecutorType.RS_OPEN_PRIORITY_REGION,
1843 conf.getInt("hbase.regionserver.executor.openpriorityregion.threads", 3));
1844 this.service.startExecutorService(ExecutorType.RS_CLOSE_REGION,
1845 conf.getInt("hbase.regionserver.executor.closeregion.threads", 3));
1846 this.service.startExecutorService(ExecutorType.RS_CLOSE_META,
1847 conf.getInt("hbase.regionserver.executor.closemeta.threads", 1));
1848 if (conf.getBoolean(StoreScanner.STORESCANNER_PARALLEL_SEEK_ENABLE, false)) {
1849 this.service.startExecutorService(ExecutorType.RS_PARALLEL_SEEK,
1850 conf.getInt("hbase.storescanner.parallel.seek.threads", 10));
1851 }
1852 this.service.startExecutorService(ExecutorType.RS_LOG_REPLAY_OPS, conf.getInt(
1853 "hbase.regionserver.wal.max.splitters", SplitLogWorkerCoordination.DEFAULT_MAX_SPLITTERS));
1854
1855 if (ServerRegionReplicaUtil.isRegionReplicaWaitForPrimaryFlushEnabled(conf)) {
1856 this.service.startExecutorService(ExecutorType.RS_REGION_REPLICA_FLUSH_OPS,
1857 conf.getInt("hbase.regionserver.region.replica.flusher.threads",
1858 conf.getInt("hbase.regionserver.executor.openregion.threads", 3)));
1859 }
1860
1861 Threads.setDaemonThreadRunning(this.walRoller.getThread(), getName() + ".logRoller",
1862 uncaughtExceptionHandler);
1863 this.cacheFlusher.start(uncaughtExceptionHandler);
1864
1865 if (this.compactionChecker != null) choreService.scheduleChore(compactionChecker);
1866 if (this.periodicFlusher != null) choreService.scheduleChore(periodicFlusher);
1867 if (this.healthCheckChore != null) choreService.scheduleChore(healthCheckChore);
1868 if (this.nonceManagerChore != null) choreService.scheduleChore(nonceManagerChore);
1869 if (this.storefileRefresher != null) choreService.scheduleChore(storefileRefresher);
1870 if (this.movedRegionsCleaner != null) choreService.scheduleChore(movedRegionsCleaner);
1871 if (this.fsUtilizationChore != null) choreService.scheduleChore(fsUtilizationChore);
1872
1873
1874
1875 Threads.setDaemonThreadRunning(this.leases.getThread(), getName() + ".leaseChecker",
1876 uncaughtExceptionHandler);
1877
1878 if (this.replicationSourceHandler == this.replicationSinkHandler &&
1879 this.replicationSourceHandler != null) {
1880 this.replicationSourceHandler.startReplicationService();
1881 } else {
1882 if (this.replicationSourceHandler != null) {
1883 this.replicationSourceHandler.startReplicationService();
1884 }
1885 if (this.replicationSinkHandler != null) {
1886 this.replicationSinkHandler.startReplicationService();
1887 }
1888 }
1889
1890
1891
1892
1893
1894 Configuration sinkConf = HBaseConfiguration.create(conf);
1895 sinkConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
1896 conf.getInt("hbase.log.replay.retries.number", 8));
1897 sinkConf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY,
1898 conf.getInt("hbase.log.replay.rpc.timeout", 30000));
1899 sinkConf.setInt("hbase.client.serverside.retries.multiplier", 1);
1900 this.splitLogWorker = new SplitLogWorker(this, sinkConf, this, this, walFactory);
1901 splitLogWorker.start();
1902 }
1903
1904
1905
1906
1907
1908
1909 private int putUpWebUI() throws IOException {
1910 int port = this.conf.getInt(HConstants.REGIONSERVER_INFO_PORT,
1911 HConstants.DEFAULT_REGIONSERVER_INFOPORT);
1912 String addr = this.conf.get("hbase.regionserver.info.bindAddress", "0.0.0.0");
1913
1914 if(this instanceof HMaster) {
1915 port = conf.getInt(HConstants.MASTER_INFO_PORT,
1916 HConstants.DEFAULT_MASTER_INFOPORT);
1917 addr = this.conf.get("hbase.master.info.bindAddress", "0.0.0.0");
1918 }
1919
1920 if (port < 0) return port;
1921
1922 if (!Addressing.isLocalAddress(InetAddress.getByName(addr))) {
1923 String msg =
1924 "Failed to start http info server. Address " + addr
1925 + " does not belong to this host. Correct configuration parameter: "
1926 + "hbase.regionserver.info.bindAddress";
1927 LOG.error(msg);
1928 throw new IOException(msg);
1929 }
1930
1931 boolean auto = this.conf.getBoolean(HConstants.REGIONSERVER_INFO_PORT_AUTO,
1932 false);
1933 while (true) {
1934 try {
1935 this.infoServer = new InfoServer(getProcessName(), addr, port, false, this.conf);
1936 infoServer.addServlet("dump", "/dump", getDumpServlet());
1937 configureInfoServer();
1938 this.infoServer.start();
1939 break;
1940 } catch (BindException e) {
1941 if (!auto) {
1942
1943 LOG.error("Failed binding http info server to port: " + port);
1944 throw e;
1945 }
1946
1947 LOG.info("Failed binding http info server to port: " + port);
1948 port++;
1949 }
1950 }
1951 port = this.infoServer.getPort();
1952 conf.setInt(HConstants.REGIONSERVER_INFO_PORT, port);
1953 int masterInfoPort = conf.getInt(HConstants.MASTER_INFO_PORT,
1954 HConstants.DEFAULT_MASTER_INFOPORT);
1955 conf.setInt("hbase.master.info.port.orig", masterInfoPort);
1956 conf.setInt(HConstants.MASTER_INFO_PORT, port);
1957 return port;
1958 }
1959
1960
1961
1962
1963 private boolean isHealthy() {
1964 if (!fsOk) {
1965
1966 return false;
1967 }
1968
1969 if (!(leases.isAlive()
1970 && cacheFlusher.isAlive() && walRoller.isAlive()
1971 && this.compactionChecker.isScheduled()
1972 && this.periodicFlusher.isScheduled())) {
1973 stop("One or more threads are no longer alive -- stop");
1974 return false;
1975 }
1976 final LogRoller metawalRoller = this.metawalRoller.get();
1977 if (metawalRoller != null && !metawalRoller.isAlive()) {
1978 stop("Meta WAL roller thread is no longer alive -- stop");
1979 return false;
1980 }
1981 return true;
1982 }
1983
1984 private static final byte[] UNSPECIFIED_REGION = new byte[]{};
1985
1986 @Override
1987 public WAL getWAL(HRegionInfo regionInfo) throws IOException {
1988 WAL wal;
1989 LogRoller roller = walRoller;
1990
1991 if (regionInfo != null && regionInfo.isMetaTable() &&
1992 regionInfo.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1993 roller = ensureMetaWALRoller();
1994 wal = walFactory.getMetaWAL(regionInfo.getEncodedNameAsBytes());
1995 } else if (regionInfo == null) {
1996 wal = walFactory.getWAL(UNSPECIFIED_REGION);
1997 } else {
1998 wal = walFactory.getWAL(regionInfo.getEncodedNameAsBytes());
1999 }
2000 roller.addWAL(wal);
2001 return wal;
2002 }
2003
2004 @Override
2005 public ClusterConnection getConnection() {
2006 return this.clusterConnection;
2007 }
2008
2009 @Override
2010 public MetaTableLocator getMetaTableLocator() {
2011 return this.metaTableLocator;
2012 }
2013
2014 @Override
2015 public void stop(final String msg) {
2016 if (!this.stopped) {
2017 try {
2018 if (this.rsHost != null) {
2019 this.rsHost.preStop(msg);
2020 }
2021 this.stopped = true;
2022 LOG.info("STOPPED: " + msg);
2023
2024 sleeper.skipSleepCycle();
2025 } catch (IOException exp) {
2026 LOG.warn("The region server did not stop", exp);
2027 }
2028 }
2029 }
2030
2031 public void waitForServerOnline(){
2032 while (!isStopped() && !isOnline()) {
2033 synchronized (online) {
2034 try {
2035 online.wait(msgInterval);
2036 } catch (InterruptedException ie) {
2037 Thread.currentThread().interrupt();
2038 break;
2039 }
2040 }
2041 }
2042 }
2043
2044 @Override
2045 public void postOpenDeployTasks(final Region r) throws KeeperException, IOException {
2046 postOpenDeployTasks(new PostOpenDeployContext(r, -1));
2047 }
2048
2049 @Override
2050 public void postOpenDeployTasks(final PostOpenDeployContext context)
2051 throws KeeperException, IOException {
2052 Region r = context.getRegion();
2053 long masterSystemTime = context.getMasterSystemTime();
2054 Preconditions.checkArgument(r instanceof HRegion, "r must be an HRegion");
2055 rpcServices.checkOpen();
2056 LOG.info("Post open deploy tasks for " + r.getRegionInfo().getRegionNameAsString());
2057
2058 for (Store s : r.getStores()) {
2059 if (s.hasReferences() || s.needsCompaction()) {
2060 this.compactSplitThread.requestSystemCompaction(r, s, "Opening Region");
2061 }
2062 }
2063 long openSeqNum = r.getOpenSeqNum();
2064 if (openSeqNum == HConstants.NO_SEQNUM) {
2065
2066 LOG.error("No sequence number found when opening " +
2067 r.getRegionInfo().getRegionNameAsString());
2068 openSeqNum = 0;
2069 }
2070
2071
2072 updateRecoveringRegionLastFlushedSequenceId(r);
2073
2074
2075 if (r.getRegionInfo().isMetaRegion()) {
2076 MetaTableLocator.setMetaLocation(getZooKeeper(), serverName, r.getRegionInfo().getReplicaId(),
2077 State.OPEN);
2078 } else if (useZKForAssignment) {
2079 MetaTableAccessor.updateRegionLocation(getConnection(), r.getRegionInfo(),
2080 this.serverName, openSeqNum, masterSystemTime);
2081 }
2082 if (!useZKForAssignment && !reportRegionStateTransition(new RegionStateTransitionContext(
2083 TransitionCode.OPENED, openSeqNum, masterSystemTime, r.getRegionInfo()))) {
2084 throw new IOException("Failed to report opened region to master: "
2085 + r.getRegionInfo().getRegionNameAsString());
2086 }
2087
2088 triggerFlushInPrimaryRegion((HRegion)r);
2089
2090 LOG.debug("Finished post open deploy task for " + r.getRegionInfo().getRegionNameAsString());
2091 }
2092
2093 @Override
2094 public boolean reportRegionStateTransition(TransitionCode code, HRegionInfo... hris) {
2095 return reportRegionStateTransition(code, HConstants.NO_SEQNUM, hris);
2096 }
2097
2098 @Override
2099 public boolean reportRegionStateTransition(
2100 TransitionCode code, long openSeqNum, HRegionInfo... hris) {
2101 return reportRegionStateTransition(
2102 new RegionStateTransitionContext(code, HConstants.NO_SEQNUM, -1, hris));
2103 }
2104
2105 @Override
2106 public boolean reportRegionStateTransition(final RegionStateTransitionContext context) {
2107 TransitionCode code = context.getCode();
2108 long openSeqNum = context.getOpenSeqNum();
2109 long masterSystemTime = context.getMasterSystemTime();
2110 HRegionInfo[] hris = context.getHris();
2111
2112 ReportRegionStateTransitionRequest.Builder builder =
2113 ReportRegionStateTransitionRequest.newBuilder();
2114 builder.setServer(ProtobufUtil.toServerName(serverName));
2115 RegionStateTransition.Builder transition = builder.addTransitionBuilder();
2116 transition.setTransitionCode(code);
2117 if (code == TransitionCode.OPENED && openSeqNum >= 0) {
2118 transition.setOpenSeqNum(openSeqNum);
2119 }
2120 for (HRegionInfo hri: hris) {
2121 transition.addRegionInfo(HRegionInfo.convert(hri));
2122 }
2123 ReportRegionStateTransitionRequest request = builder.build();
2124 while (keepLooping()) {
2125 RegionServerStatusService.BlockingInterface rss = rssStub;
2126 try {
2127 if (rss == null) {
2128 createRegionServerStatusStub();
2129 continue;
2130 }
2131 ReportRegionStateTransitionResponse response =
2132 rss.reportRegionStateTransition(null, request);
2133 if (response.hasErrorMessage()) {
2134 LOG.info("Failed to transition " + hris[0]
2135 + " to " + code + ": " + response.getErrorMessage());
2136 return false;
2137 }
2138 return true;
2139 } catch (ServiceException se) {
2140 IOException ioe = ProtobufUtil.getRemoteException(se);
2141 LOG.info("Failed to report region transition, will retry", ioe);
2142 if (rssStub == rss) {
2143 rssStub = null;
2144 }
2145 }
2146 }
2147 return false;
2148 }
2149
2150
2151
2152
2153
2154 void triggerFlushInPrimaryRegion(final HRegion region) {
2155 if (ServerRegionReplicaUtil.isDefaultReplica(region.getRegionInfo())) {
2156 return;
2157 }
2158 if (!ServerRegionReplicaUtil.isRegionReplicaReplicationEnabled(region.conf) ||
2159 !ServerRegionReplicaUtil.isRegionReplicaWaitForPrimaryFlushEnabled(
2160 region.conf)) {
2161 region.setReadsEnabled(true);
2162 return;
2163 }
2164
2165 region.setReadsEnabled(false);
2166
2167
2168
2169 this.service.submit(
2170 new RegionReplicaFlushHandler(this, clusterConnection,
2171 rpcRetryingCallerFactory, rpcControllerFactory, operationTimeout, region));
2172 }
2173
2174 @Override
2175 public RpcServerInterface getRpcServer() {
2176 return rpcServices.rpcServer;
2177 }
2178
2179 @VisibleForTesting
2180 public RSRpcServices getRSRpcServices() {
2181 return rpcServices;
2182 }
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194 @Override
2195 public void abort(String reason, Throwable cause) {
2196 String msg = "ABORTING region server " + this + ": " + reason;
2197 if (cause != null) {
2198 LOG.fatal(msg, cause);
2199 } else {
2200 LOG.fatal(msg);
2201 }
2202 this.abortRequested = true;
2203
2204
2205
2206 LOG.fatal("RegionServer abort: loaded coprocessors are: " +
2207 CoprocessorHost.getLoadedCoprocessors());
2208
2209 try {
2210 LOG.info("Dump of metrics as JSON on abort: " + JSONBean.dumpRegionServerMetrics());
2211 } catch (MalformedObjectNameException | IOException e) {
2212 LOG.warn("Failed dumping metrics", e);
2213 }
2214
2215
2216 try {
2217 if (cause != null) {
2218 msg += "\nCause:\n" + StringUtils.stringifyException(cause);
2219 }
2220
2221 if (rssStub != null && this.serverName != null) {
2222 ReportRSFatalErrorRequest.Builder builder =
2223 ReportRSFatalErrorRequest.newBuilder();
2224 ServerName sn =
2225 ServerName.parseVersionedServerName(this.serverName.getVersionedBytes());
2226 builder.setServer(ProtobufUtil.toServerName(sn));
2227 builder.setErrorMessage(msg);
2228 rssStub.reportRSFatalError(null, builder.build());
2229 }
2230 } catch (Throwable t) {
2231 LOG.warn("Unable to report fatal error to master", t);
2232 }
2233 stop(reason);
2234 }
2235
2236
2237
2238
2239 public void abort(String reason) {
2240 abort(reason, null);
2241 }
2242
2243 @Override
2244 public boolean isAborted() {
2245 return this.abortRequested;
2246 }
2247
2248
2249
2250
2251
2252
2253 protected void kill() {
2254 this.killed = true;
2255 abort("Simulated kill");
2256 }
2257
2258
2259
2260
2261 protected void sendShutdownInterrupt() {
2262 }
2263
2264
2265
2266
2267
2268 protected void stopServiceThreads() {
2269
2270
2271 if (this.choreService != null) choreService.shutdown();
2272 if (this.nonceManagerChore != null) nonceManagerChore.cancel(true);
2273 if (this.compactionChecker != null) compactionChecker.cancel(true);
2274 if (this.periodicFlusher != null) periodicFlusher.cancel(true);
2275 if (this.healthCheckChore != null) healthCheckChore.cancel(true);
2276 if (this.storefileRefresher != null) storefileRefresher.cancel(true);
2277 if (this.movedRegionsCleaner != null) movedRegionsCleaner.cancel(true);
2278 if (this.fsUtilizationChore != null) fsUtilizationChore.cancel(true);
2279
2280 if (this.cacheFlusher != null) {
2281 this.cacheFlusher.join();
2282 }
2283
2284 if (this.spanReceiverHost != null) {
2285 this.spanReceiverHost.closeReceivers();
2286 }
2287 if (this.walRoller != null) {
2288 Threads.shutdown(this.walRoller.getThread());
2289 }
2290 final LogRoller metawalRoller = this.metawalRoller.get();
2291 if (metawalRoller != null) {
2292 Threads.shutdown(metawalRoller.getThread());
2293 }
2294 if (this.compactSplitThread != null) {
2295 this.compactSplitThread.join();
2296 }
2297 if (this.service != null) this.service.shutdown();
2298 if (this.replicationSourceHandler != null &&
2299 this.replicationSourceHandler == this.replicationSinkHandler) {
2300 this.replicationSourceHandler.stopReplicationService();
2301 } else {
2302 if (this.replicationSourceHandler != null) {
2303 this.replicationSourceHandler.stopReplicationService();
2304 }
2305 if (this.replicationSinkHandler != null) {
2306 this.replicationSinkHandler.stopReplicationService();
2307 }
2308 }
2309 }
2310
2311
2312
2313
2314
2315 public ReplicationSourceService getReplicationSourceService() {
2316 return replicationSourceHandler;
2317 }
2318
2319
2320
2321
2322
2323 ReplicationSinkService getReplicationSinkService() {
2324 return replicationSinkHandler;
2325 }
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335 @VisibleForTesting
2336 protected synchronized ServerName createRegionServerStatusStub() {
2337 if (rssStub != null) {
2338 return masterAddressTracker.getMasterAddress();
2339 }
2340 ServerName sn = null;
2341 long previousLogTime = 0;
2342 boolean refresh = false;
2343 RegionServerStatusService.BlockingInterface intf = null;
2344 boolean interrupted = false;
2345 try {
2346 while (keepLooping()) {
2347 sn = this.masterAddressTracker.getMasterAddress(refresh);
2348 if (sn == null) {
2349 if (!keepLooping()) {
2350
2351 LOG.debug("No master found and cluster is stopped; bailing out");
2352 return null;
2353 }
2354 if (System.currentTimeMillis() > (previousLogTime + 1000)) {
2355 LOG.debug("No master found; retry");
2356 previousLogTime = System.currentTimeMillis();
2357 }
2358 refresh = true;
2359 if (sleep(200)) {
2360 interrupted = true;
2361 }
2362 continue;
2363 }
2364
2365
2366 if (this instanceof HMaster && sn.equals(getServerName())) {
2367 intf = ((HMaster)this).getMasterRpcServices();
2368 break;
2369 }
2370 try {
2371 BlockingRpcChannel channel =
2372 this.rpcClient.createBlockingRpcChannel(sn, userProvider.getCurrent(),
2373 shortOperationTimeout);
2374 intf = RegionServerStatusService.newBlockingStub(channel);
2375 break;
2376 } catch (IOException e) {
2377 if (System.currentTimeMillis() > (previousLogTime + 1000)) {
2378 e = e instanceof RemoteException ?
2379 ((RemoteException)e).unwrapRemoteException() : e;
2380 if (e instanceof ServerNotRunningYetException) {
2381 LOG.info("Master isn't available yet, retrying");
2382 } else {
2383 LOG.warn("Unable to connect to master. Retrying. Error was:", e);
2384 }
2385 previousLogTime = System.currentTimeMillis();
2386 }
2387 if (sleep(200)) {
2388 interrupted = true;
2389 }
2390 }
2391 }
2392 } finally {
2393 if (interrupted) {
2394 Thread.currentThread().interrupt();
2395 }
2396 }
2397 rssStub = intf;
2398 return sn;
2399 }
2400
2401
2402
2403
2404
2405 private boolean keepLooping() {
2406 return !this.stopped && isClusterUp();
2407 }
2408
2409
2410
2411
2412
2413
2414
2415
2416 private RegionServerStartupResponse reportForDuty() throws IOException {
2417 ServerName masterServerName = createRegionServerStatusStub();
2418 if (masterServerName == null) return null;
2419 RegionServerStartupResponse result = null;
2420 try {
2421 rpcServices.requestCount.set(0);
2422 rpcServices.rpcGetRequestCount.set(0);
2423 rpcServices.rpcScanRequestCount.set(0);
2424 rpcServices.rpcMultiRequestCount.set(0);
2425 rpcServices.rpcMutateRequestCount.set(0);
2426 LOG.info("reportForDuty to master=" + masterServerName + " with port="
2427 + rpcServices.isa.getPort() + ", startcode=" + this.startcode);
2428 long now = EnvironmentEdgeManager.currentTime();
2429 int port = rpcServices.isa.getPort();
2430 RegionServerStartupRequest.Builder request = RegionServerStartupRequest.newBuilder();
2431 if (shouldUseThisHostnameInstead()) {
2432 request.setUseThisHostnameInstead(useThisHostnameInstead);
2433 }
2434 request.setPort(port);
2435 request.setServerStartCode(this.startcode);
2436 request.setServerCurrentTime(now);
2437 result = this.rssStub.regionServerStartup(null, request.build());
2438 } catch (ServiceException se) {
2439 IOException ioe = ProtobufUtil.getRemoteException(se);
2440 if (ioe instanceof ClockOutOfSyncException) {
2441 LOG.fatal("Master rejected startup because clock is out of sync", ioe);
2442
2443 throw ioe;
2444 } else if (ioe instanceof ServerNotRunningYetException) {
2445 LOG.debug("Master is not running yet");
2446 } else {
2447 LOG.warn("error telling master we are up", se);
2448 }
2449 rssStub = null;
2450 }
2451 return result;
2452 }
2453
2454 @Override
2455 public RegionStoreSequenceIds getLastSequenceId(byte[] encodedRegionName) {
2456 try {
2457 GetLastFlushedSequenceIdRequest req =
2458 RequestConverter.buildGetLastFlushedSequenceIdRequest(encodedRegionName);
2459 RegionServerStatusService.BlockingInterface rss = rssStub;
2460 if (rss == null) {
2461 createRegionServerStatusStub();
2462 rss = rssStub;
2463 if (rss == null) {
2464
2465 LOG.warn("Unable to connect to the master to check " + "the last flushed sequence id");
2466 return RegionStoreSequenceIds.newBuilder().setLastFlushedSequenceId(HConstants.NO_SEQNUM)
2467 .build();
2468 }
2469 }
2470 GetLastFlushedSequenceIdResponse resp = rss.getLastFlushedSequenceId(null, req);
2471 return RegionStoreSequenceIds.newBuilder()
2472 .setLastFlushedSequenceId(resp.getLastFlushedSequenceId())
2473 .addAllStoreSequenceId(resp.getStoreLastFlushedSequenceIdList()).build();
2474 } catch (ServiceException e) {
2475 LOG.warn("Unable to connect to the master to check the last flushed sequence id", e);
2476 return RegionStoreSequenceIds.newBuilder().setLastFlushedSequenceId(HConstants.NO_SEQNUM)
2477 .build();
2478 }
2479 }
2480
2481
2482
2483
2484
2485
2486 protected void closeAllRegions(final boolean abort) {
2487 closeUserRegions(abort);
2488 closeMetaTableRegions(abort);
2489 }
2490
2491
2492
2493
2494
2495 void closeMetaTableRegions(final boolean abort) {
2496 Region meta = null;
2497 this.lock.writeLock().lock();
2498 try {
2499 for (Map.Entry<String, Region> e: onlineRegions.entrySet()) {
2500 HRegionInfo hri = e.getValue().getRegionInfo();
2501 if (hri.isMetaRegion()) {
2502 meta = e.getValue();
2503 }
2504 if (meta != null) break;
2505 }
2506 } finally {
2507 this.lock.writeLock().unlock();
2508 }
2509 if (meta != null) closeRegionIgnoreErrors(meta.getRegionInfo(), abort);
2510 }
2511
2512
2513
2514
2515
2516
2517
2518 void closeUserRegions(final boolean abort) {
2519 this.lock.writeLock().lock();
2520 try {
2521 for (Map.Entry<String, Region> e: this.onlineRegions.entrySet()) {
2522 Region r = e.getValue();
2523 if (!r.getRegionInfo().isMetaTable() && r.isAvailable()) {
2524
2525 closeRegionIgnoreErrors(r.getRegionInfo(), abort);
2526 }
2527 }
2528 } finally {
2529 this.lock.writeLock().unlock();
2530 }
2531 }
2532
2533
2534 public InfoServer getInfoServer() {
2535 return infoServer;
2536 }
2537
2538
2539
2540
2541 @Override
2542 public boolean isStopped() {
2543 return this.stopped;
2544 }
2545
2546 @Override
2547 public boolean isStopping() {
2548 return this.stopping;
2549 }
2550
2551 @Override
2552 public Map<String, Region> getRecoveringRegions() {
2553 return this.recoveringRegions;
2554 }
2555
2556
2557
2558
2559
2560 @Override
2561 public Configuration getConfiguration() {
2562 return conf;
2563 }
2564
2565
2566 ReentrantReadWriteLock.WriteLock getWriteLock() {
2567 return lock.writeLock();
2568 }
2569
2570 public int getNumberOfOnlineRegions() {
2571 return this.onlineRegions.size();
2572 }
2573
2574 boolean isOnlineRegionsEmpty() {
2575 return this.onlineRegions.isEmpty();
2576 }
2577
2578
2579
2580
2581
2582
2583 public Collection<Region> getOnlineRegionsLocalContext() {
2584 Collection<Region> regions = this.onlineRegions.values();
2585 return Collections.unmodifiableCollection(regions);
2586 }
2587
2588 @Override
2589 public void addToOnlineRegions(Region region) {
2590 this.onlineRegions.put(region.getRegionInfo().getEncodedName(), region);
2591 configurationManager.registerObserver(region);
2592 }
2593
2594
2595
2596
2597
2598
2599 SortedMap<Long, Region> getCopyOfOnlineRegionsSortedBySize() {
2600
2601 SortedMap<Long, Region> sortedRegions = new TreeMap<Long, Region>(
2602 new Comparator<Long>() {
2603 @Override
2604 public int compare(Long a, Long b) {
2605 return -1 * a.compareTo(b);
2606 }
2607 });
2608
2609 for (Region region : this.onlineRegions.values()) {
2610 sortedRegions.put(region.getMemstoreSize(), region);
2611 }
2612 return sortedRegions;
2613 }
2614
2615
2616
2617
2618 public long getStartcode() {
2619 return this.startcode;
2620 }
2621
2622
2623 @Override
2624 public FlushRequester getFlushRequester() {
2625 return this.cacheFlusher;
2626 }
2627
2628
2629
2630
2631
2632
2633
2634 protected HRegionInfo[] getMostLoadedRegions() {
2635 ArrayList<HRegionInfo> regions = new ArrayList<HRegionInfo>();
2636 for (Region r : onlineRegions.values()) {
2637 if (!r.isAvailable()) {
2638 continue;
2639 }
2640 if (regions.size() < numRegionsToReport) {
2641 regions.add(r.getRegionInfo());
2642 } else {
2643 break;
2644 }
2645 }
2646 return regions.toArray(new HRegionInfo[regions.size()]);
2647 }
2648
2649 @Override
2650 public Leases getLeases() {
2651 return leases;
2652 }
2653
2654
2655
2656
2657 protected Path getRootDir() {
2658 return rootDir;
2659 }
2660
2661
2662
2663
2664 @Override
2665 public FileSystem getFileSystem() {
2666 return fs;
2667 }
2668
2669
2670
2671
2672 protected Path getWALRootDir() {
2673 return walRootDir;
2674 }
2675
2676
2677
2678
2679 protected FileSystem getWALFileSystem() {
2680 return walFs;
2681 }
2682
2683 @Override
2684 public String toString() {
2685 return getServerName().toString();
2686 }
2687
2688
2689
2690
2691
2692
2693 public int getThreadWakeFrequency() {
2694 return threadWakeFrequency;
2695 }
2696
2697 @Override
2698 public ZooKeeperWatcher getZooKeeper() {
2699 return zooKeeper;
2700 }
2701
2702 @Override
2703 public BaseCoordinatedStateManager getCoordinatedStateManager() {
2704 return csm;
2705 }
2706
2707 @Override
2708 public ServerName getServerName() {
2709 return serverName;
2710 }
2711
2712 @Override
2713 public CompactionRequestor getCompactionRequester() {
2714 return this.compactSplitThread;
2715 }
2716
2717 public RegionServerCoprocessorHost getRegionServerCoprocessorHost(){
2718 return this.rsHost;
2719 }
2720
2721 @Override
2722 public ConcurrentMap<byte[], Boolean> getRegionsInTransitionInRS() {
2723 return this.regionsInTransitionInRS;
2724 }
2725
2726 @Override
2727 public ExecutorService getExecutorService() {
2728 return service;
2729 }
2730
2731 @Override
2732 public ChoreService getChoreService() {
2733 return choreService;
2734 }
2735
2736 @Override
2737 public RegionServerRpcQuotaManager getRegionServerRpcQuotaManager() {
2738 return rsQuotaManager;
2739 }
2740
2741
2742
2743
2744
2745
2746
2747
2748 static private void createNewReplicationInstance(Configuration conf,
2749 HRegionServer server, FileSystem walFs, Path walDir, Path oldWALDir) throws IOException{
2750
2751
2752 if (!conf.getBoolean(HConstants.REPLICATION_ENABLE_KEY,
2753 HConstants.REPLICATION_ENABLE_DEFAULT)) {
2754 return;
2755 }
2756
2757
2758 String sourceClassname = conf.get(HConstants.REPLICATION_SOURCE_SERVICE_CLASSNAME,
2759 HConstants.REPLICATION_SERVICE_CLASSNAME_DEFAULT);
2760
2761
2762 String sinkClassname = conf.get(HConstants.REPLICATION_SINK_SERVICE_CLASSNAME,
2763 HConstants.REPLICATION_SERVICE_CLASSNAME_DEFAULT);
2764
2765
2766
2767 if (sourceClassname.equals(sinkClassname)) {
2768 server.replicationSourceHandler = (ReplicationSourceService)
2769 newReplicationInstance(sourceClassname,
2770 conf, server, walFs, walDir, oldWALDir);
2771 server.replicationSinkHandler = (ReplicationSinkService)
2772 server.replicationSourceHandler;
2773 } else {
2774 server.replicationSourceHandler = (ReplicationSourceService)
2775 newReplicationInstance(sourceClassname,
2776 conf, server, walFs, walDir, oldWALDir);
2777 server.replicationSinkHandler = (ReplicationSinkService)
2778 newReplicationInstance(sinkClassname,
2779 conf, server, walFs, walDir, oldWALDir);
2780 }
2781 }
2782
2783 static private ReplicationService newReplicationInstance(String classname,
2784 Configuration conf, HRegionServer server, FileSystem walFs, Path walDir,
2785 Path oldLogDir) throws IOException{
2786
2787 Class<?> clazz = null;
2788 try {
2789 ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
2790 clazz = Class.forName(classname, true, classLoader);
2791 } catch (java.lang.ClassNotFoundException nfe) {
2792 throw new IOException("Could not find class for " + classname);
2793 }
2794
2795
2796 ReplicationService service = (ReplicationService)
2797 ReflectionUtils.newInstance(clazz, conf);
2798 service.initialize(server, walFs, walDir, oldLogDir);
2799 return service;
2800 }
2801
2802
2803
2804
2805
2806
2807
2808
2809 public static HRegionServer constructRegionServer(
2810 Class<? extends HRegionServer> regionServerClass,
2811 final Configuration conf2, CoordinatedStateManager cp) {
2812 try {
2813 Constructor<? extends HRegionServer> c = regionServerClass
2814 .getConstructor(Configuration.class, CoordinatedStateManager.class);
2815 return c.newInstance(conf2, cp);
2816 } catch (Exception e) {
2817 throw new RuntimeException("Failed construction of " + "Regionserver: "
2818 + regionServerClass.toString(), e);
2819 }
2820 }
2821
2822
2823
2824
2825 public static void main(String[] args) throws Exception {
2826 VersionInfo.logVersion();
2827 Configuration conf = HBaseConfiguration.create();
2828 @SuppressWarnings("unchecked")
2829 Class<? extends HRegionServer> regionServerClass = (Class<? extends HRegionServer>) conf
2830 .getClass(HConstants.REGION_SERVER_IMPL, HRegionServer.class);
2831
2832 new HRegionServerCommandLine(regionServerClass).doMain(args);
2833 }
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845 @Override
2846 public List<Region> getOnlineRegions(TableName tableName) {
2847 List<Region> tableRegions = new ArrayList<Region>();
2848 synchronized (this.onlineRegions) {
2849 for (Region region: this.onlineRegions.values()) {
2850 HRegionInfo regionInfo = region.getRegionInfo();
2851 if(regionInfo.getTable().equals(tableName)) {
2852 tableRegions.add(region);
2853 }
2854 }
2855 }
2856 return tableRegions;
2857 }
2858
2859
2860
2861
2862
2863
2864 public List<Region> getOnlineRegions() {
2865 List<Region> allRegions = new ArrayList<Region>();
2866 synchronized (this.onlineRegions) {
2867
2868 allRegions.addAll(onlineRegions.values());
2869 }
2870 return allRegions;
2871 }
2872
2873
2874
2875
2876
2877
2878 @Override
2879 public Set<TableName> getOnlineTables() {
2880 Set<TableName> tables = new HashSet<TableName>();
2881 synchronized (this.onlineRegions) {
2882 for (Region region: this.onlineRegions.values()) {
2883 tables.add(region.getTableDesc().getTableName());
2884 }
2885 }
2886 return tables;
2887 }
2888
2889
2890 public String[] getRegionServerCoprocessors() {
2891 TreeSet<String> coprocessors = new TreeSet<String>();
2892 try {
2893 coprocessors.addAll(getWAL(null).getCoprocessorHost().getCoprocessors());
2894 } catch (IOException exception) {
2895 LOG.warn("Exception attempting to fetch wal coprocessor information for the common wal; " +
2896 "skipping.");
2897 LOG.debug("Exception details for failure to fetch wal coprocessor information.", exception);
2898 }
2899 Collection<Region> regions = getOnlineRegionsLocalContext();
2900 for (Region region: regions) {
2901 coprocessors.addAll(region.getCoprocessorHost().getCoprocessors());
2902 try {
2903 coprocessors.addAll(getWAL(region.getRegionInfo()).getCoprocessorHost().getCoprocessors());
2904 } catch (IOException exception) {
2905 LOG.warn("Exception attempting to fetch wal coprocessor information for region " + region +
2906 "; skipping.");
2907 LOG.debug("Exception details for failure to fetch wal coprocessor information.", exception);
2908 }
2909 }
2910 return coprocessors.toArray(new String[coprocessors.size()]);
2911 }
2912
2913
2914
2915
2916
2917 private void closeRegionIgnoreErrors(HRegionInfo region, final boolean abort) {
2918 try {
2919 CloseRegionCoordination.CloseRegionDetails details =
2920 csm.getCloseRegionCoordination().getDetaultDetails();
2921 if (!closeRegion(region.getEncodedName(), abort, details, null)) {
2922 LOG.warn("Failed to close " + region.getRegionNameAsString() +
2923 " - ignoring and continuing");
2924 }
2925 } catch (IOException e) {
2926 LOG.warn("Failed to close " + region.getRegionNameAsString() +
2927 " - ignoring and continuing", e);
2928 }
2929 }
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951 protected boolean closeRegion(String encodedName, final boolean abort,
2952 CloseRegionCoordination.CloseRegionDetails crd, final ServerName sn)
2953 throws NotServingRegionException, RegionAlreadyInTransitionException {
2954
2955 Region actualRegion = this.getFromOnlineRegions(encodedName);
2956 if ((actualRegion != null) && (actualRegion.getCoprocessorHost() != null)) {
2957 try {
2958 actualRegion.getCoprocessorHost().preClose(false);
2959 } catch (IOException exp) {
2960 LOG.warn("Unable to close region: the coprocessor launched an error ", exp);
2961 return false;
2962 }
2963 }
2964
2965 final Boolean previous = this.regionsInTransitionInRS.putIfAbsent(encodedName.getBytes(),
2966 Boolean.FALSE);
2967
2968 if (Boolean.TRUE.equals(previous)) {
2969 LOG.info("Received CLOSE for the region:" + encodedName + " , which we are already " +
2970 "trying to OPEN. Cancelling OPENING.");
2971 if (!regionsInTransitionInRS.replace(encodedName.getBytes(), previous, Boolean.FALSE)){
2972
2973
2974 LOG.warn("The opening for region " + encodedName + " was done before we could cancel it." +
2975 " Doing a standard close now");
2976 return closeRegion(encodedName, abort, crd, sn);
2977 }
2978
2979 actualRegion = this.getFromOnlineRegions(encodedName);
2980 if (actualRegion == null) {
2981 LOG.info("The opening previously in progress has been cancelled by a CLOSE request.");
2982
2983 throw new RegionAlreadyInTransitionException("The region " + encodedName +
2984 " was opening but not yet served. Opening is cancelled.");
2985 }
2986 } else if (Boolean.FALSE.equals(previous)) {
2987 LOG.info("Received CLOSE for the region: " + encodedName +
2988 ", which we are already trying to CLOSE, but not completed yet");
2989
2990
2991
2992
2993
2994
2995 throw new RegionAlreadyInTransitionException("The region " + encodedName +
2996 " was already closing. New CLOSE request is ignored.");
2997 }
2998
2999 if (actualRegion == null) {
3000 LOG.error("Received CLOSE for a region which is not online, and we're not opening.");
3001 this.regionsInTransitionInRS.remove(encodedName.getBytes());
3002
3003 throw new NotServingRegionException("The region " + encodedName +
3004 " is not online, and is not opening.");
3005 }
3006
3007 CloseRegionHandler crh;
3008 final HRegionInfo hri = actualRegion.getRegionInfo();
3009 if (hri.isMetaRegion()) {
3010 crh = new CloseMetaHandler(this, this, hri, abort,
3011 csm.getCloseRegionCoordination(), crd);
3012 } else {
3013 crh = new CloseRegionHandler(this, this, hri, abort,
3014 csm.getCloseRegionCoordination(), crd, sn);
3015 }
3016 this.service.submit(crh);
3017 return true;
3018 }
3019
3020
3021
3022
3023
3024
3025 public Region getOnlineRegion(final byte[] regionName) {
3026 String encodedRegionName = HRegionInfo.encodeRegionName(regionName);
3027 return this.onlineRegions.get(encodedRegionName);
3028 }
3029
3030 public InetSocketAddress[] getRegionBlockLocations(final String encodedRegionName) {
3031 return this.regionFavoredNodesMap.get(encodedRegionName);
3032 }
3033
3034 @Override
3035 public Region getFromOnlineRegions(final String encodedRegionName) {
3036 return this.onlineRegions.get(encodedRegionName);
3037 }
3038
3039
3040 @Override
3041 public boolean removeFromOnlineRegions(final Region r, ServerName destination) {
3042 Region toReturn = this.onlineRegions.remove(r.getRegionInfo().getEncodedName());
3043
3044 if (destination != null) {
3045 long closeSeqNum = r.getMaxFlushedSeqId();
3046 if (closeSeqNum == HConstants.NO_SEQNUM) {
3047
3048 closeSeqNum = r.getOpenSeqNum();
3049 if (closeSeqNum == HConstants.NO_SEQNUM) {
3050 closeSeqNum = 0;
3051 }
3052 }
3053 addToMovedRegions(r.getRegionInfo().getEncodedName(), destination, closeSeqNum);
3054 }
3055 this.regionFavoredNodesMap.remove(r.getRegionInfo().getEncodedName());
3056 return toReturn != null;
3057 }
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067 protected Region getRegion(final byte[] regionName)
3068 throws NotServingRegionException {
3069 String encodedRegionName = HRegionInfo.encodeRegionName(regionName);
3070 return getRegionByEncodedName(regionName, encodedRegionName);
3071 }
3072
3073 public Region getRegionByEncodedName(String encodedRegionName)
3074 throws NotServingRegionException {
3075 return getRegionByEncodedName(null, encodedRegionName);
3076 }
3077
3078 protected Region getRegionByEncodedName(byte[] regionName, String encodedRegionName)
3079 throws NotServingRegionException {
3080 Region region = this.onlineRegions.get(encodedRegionName);
3081 if (region == null) {
3082 MovedRegionInfo moveInfo = getMovedRegion(encodedRegionName);
3083 if (moveInfo != null) {
3084 throw new RegionMovedException(moveInfo.getServerName(), moveInfo.getSeqNum());
3085 }
3086 Boolean isOpening = this.regionsInTransitionInRS.get(Bytes.toBytes(encodedRegionName));
3087 String regionNameStr = regionName == null?
3088 encodedRegionName: Bytes.toStringBinary(regionName);
3089 if (isOpening != null && isOpening.booleanValue()) {
3090 throw new RegionOpeningException("Region " + regionNameStr +
3091 " is opening on " + this.serverName);
3092 }
3093 throw new NotServingRegionException("Region " + regionNameStr +
3094 " is not online on " + this.serverName);
3095 }
3096 return region;
3097 }
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109 private Throwable cleanup(final Throwable t, final String msg) {
3110
3111 if (t instanceof NotServingRegionException) {
3112 LOG.debug("NotServingRegionException; " + t.getMessage());
3113 return t;
3114 }
3115 if (msg == null) {
3116 LOG.error("", RemoteExceptionHandler.checkThrowable(t));
3117 } else {
3118 LOG.error(msg, RemoteExceptionHandler.checkThrowable(t));
3119 }
3120 if (!rpcServices.checkOOME(t)) {
3121 checkFileSystem();
3122 }
3123 return t;
3124 }
3125
3126
3127
3128
3129
3130
3131
3132
3133 protected IOException convertThrowableToIOE(final Throwable t, final String msg) {
3134 return (t instanceof IOException ? (IOException) t : msg == null
3135 || msg.length() == 0 ? new IOException(t) : new IOException(msg, t));
3136 }
3137
3138
3139
3140
3141
3142
3143
3144 public boolean checkFileSystem() {
3145 if (this.fsOk && this.fs != null) {
3146 try {
3147 FSUtils.checkFileSystemAvailable(this.fs);
3148 } catch (IOException e) {
3149 abort("File System not available", e);
3150 this.fsOk = false;
3151 }
3152 }
3153 return this.fsOk;
3154 }
3155
3156 @Override
3157 public void updateRegionFavoredNodesMapping(String encodedRegionName,
3158 List<org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName> favoredNodes) {
3159 InetSocketAddress[] addr = new InetSocketAddress[favoredNodes.size()];
3160
3161
3162 for (int i = 0; i < favoredNodes.size(); i++) {
3163 addr[i] = InetSocketAddress.createUnresolved(favoredNodes.get(i).getHostName(),
3164 favoredNodes.get(i).getPort());
3165 }
3166 regionFavoredNodesMap.put(encodedRegionName, addr);
3167 }
3168
3169
3170
3171
3172
3173
3174
3175 @Override
3176 public InetSocketAddress[] getFavoredNodesForRegion(String encodedRegionName) {
3177 return regionFavoredNodesMap.get(encodedRegionName);
3178 }
3179
3180 @Override
3181 public ServerNonceManager getNonceManager() {
3182 return this.nonceManager;
3183 }
3184
3185 private static class MovedRegionInfo {
3186 private final ServerName serverName;
3187 private final long seqNum;
3188 private final long ts;
3189
3190 public MovedRegionInfo(ServerName serverName, long closeSeqNum) {
3191 this.serverName = serverName;
3192 this.seqNum = closeSeqNum;
3193 ts = EnvironmentEdgeManager.currentTime();
3194 }
3195
3196 public ServerName getServerName() {
3197 return serverName;
3198 }
3199
3200 public long getSeqNum() {
3201 return seqNum;
3202 }
3203
3204 public long getMoveTime() {
3205 return ts;
3206 }
3207 }
3208
3209
3210
3211 protected Map<String, MovedRegionInfo> movedRegions =
3212 new ConcurrentHashMap<String, MovedRegionInfo>(3000);
3213
3214
3215
3216 private static final int TIMEOUT_REGION_MOVED = (2 * 60 * 1000);
3217
3218 protected void addToMovedRegions(String encodedName, ServerName destination, long closeSeqNum) {
3219 if (ServerName.isSameHostnameAndPort(destination, this.getServerName())) {
3220 LOG.warn("Not adding moved region record: " + encodedName + " to self.");
3221 return;
3222 }
3223 LOG.info("Adding moved region record: "
3224 + encodedName + " to " + destination + " as of " + closeSeqNum);
3225 movedRegions.put(encodedName, new MovedRegionInfo(destination, closeSeqNum));
3226 }
3227
3228 void removeFromMovedRegions(String encodedName) {
3229 movedRegions.remove(encodedName);
3230 }
3231
3232 private MovedRegionInfo getMovedRegion(final String encodedRegionName) {
3233 MovedRegionInfo dest = movedRegions.get(encodedRegionName);
3234
3235 long now = EnvironmentEdgeManager.currentTime();
3236 if (dest != null) {
3237 if (dest.getMoveTime() > (now - TIMEOUT_REGION_MOVED)) {
3238 return dest;
3239 } else {
3240 movedRegions.remove(encodedRegionName);
3241 }
3242 }
3243
3244 return null;
3245 }
3246
3247
3248
3249
3250 protected void cleanMovedRegions() {
3251 final long cutOff = System.currentTimeMillis() - TIMEOUT_REGION_MOVED;
3252 Iterator<Entry<String, MovedRegionInfo>> it = movedRegions.entrySet().iterator();
3253
3254 while (it.hasNext()){
3255 Map.Entry<String, MovedRegionInfo> e = it.next();
3256 if (e.getValue().getMoveTime() < cutOff) {
3257 it.remove();
3258 }
3259 }
3260 }
3261
3262
3263
3264
3265
3266 protected int movedRegionCleanerPeriod() {
3267 return TIMEOUT_REGION_MOVED;
3268 }
3269
3270
3271
3272
3273
3274 protected final static class MovedRegionsCleaner extends ScheduledChore implements Stoppable {
3275 private HRegionServer regionServer;
3276 Stoppable stoppable;
3277
3278 private MovedRegionsCleaner(
3279 HRegionServer regionServer, Stoppable stoppable){
3280 super("MovedRegionsCleaner for region " + regionServer, stoppable,
3281 regionServer.movedRegionCleanerPeriod());
3282 this.regionServer = regionServer;
3283 this.stoppable = stoppable;
3284 }
3285
3286 static MovedRegionsCleaner create(HRegionServer rs){
3287 Stoppable stoppable = new Stoppable() {
3288 private volatile boolean isStopped = false;
3289 @Override public void stop(String why) { isStopped = true;}
3290 @Override public boolean isStopped() {return isStopped;}
3291 };
3292
3293 return new MovedRegionsCleaner(rs, stoppable);
3294 }
3295
3296 @Override
3297 protected void chore() {
3298 regionServer.cleanMovedRegions();
3299 }
3300
3301 @Override
3302 public void stop(String why) {
3303 stoppable.stop(why);
3304 }
3305
3306 @Override
3307 public boolean isStopped() {
3308 return stoppable.isStopped();
3309 }
3310 }
3311
3312 private String getMyEphemeralNodePath() {
3313 return ZKUtil.joinZNode(this.zooKeeper.rsZNode, getServerName().toString());
3314 }
3315
3316 private boolean isHealthCheckerConfigured() {
3317 String healthScriptLocation = this.conf.get(HConstants.HEALTH_SCRIPT_LOC);
3318 return org.apache.commons.lang.StringUtils.isNotBlank(healthScriptLocation);
3319 }
3320
3321
3322
3323
3324 public CompactSplitThread getCompactSplitThread() {
3325 return this.compactSplitThread;
3326 }
3327
3328
3329
3330
3331
3332
3333
3334
3335 private void updateRecoveringRegionLastFlushedSequenceId(Region r) throws KeeperException,
3336 IOException {
3337 if (!r.isRecovering()) {
3338
3339 return;
3340 }
3341
3342 HRegionInfo regionInfo = r.getRegionInfo();
3343 ZooKeeperWatcher zkw = getZooKeeper();
3344 String previousRSName = this.getLastFailedRSFromZK(regionInfo.getEncodedName());
3345 Map<byte[], Long> maxSeqIdInStores = r.getMaxStoreSeqId();
3346 long minSeqIdForLogReplay = -1;
3347 for (Long storeSeqIdForReplay : maxSeqIdInStores.values()) {
3348 if (minSeqIdForLogReplay == -1 || storeSeqIdForReplay < minSeqIdForLogReplay) {
3349 minSeqIdForLogReplay = storeSeqIdForReplay;
3350 }
3351 }
3352
3353 try {
3354 long lastRecordedFlushedSequenceId = -1;
3355 String nodePath = ZKUtil.joinZNode(this.zooKeeper.recoveringRegionsZNode,
3356 regionInfo.getEncodedName());
3357
3358 byte[] data;
3359 try {
3360 data = ZKUtil.getData(zkw, nodePath);
3361 } catch (InterruptedException e) {
3362 throw new InterruptedIOException();
3363 }
3364 if (data != null) {
3365 lastRecordedFlushedSequenceId = ZKSplitLog.parseLastFlushedSequenceIdFrom(data);
3366 }
3367 if (data == null || lastRecordedFlushedSequenceId < minSeqIdForLogReplay) {
3368 ZKUtil.setData(zkw, nodePath, ZKUtil.positionToByteArray(minSeqIdForLogReplay));
3369 }
3370 if (previousRSName != null) {
3371
3372 nodePath = ZKUtil.joinZNode(nodePath, previousRSName);
3373 ZKUtil.setData(zkw, nodePath,
3374 ZKUtil.regionSequenceIdsToByteArray(minSeqIdForLogReplay, maxSeqIdInStores));
3375 LOG.debug("Update last flushed sequence id of region " + regionInfo.getEncodedName() +
3376 " for " + previousRSName);
3377 } else {
3378 LOG.warn("Can't find failed region server for recovering region " +
3379 regionInfo.getEncodedName());
3380 }
3381 } catch (NoNodeException ignore) {
3382 LOG.debug("Region " + regionInfo.getEncodedName() +
3383 " must have completed recovery because its recovery znode has been removed", ignore);
3384 }
3385 }
3386
3387
3388
3389
3390
3391
3392 private String getLastFailedRSFromZK(String encodedRegionName) throws KeeperException {
3393 String result = null;
3394 long maxZxid = 0;
3395 ZooKeeperWatcher zkw = this.getZooKeeper();
3396 String nodePath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, encodedRegionName);
3397 List<String> failedServers = ZKUtil.listChildrenNoWatch(zkw, nodePath);
3398 if (failedServers == null || failedServers.isEmpty()) {
3399 return result;
3400 }
3401 for (String failedServer : failedServers) {
3402 String rsPath = ZKUtil.joinZNode(nodePath, failedServer);
3403 Stat stat = new Stat();
3404 ZKUtil.getDataNoWatch(zkw, rsPath, stat);
3405 if (maxZxid < stat.getCzxid()) {
3406 maxZxid = stat.getCzxid();
3407 result = failedServer;
3408 }
3409 }
3410 return result;
3411 }
3412
3413 public CoprocessorServiceResponse execRegionServerService(final RpcController controller,
3414 final CoprocessorServiceRequest serviceRequest) throws ServiceException {
3415 try {
3416 ServerRpcController execController = new ServerRpcController();
3417 CoprocessorServiceCall call = serviceRequest.getCall();
3418 String serviceName = call.getServiceName();
3419 String methodName = call.getMethodName();
3420 if (!coprocessorServiceHandlers.containsKey(serviceName)) {
3421 throw new UnknownProtocolException(null,
3422 "No registered coprocessor service found for name " + serviceName);
3423 }
3424 Service service = coprocessorServiceHandlers.get(serviceName);
3425 Descriptors.ServiceDescriptor serviceDesc = service.getDescriptorForType();
3426 Descriptors.MethodDescriptor methodDesc = serviceDesc.findMethodByName(methodName);
3427 if (methodDesc == null) {
3428 throw new UnknownProtocolException(service.getClass(), "Unknown method " + methodName
3429 + " called on service " + serviceName);
3430 }
3431 Message.Builder builderForType = service.getRequestPrototype(methodDesc).newBuilderForType();
3432 ProtobufUtil.mergeFrom(builderForType, call.getRequest());
3433 Message request = builderForType.build();
3434 final Message.Builder responseBuilder =
3435 service.getResponsePrototype(methodDesc).newBuilderForType();
3436 service.callMethod(methodDesc, controller, request, new RpcCallback<Message>() {
3437 @Override
3438 public void run(Message message) {
3439 if (message != null) {
3440 responseBuilder.mergeFrom(message);
3441 }
3442 }
3443 });
3444 Message execResult = responseBuilder.build();
3445 if (execController.getFailedOn() != null) {
3446 throw execController.getFailedOn();
3447 }
3448 ClientProtos.CoprocessorServiceResponse.Builder builder =
3449 ClientProtos.CoprocessorServiceResponse.newBuilder();
3450 builder.setRegion(RequestConverter.buildRegionSpecifier(RegionSpecifierType.REGION_NAME,
3451 HConstants.EMPTY_BYTE_ARRAY));
3452 builder.setValue(builder.getValueBuilder().setName(execResult.getClass().getName())
3453 .setValue(execResult.toByteString()));
3454 return builder.build();
3455 } catch (IOException ie) {
3456 throw new ServiceException(ie);
3457 }
3458 }
3459
3460
3461
3462
3463 public CacheConfig getCacheConfig() {
3464 return this.cacheConfig;
3465 }
3466
3467
3468
3469
3470 protected ConfigurationManager getConfigurationManager() {
3471 return configurationManager;
3472 }
3473
3474
3475
3476
3477 public TableDescriptors getTableDescriptors() {
3478 return this.tableDescriptors;
3479 }
3480
3481
3482
3483
3484 public void updateConfiguration() {
3485 LOG.info("Reloading the configuration from disk.");
3486
3487 conf.reloadConfiguration();
3488 configurationManager.notifyAllObservers(conf);
3489 }
3490
3491 @Override
3492 public HeapMemoryManager getHeapMemoryManager() {
3493 return hMemManager;
3494 }
3495
3496 @Override
3497 public double getCompactionPressure() {
3498 double max = 0;
3499 for (Region region : onlineRegions.values()) {
3500 for (Store store : region.getStores()) {
3501 double normCount = store.getCompactionPressure();
3502 if (normCount > max) {
3503 max = normCount;
3504 }
3505 }
3506 }
3507 return max;
3508 }
3509
3510 @Override
3511 public MetricsRegionServer getMetrics() {
3512 return metricsRegionServer;
3513 }
3514
3515 @Override
3516 public RegionServerSpaceQuotaManager getRegionServerSpaceQuotaManager() {
3517 return this.rsSpaceQuotaManager;
3518 }
3519 }