View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import java.io.IOException;
22  import java.io.InterruptedIOException;
23  import java.lang.Thread.UncaughtExceptionHandler;
24  import java.lang.management.ManagementFactory;
25  import java.lang.management.MemoryUsage;
26  import java.lang.reflect.Constructor;
27  import java.net.BindException;
28  import java.net.InetAddress;
29  import java.net.InetSocketAddress;
30  import java.util.ArrayList;
31  import java.util.Collection;
32  import java.util.Collections;
33  import java.util.Comparator;
34  import java.util.HashMap;
35  import java.util.HashSet;
36  import java.util.Iterator;
37  import java.util.List;
38  import java.util.Map;
39  import java.util.Objects;
40  import java.util.Map.Entry;
41  import java.util.Set;
42  import java.util.SortedMap;
43  import java.util.TreeMap;
44  import java.util.TreeSet;
45  import java.util.concurrent.ConcurrentHashMap;
46  import java.util.concurrent.ConcurrentMap;
47  import java.util.concurrent.ConcurrentSkipListMap;
48  import java.util.concurrent.CountDownLatch;
49  import java.util.concurrent.TimeUnit;
50  import java.util.concurrent.atomic.AtomicBoolean;
51  import java.util.concurrent.atomic.AtomicReference;
52  import java.util.concurrent.locks.ReentrantReadWriteLock;
53  
54  import javax.management.MalformedObjectNameException;
55  import javax.management.ObjectName;
56  import javax.servlet.http.HttpServlet;
57  
58  import org.apache.commons.lang.math.RandomUtils;
59  import org.apache.commons.logging.Log;
60  import org.apache.commons.logging.LogFactory;
61  import org.apache.hadoop.conf.Configuration;
62  import org.apache.hadoop.fs.FileSystem;
63  import org.apache.hadoop.fs.Path;
64  import org.apache.hadoop.hbase.ChoreService;
65  import org.apache.hadoop.hbase.ClockOutOfSyncException;
66  import org.apache.hadoop.hbase.CoordinatedStateManager;
67  import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
68  import org.apache.hadoop.hbase.DoNotRetryIOException;
69  import org.apache.hadoop.hbase.HBaseConfiguration;
70  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
71  import org.apache.hadoop.hbase.HConstants;
72  import org.apache.hadoop.hbase.HRegionInfo;
73  import org.apache.hadoop.hbase.HealthCheckChore;
74  import org.apache.hadoop.hbase.MetaTableAccessor;
75  import org.apache.hadoop.hbase.NotServingRegionException;
76  import org.apache.hadoop.hbase.PleaseHoldException;
77  import org.apache.hadoop.hbase.RemoteExceptionHandler;
78  import org.apache.hadoop.hbase.ScheduledChore;
79  import org.apache.hadoop.hbase.ServerName;
80  import org.apache.hadoop.hbase.Stoppable;
81  import org.apache.hadoop.hbase.TableDescriptors;
82  import org.apache.hadoop.hbase.TableName;
83  import org.apache.hadoop.hbase.YouAreDeadException;
84  import org.apache.hadoop.hbase.ZNodeClearer;
85  import org.apache.hadoop.hbase.backup.impl.BackupManager;
86  import org.apache.hadoop.hbase.classification.InterfaceAudience;
87  import org.apache.hadoop.hbase.client.ClusterConnection;
88  import org.apache.hadoop.hbase.client.ConnectionFactory;
89  import org.apache.hadoop.hbase.client.ConnectionUtils;
90  import org.apache.hadoop.hbase.client.RpcRetryingCallerFactory;
91  import org.apache.hadoop.hbase.conf.ConfigurationManager;
92  import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
93  import org.apache.hadoop.hbase.coordination.CloseRegionCoordination;
94  import org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination;
95  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
96  import org.apache.hadoop.hbase.exceptions.RegionMovedException;
97  import org.apache.hadoop.hbase.exceptions.RegionOpeningException;
98  import org.apache.hadoop.hbase.exceptions.UnknownProtocolException;
99  import org.apache.hadoop.hbase.executor.ExecutorService;
100 import org.apache.hadoop.hbase.executor.ExecutorType;
101 import org.apache.hadoop.hbase.fs.HFileSystem;
102 import org.apache.hadoop.hbase.http.HttpServer;
103 import org.apache.hadoop.hbase.http.InfoServer;
104 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
105 import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils;
106 import org.apache.hadoop.hbase.ipc.RpcClient;
107 import org.apache.hadoop.hbase.ipc.RpcClientFactory;
108 import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
109 import org.apache.hadoop.hbase.ipc.RpcServerInterface;
110 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
111 import org.apache.hadoop.hbase.ipc.ServerRpcController;
112 import org.apache.hadoop.hbase.master.HMaster;
113 import org.apache.hadoop.hbase.master.RegionState.State;
114 import org.apache.hadoop.hbase.master.TableLockManager;
115 import org.apache.hadoop.hbase.mob.MobCacheConfig;
116 import org.apache.hadoop.hbase.procedure.RegionServerProcedureManagerHost;
117 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
118 import org.apache.hadoop.hbase.protobuf.RequestConverter;
119 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
120 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceCall;
121 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
122 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
123 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos;
124 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad;
125 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionStoreSequenceIds;
126 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.Coprocessor;
127 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.Coprocessor.Builder;
128 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
129 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo;
130 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier;
131 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType;
132 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdRequest;
133 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdResponse;
134 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
135 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
136 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupResponse;
137 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStatusService;
138 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionSpaceUse;
139 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportRequest;
140 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition;
141 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
142 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorRequest;
143 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest;
144 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionResponse;
145 import org.apache.hadoop.hbase.quotas.FileSystemUtilizationChore;
146 import org.apache.hadoop.hbase.quotas.QuotaUtil;
147 import org.apache.hadoop.hbase.quotas.RegionServerRpcQuotaManager;
148 import org.apache.hadoop.hbase.quotas.RegionServerSpaceQuotaManager;
149 import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
150 import org.apache.hadoop.hbase.regionserver.handler.CloseMetaHandler;
151 import org.apache.hadoop.hbase.regionserver.handler.CloseRegionHandler;
152 import org.apache.hadoop.hbase.regionserver.handler.RegionReplicaFlushHandler;
153 import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
154 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
155 import org.apache.hadoop.hbase.replication.regionserver.Replication;
156 import org.apache.hadoop.hbase.replication.regionserver.ReplicationLoad;
157 import org.apache.hadoop.hbase.security.Superusers;
158 import org.apache.hadoop.hbase.security.UserProvider;
159 import org.apache.hadoop.hbase.trace.SpanReceiverHost;
160 import org.apache.hadoop.hbase.util.Addressing;
161 import org.apache.hadoop.hbase.util.ByteStringer;
162 import org.apache.hadoop.hbase.util.Bytes;
163 import org.apache.hadoop.hbase.util.CompressionTest;
164 import org.apache.hadoop.hbase.util.ConfigUtil;
165 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
166 import org.apache.hadoop.hbase.util.FSTableDescriptors;
167 import org.apache.hadoop.hbase.util.FSUtils;
168 import org.apache.hadoop.hbase.util.HasThread;
169 import org.apache.hadoop.hbase.util.JSONBean;
170 import org.apache.hadoop.hbase.util.JvmPauseMonitor;
171 import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
172 import org.apache.hadoop.hbase.util.Sleeper;
173 import org.apache.hadoop.hbase.util.Threads;
174 import org.apache.hadoop.hbase.util.VersionInfo;
175 import org.apache.hadoop.hbase.wal.DefaultWALProvider;
176 import org.apache.hadoop.hbase.wal.WAL;
177 import org.apache.hadoop.hbase.wal.WALFactory;
178 import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
179 import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
180 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
181 import org.apache.hadoop.hbase.zookeeper.RecoveringRegionWatcher;
182 import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
183 import org.apache.hadoop.hbase.zookeeper.ZKSplitLog;
184 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
185 import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker;
186 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
187 import org.apache.hadoop.ipc.RemoteException;
188 import org.apache.hadoop.metrics.util.MBeanUtil;
189 import org.apache.hadoop.util.ReflectionUtils;
190 import org.apache.hadoop.util.StringUtils;
191 import org.apache.zookeeper.KeeperException;
192 import org.apache.zookeeper.KeeperException.NoNodeException;
193 import org.apache.zookeeper.data.Stat;
194 
195 import com.google.common.annotations.VisibleForTesting;
196 import com.google.common.base.Preconditions;
197 import com.google.common.collect.Maps;
198 import com.google.protobuf.BlockingRpcChannel;
199 import com.google.protobuf.Descriptors;
200 import com.google.protobuf.Message;
201 import com.google.protobuf.RpcCallback;
202 import com.google.protobuf.RpcController;
203 import com.google.protobuf.Service;
204 import com.google.protobuf.ServiceException;
205 
206 /**
207  * HRegionServer makes a set of HRegions available to clients. It checks in with
208  * the HMaster. There are many HRegionServers in a single HBase deployment.
209  */
210 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
211 @SuppressWarnings("deprecation")
212 public class HRegionServer extends HasThread implements
213     RegionServerServices, LastSequenceId {
214 
215   public static final Log LOG = LogFactory.getLog(HRegionServer.class);
216 
217   /*
218    * Strings to be used in forming the exception message for
219    * RegionsAlreadyInTransitionException.
220    */
221   protected static final String OPEN = "OPEN";
222   protected static final String CLOSE = "CLOSE";
223 
224   //RegionName vs current action in progress
225   //true - if open region action in progress
226   //false - if close region action in progress
227   protected final ConcurrentMap<byte[], Boolean> regionsInTransitionInRS =
228     new ConcurrentSkipListMap<byte[], Boolean>(Bytes.BYTES_COMPARATOR);
229 
230   // Cache flushing
231   protected MemStoreFlusher cacheFlusher;
232 
233   protected HeapMemoryManager hMemManager;
234   protected CountDownLatch initLatch = null;
235 
236   /**
237    * Cluster connection to be shared by services.
238    * Initialized at server startup and closed when server shuts down.
239    * Clients must never close it explicitly.
240    */
241   protected ClusterConnection clusterConnection;
242 
243   /*
244    * Long-living meta table locator, which is created when the server is started and stopped
245    * when server shuts down. References to this locator shall be used to perform according
246    * operations in EventHandlers. Primary reason for this decision is to make it mockable
247    * for tests.
248    */
249   protected MetaTableLocator metaTableLocator;
250 
251   // Watch if a region is out of recovering state from ZooKeeper
252   @SuppressWarnings("unused")
253   private RecoveringRegionWatcher recoveringRegionWatcher;
254 
255   /**
256    * Go here to get table descriptors.
257    */
258   protected TableDescriptors tableDescriptors;
259 
260   // Replication services. If no replication, this handler will be null.
261   protected ReplicationSourceService replicationSourceHandler;
262   protected ReplicationSinkService replicationSinkHandler;
263 
264   // Compactions
265   public CompactSplitThread compactSplitThread;
266 
267   /**
268    * Map of regions currently being served by this region server. Key is the
269    * encoded region name.  All access should be synchronized.
270    */
271   protected final Map<String, Region> onlineRegions = new ConcurrentHashMap<String, Region>();
272 
273   /**
274    * Map of encoded region names to the DataNode locations they should be hosted on
275    * We store the value as InetSocketAddress since this is used only in HDFS
276    * API (create() that takes favored nodes as hints for placing file blocks).
277    * We could have used ServerName here as the value class, but we'd need to
278    * convert it to InetSocketAddress at some point before the HDFS API call, and
279    * it seems a bit weird to store ServerName since ServerName refers to RegionServers
280    * and here we really mean DataNode locations.
281    */
282   protected final Map<String, InetSocketAddress[]> regionFavoredNodesMap =
283       new ConcurrentHashMap<String, InetSocketAddress[]>();
284 
285   /**
286    * Set of regions currently being in recovering state which means it can accept writes(edits from
287    * previous failed region server) but not reads. A recovering region is also an online region.
288    */
289   protected final Map<String, Region> recoveringRegions = Collections
290       .synchronizedMap(new HashMap<String, Region>());
291 
292   // Leases
293   protected Leases leases;
294 
295   // Instance of the hbase executor service.
296   protected ExecutorService service;
297 
298   // If false, the file system has become unavailable
299   protected volatile boolean fsOk;
300   protected HFileSystem fs;
301   protected HFileSystem walFs;
302 
303   // Set when a report to the master comes back with a message asking us to
304   // shutdown. Also set by call to stop when debugging or running unit tests
305   // of HRegionServer in isolation.
306   private volatile boolean stopped = false;
307 
308   // Go down hard. Used if file system becomes unavailable and also in
309   // debugging and unit tests.
310   private volatile boolean abortRequested;
311 
312   ConcurrentMap<String, Integer> rowlocks = new ConcurrentHashMap<String, Integer>();
313 
314   // A state before we go into stopped state.  At this stage we're closing user
315   // space regions.
316   private boolean stopping = false;
317 
318   private volatile boolean killed = false;
319 
320   protected final Configuration conf;
321 
322   private Path rootDir;
323   private Path walRootDir;
324 
325   protected final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
326 
327   final int numRetries;
328   protected final int threadWakeFrequency;
329   protected final int msgInterval;
330 
331   protected final int numRegionsToReport;
332 
333   // Stub to do region server status calls against the master.
334   private volatile RegionServerStatusService.BlockingInterface rssStub;
335   // RPC client. Used to make the stub above that does region server status checking.
336   RpcClient rpcClient;
337 
338   private RpcRetryingCallerFactory rpcRetryingCallerFactory;
339   private RpcControllerFactory rpcControllerFactory;
340 
341   private UncaughtExceptionHandler uncaughtExceptionHandler;
342 
343   // Info server. Default access so can be used by unit tests. REGIONSERVER
344   // is name of the webapp and the attribute name used stuffing this instance
345   // into web context.
346   protected InfoServer infoServer;
347   private JvmPauseMonitor pauseMonitor;
348 
349   /** region server process name */
350   public static final String REGIONSERVER = "regionserver";
351 
352   MetricsRegionServer metricsRegionServer;
353   MetricsTable metricsTable;
354   private SpanReceiverHost spanReceiverHost;
355 
356   /**
357    * ChoreService used to schedule tasks that we want to run periodically
358    */
359   private final ChoreService choreService;
360 
361   /*
362    * Check for compactions requests.
363    */
364   ScheduledChore compactionChecker;
365 
366   /*
367    * Check for flushes
368    */
369   ScheduledChore periodicFlusher;
370 
371   protected volatile WALFactory walFactory;
372 
373   // WAL roller. log is protected rather than private to avoid
374   // eclipse warning when accessed by inner classes
375   final LogRoller walRoller;
376   // Lazily initialized if this RegionServer hosts a meta table.
377   final AtomicReference<LogRoller> metawalRoller = new AtomicReference<LogRoller>();
378 
379   // flag set after we're done setting up server threads
380   final AtomicBoolean online = new AtomicBoolean(false);
381 
382   // zookeeper connection and watcher
383   protected ZooKeeperWatcher zooKeeper;
384 
385   // master address tracker
386   private MasterAddressTracker masterAddressTracker;
387 
388   // Cluster Status Tracker
389   protected ClusterStatusTracker clusterStatusTracker;
390 
391   // Log Splitting Worker
392   private SplitLogWorker splitLogWorker;
393 
394   // A sleeper that sleeps for msgInterval.
395   protected final Sleeper sleeper;
396 
397   private final int operationTimeout;
398   private final int shortOperationTimeout;
399 
400   private final RegionServerAccounting regionServerAccounting;
401 
402   // Cache configuration and block cache reference
403   protected CacheConfig cacheConfig;
404   // Cache configuration for mob
405   final MobCacheConfig mobCacheConfig;
406 
407   /** The health check chore. */
408   private HealthCheckChore healthCheckChore;
409 
410   /** The nonce manager chore. */
411   private ScheduledChore nonceManagerChore;
412 
413   private Map<String, Service> coprocessorServiceHandlers = Maps.newHashMap();
414 
415   /**
416    * The server name the Master sees us as.  Its made from the hostname the
417    * master passes us, port, and server startcode. Gets set after registration
418    * against  Master.
419    */
420   protected ServerName serverName;
421 
422   /*
423    * hostname specified by hostname config
424    */
425   private String useThisHostnameInstead;
426 
427   // key to the config parameter of server hostname
428   // the specification of server hostname is optional. The hostname should be resolvable from
429   // both master and region server
430   final static String RS_HOSTNAME_KEY = "hbase.regionserver.hostname";
431 
432   final static String MASTER_HOSTNAME_KEY = "hbase.master.hostname";
433 
434   // HBASE-18226: This config and hbase.regionserver.hostname are mutually exclusive.
435   // Exception will be thrown if both are used.
436   final static String RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY =
437     "hbase.regionserver.hostname.disable.master.reversedns";
438 
439   /**
440    * This servers startcode.
441    */
442   protected final long startcode;
443 
444   /**
445    * Unique identifier for the cluster we are a part of.
446    */
447   private String clusterId;
448 
449   /**
450    * MX Bean for RegionServerInfo
451    */
452   private ObjectName mxBean = null;
453 
454   /**
455    * Chore to clean periodically the moved region list
456    */
457   private MovedRegionsCleaner movedRegionsCleaner;
458 
459   // chore for refreshing store files for secondary regions
460   private StorefileRefresherChore storefileRefresher;
461 
462   private RegionServerCoprocessorHost rsHost;
463 
464   private RegionServerProcedureManagerHost rspmHost;
465 
466   private RegionServerRpcQuotaManager rsQuotaManager;
467   private RegionServerSpaceQuotaManager rsSpaceQuotaManager;
468 
469   // Table level lock manager for locking for region operations
470   protected TableLockManager tableLockManager;
471 
472   /**
473    * Nonce manager. Nonces are used to make operations like increment and append idempotent
474    * in the case where client doesn't receive the response from a successful operation and
475    * retries. We track the successful ops for some time via a nonce sent by client and handle
476    * duplicate operations (currently, by failing them; in future we might use MVCC to return
477    * result). Nonces are also recovered from WAL during, recovery; however, the caveats (from
478    * HBASE-3787) are:
479    * - WAL recovery is optimized, and under high load we won't read nearly nonce-timeout worth
480    *   of past records. If we don't read the records, we don't read and recover the nonces.
481    *   Some WALs within nonce-timeout at recovery may not even be present due to rolling/cleanup.
482    * - There's no WAL recovery during normal region move, so nonces will not be transfered.
483    * We can have separate additional "Nonce WAL". It will just contain bunch of numbers and
484    * won't be flushed on main path - because WAL itself also contains nonces, if we only flush
485    * it before memstore flush, for a given nonce we will either see it in the WAL (if it was
486    * never flushed to disk, it will be part of recovery), or we'll see it as part of the nonce
487    * log (or both occasionally, which doesn't matter). Nonce log file can be deleted after the
488    * latest nonce in it expired. It can also be recovered during move.
489    */
490   final ServerNonceManager nonceManager;
491 
492   private UserProvider userProvider;
493 
494   protected final RSRpcServices rpcServices;
495 
496   protected BaseCoordinatedStateManager csm;
497 
498   private final boolean useZKForAssignment;
499 
500   /**
501    * Configuration manager is used to register/deregister and notify the configuration observers
502    * when the regionserver is notified that there was a change in the on disk configs.
503    */
504   protected final ConfigurationManager configurationManager;
505 
506   protected FileSystemUtilizationChore fsUtilizationChore;
507 
508   /**
509    * Starts a HRegionServer at the default location.
510    * @param conf
511    * @throws IOException
512    * @throws InterruptedException
513    */
514   public HRegionServer(Configuration conf) throws IOException, InterruptedException {
515     this(conf, CoordinatedStateManagerFactory.getCoordinatedStateManager(conf));
516   }
517 
518   /**
519    * Starts a HRegionServer at the default location
520    * @param conf
521    * @param csm implementation of CoordinatedStateManager to be used
522    * @throws IOException
523    * @throws InterruptedException
524    */
525   public HRegionServer(Configuration conf, CoordinatedStateManager csm)
526       throws IOException, InterruptedException {
527     this.fsOk = true;
528     this.conf = conf;
529     checkCodecs(this.conf);
530     this.userProvider = UserProvider.instantiate(conf);
531     FSUtils.setupShortCircuitRead(this.conf);
532 
533     Replication.decorateRegionServerConfiguration(this.conf);
534 
535     // Disable usage of meta replicas in the regionserver
536     this.conf.setBoolean(HConstants.USE_META_REPLICAS, false);
537     BackupManager.decorateRSConfiguration(conf);
538     // Config'ed params
539     this.numRetries = this.conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
540         HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
541     this.threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000);
542     this.msgInterval = conf.getInt("hbase.regionserver.msginterval", 3 * 1000);
543 
544     this.sleeper = new Sleeper(this.msgInterval, this);
545 
546     boolean isNoncesEnabled = conf.getBoolean(HConstants.HBASE_RS_NONCES_ENABLED, true);
547     this.nonceManager = isNoncesEnabled ? new ServerNonceManager(this.conf) : null;
548 
549     this.numRegionsToReport = conf.getInt(
550       "hbase.regionserver.numregionstoreport", 10);
551 
552     this.operationTimeout = conf.getInt(
553       HConstants.HBASE_CLIENT_OPERATION_TIMEOUT,
554       HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT);
555 
556     this.shortOperationTimeout = conf.getInt(
557       HConstants.HBASE_RPC_SHORTOPERATION_TIMEOUT_KEY,
558       HConstants.DEFAULT_HBASE_RPC_SHORTOPERATION_TIMEOUT);
559 
560     this.abortRequested = false;
561     this.stopped = false;
562 
563     rpcServices = createRpcServices();
564     this.startcode = System.currentTimeMillis();
565     if (this instanceof HMaster) {
566       useThisHostnameInstead = conf.get(MASTER_HOSTNAME_KEY);
567     } else {
568       useThisHostnameInstead = conf.get(RS_HOSTNAME_KEY);
569       if (conf.getBoolean(RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY, false)) {
570         if (shouldUseThisHostnameInstead()) {
571           String msg = RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY + " and " + RS_HOSTNAME_KEY +
572             " are mutually exclusive. Do not set " + RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY +
573             " to true while " + RS_HOSTNAME_KEY + " is used";
574           throw new IOException(msg);
575         } else {
576           useThisHostnameInstead = rpcServices.isa.getHostName();
577         }
578       }
579     }
580     String hostName = shouldUseThisHostnameInstead() ? useThisHostnameInstead :
581       rpcServices.isa.getHostName();
582     serverName = ServerName.valueOf(hostName, rpcServices.isa.getPort(), startcode);
583 
584     rpcControllerFactory = RpcControllerFactory.instantiate(this.conf);
585     rpcRetryingCallerFactory = RpcRetryingCallerFactory.instantiate(this.conf);
586 
587     // login the zookeeper client principal (if using security)
588     ZKUtil.loginClient(this.conf, "hbase.zookeeper.client.keytab.file",
589       "hbase.zookeeper.client.kerberos.principal", hostName);
590     // login the server principal (if using secure Hadoop)
591     login(userProvider, hostName);
592     // init superusers and add the server principal (if using security)
593     // or process owner as default super user.
594     Superusers.initialize(conf);
595 
596     regionServerAccounting = new RegionServerAccounting();
597     cacheConfig = new CacheConfig(conf);
598     mobCacheConfig = new MobCacheConfig(conf);
599     uncaughtExceptionHandler = new UncaughtExceptionHandler() {
600       @Override
601       public void uncaughtException(Thread t, Throwable e) {
602         abort("Uncaught exception in service thread " + t.getName(), e);
603       }
604     };
605 
606     useZKForAssignment = ConfigUtil.useZKForAssignment(conf);
607 
608     initializeFileSystem();
609 
610     service = new ExecutorService(getServerName().toShortString());
611     spanReceiverHost = SpanReceiverHost.getInstance(getConfiguration());
612 
613     // Some unit tests don't need a cluster, so no zookeeper at all
614     if (!conf.getBoolean("hbase.testing.nocluster", false)) {
615       // Open connection to zookeeper and set primary watcher
616       zooKeeper = new ZooKeeperWatcher(conf, getProcessName() + ":" +
617         rpcServices.isa.getPort(), this, canCreateBaseZNode());
618 
619       this.csm = (BaseCoordinatedStateManager) csm;
620       this.csm.initialize(this);
621       this.csm.start();
622 
623       tableLockManager = TableLockManager.createTableLockManager(
624         conf, zooKeeper, serverName);
625 
626       masterAddressTracker = new MasterAddressTracker(getZooKeeper(), this);
627       masterAddressTracker.start();
628 
629       clusterStatusTracker = new ClusterStatusTracker(zooKeeper, this);
630       clusterStatusTracker.start();
631     }
632     this.configurationManager = new ConfigurationManager();
633 
634     rpcServices.start();
635     putUpWebUI();
636     this.walRoller = new LogRoller(this, this);
637     this.choreService = new ChoreService(getServerName().toString());
638   }
639 
640   protected void setInitLatch(CountDownLatch latch) {
641     this.initLatch = latch;
642   }
643 
644   private void initializeFileSystem() throws IOException {
645     // Get fs instance used by this RS.  Do we use checksum verification in the hbase? If hbase
646     // checksum verification enabled, then automatically switch off hdfs checksum verification.
647     boolean useHBaseChecksum = conf.getBoolean(HConstants.HBASE_CHECKSUM_VERIFICATION, true);
648     FSUtils.setFsDefault(this.conf, FSUtils.getWALRootDir(this.conf));
649     this.walFs = new HFileSystem(this.conf, useHBaseChecksum);
650     this.walRootDir = FSUtils.getWALRootDir(this.conf);
651     // Set 'fs.defaultFS' to match the filesystem on hbase.rootdir else
652     // underlying hadoop hdfs accessors will be going against wrong filesystem
653     // (unless all is set to defaults).
654     FSUtils.setFsDefault(this.conf, FSUtils.getRootDir(this.conf));
655     this.fs = new HFileSystem(this.conf, useHBaseChecksum);
656     this.rootDir = FSUtils.getRootDir(this.conf);
657     this.tableDescriptors = new FSTableDescriptors(this.conf, this.fs, this.rootDir,
658         !canUpdateTableDescriptor(), false);
659   }
660 
661   /*
662    * Returns true if configured hostname should be used
663    */
664   protected boolean shouldUseThisHostnameInstead() {
665     return useThisHostnameInstead != null && !useThisHostnameInstead.isEmpty();
666   }
667 
668   protected void login(UserProvider user, String host) throws IOException {
669     user.login("hbase.regionserver.keytab.file",
670       "hbase.regionserver.kerberos.principal", host);
671   }
672 
673   protected void waitForMasterActive(){
674   }
675 
676   protected String getProcessName() {
677     return REGIONSERVER;
678   }
679 
680   protected boolean canCreateBaseZNode() {
681     return false;
682   }
683 
684   protected boolean canUpdateTableDescriptor() {
685     return false;
686   }
687 
688   protected RSRpcServices createRpcServices() throws IOException {
689     return new RSRpcServices(this);
690   }
691 
692   protected void configureInfoServer() {
693     infoServer.addServlet("rs-status", "/rs-status", RSStatusServlet.class);
694     infoServer.setAttribute(REGIONSERVER, this);
695   }
696 
697   protected Class<? extends HttpServlet> getDumpServlet() {
698     return RSDumpServlet.class;
699   }
700 
701   @Override
702   public boolean registerService(Service instance) {
703     /*
704      * No stacking of instances is allowed for a single service name
705      */
706     Descriptors.ServiceDescriptor serviceDesc = instance.getDescriptorForType();
707     String serviceName = CoprocessorRpcUtils.getServiceName(serviceDesc);
708     if (coprocessorServiceHandlers.containsKey(serviceName)) {
709       LOG.error("Coprocessor service " + serviceName
710           + " already registered, rejecting request from " + instance);
711       return false;
712     }
713 
714     coprocessorServiceHandlers.put(serviceName, instance);
715     if (LOG.isDebugEnabled()) {
716       LOG.debug("Registered regionserver coprocessor service: service=" + serviceName);
717     }
718     return true;
719   }
720 
721   /**
722    * Create a 'smarter' HConnection, one that is capable of by-passing RPC if the request is to
723    * the local server.  Safe to use going to local or remote server.
724    * Create this instance in a method can be intercepted and mocked in tests.
725    * @throws IOException
726    */
727   @VisibleForTesting
728   protected ClusterConnection createClusterConnection() throws IOException {
729     // Create a cluster connection that when appropriate, can short-circuit and go directly to the
730     // local server if the request is to the local server bypassing RPC. Can be used for both local
731     // and remote invocations.
732     return ConnectionUtils.createShortCircuitHConnection(
733       ConnectionFactory.createConnection(conf), serverName, rpcServices, rpcServices);
734   }
735 
736   /**
737    * Run test on configured codecs to make sure supporting libs are in place.
738    * @param c
739    * @throws IOException
740    */
741   private static void checkCodecs(final Configuration c) throws IOException {
742     // check to see if the codec list is available:
743     String [] codecs = c.getStrings("hbase.regionserver.codecs", (String[])null);
744     if (codecs == null) return;
745     for (String codec : codecs) {
746       if (!CompressionTest.testCompression(codec)) {
747         throw new IOException("Compression codec " + codec +
748           " not supported, aborting RS construction");
749       }
750     }
751   }
752 
753   public String getClusterId() {
754     return this.clusterId;
755   }
756 
757   /**
758    * Setup our cluster connection if not already initialized.
759    * @throws IOException
760    */
761   protected synchronized void setupClusterConnection() throws IOException {
762     if (clusterConnection == null) {
763       clusterConnection = createClusterConnection();
764       metaTableLocator = new MetaTableLocator();
765     }
766   }
767 
768   /**
769    * All initialization needed before we go register with Master.
770    *
771    * @throws IOException
772    * @throws InterruptedException
773    */
774   private void preRegistrationInitialization(){
775     try {
776       setupClusterConnection();
777 
778       // Health checker thread.
779       if (isHealthCheckerConfigured()) {
780         int sleepTime = this.conf.getInt(HConstants.HEALTH_CHORE_WAKE_FREQ,
781           HConstants.DEFAULT_THREAD_WAKE_FREQUENCY);
782         healthCheckChore = new HealthCheckChore(sleepTime, this, getConfiguration());
783       }
784 
785       initializeZooKeeper();
786       if (!isStopped() && !isAborted()) {
787         initializeThreads();
788       }
789     } catch (Throwable t) {
790       // Call stop if error or process will stick around for ever since server
791       // puts up non-daemon threads.
792       this.rpcServices.stop();
793       abort("Initialization of RS failed.  Hence aborting RS.", t);
794     }
795   }
796 
797   /**
798    * Bring up connection to zk ensemble and then wait until a master for this
799    * cluster and then after that, wait until cluster 'up' flag has been set.
800    * This is the order in which master does things.
801    * Finally open long-living server short-circuit connection.
802    * @throws IOException
803    * @throws InterruptedException
804    */
805   private void initializeZooKeeper() throws IOException, InterruptedException {
806     // Create the master address tracker, register with zk, and start it.  Then
807     // block until a master is available.  No point in starting up if no master
808     // running.
809     blockAndCheckIfStopped(this.masterAddressTracker);
810 
811     // Wait on cluster being up.  Master will set this flag up in zookeeper
812     // when ready.
813     blockAndCheckIfStopped(this.clusterStatusTracker);
814 
815     if (this.initLatch != null) {
816       this.initLatch.await(20, TimeUnit.SECONDS);
817     }
818     // Retrieve clusterId
819     // Since cluster status is now up
820     // ID should have already been set by HMaster
821     try {
822       clusterId = ZKClusterId.readClusterIdZNode(this.zooKeeper);
823       if (clusterId == null) {
824         this.abort("Cluster ID has not been set");
825       }
826       LOG.info("ClusterId : "+clusterId);
827     } catch (KeeperException e) {
828       this.abort("Failed to retrieve Cluster ID",e);
829     }
830 
831     // In case colocated master, wait here till it's active.
832     // So backup masters won't start as regionservers.
833     // This is to avoid showing backup masters as regionservers
834     // in master web UI, or assigning any region to them.
835     waitForMasterActive();
836     if (isStopped() || isAborted()) {
837       return; // No need for further initialization
838     }
839 
840     // watch for snapshots and other procedures
841     try {
842       rspmHost = new RegionServerProcedureManagerHost();
843       rspmHost.loadProcedures(conf);
844       rspmHost.initialize(this);
845     } catch (IOException e) {
846       this.abort("Failed to reach coordination cluster when creating procedure handler.", e);
847     }
848     // register watcher for recovering regions
849     this.recoveringRegionWatcher = new RecoveringRegionWatcher(this.zooKeeper, this);
850   }
851 
852   /**
853    * Utilty method to wait indefinitely on a znode availability while checking
854    * if the region server is shut down
855    * @param tracker znode tracker to use
856    * @throws IOException any IO exception, plus if the RS is stopped
857    * @throws InterruptedException
858    */
859   private void blockAndCheckIfStopped(ZooKeeperNodeTracker tracker)
860       throws IOException, InterruptedException {
861     while (tracker.blockUntilAvailable(this.msgInterval, false) == null) {
862       if (this.stopped) {
863         throw new IOException("Received the shutdown message while waiting.");
864       }
865     }
866   }
867 
868   /**
869    * @return False if cluster shutdown in progress
870    */
871   private boolean isClusterUp() {
872     return clusterStatusTracker != null && clusterStatusTracker.isClusterUp();
873   }
874 
875   private void initializeThreads() throws IOException {
876     // Cache flushing thread.
877     this.cacheFlusher = new MemStoreFlusher(conf, this);
878 
879     // Compaction thread
880     this.compactSplitThread = new CompactSplitThread(this);
881 
882     // Background thread to check for compactions; needed if region has not gotten updates
883     // in a while. It will take care of not checking too frequently on store-by-store basis.
884     this.compactionChecker = new CompactionChecker(this, this.threadWakeFrequency, this);
885     this.periodicFlusher = new PeriodicMemstoreFlusher(this.threadWakeFrequency, this);
886     this.leases = new Leases(this.threadWakeFrequency);
887 
888     // Create the thread to clean the moved regions list
889     movedRegionsCleaner = MovedRegionsCleaner.create(this);
890 
891     if (this.nonceManager != null) {
892       // Create the scheduled chore that cleans up nonces.
893       nonceManagerChore = this.nonceManager.createCleanupScheduledChore(this);
894     }
895 
896     // Setup the Quota Manager
897     rsQuotaManager = new RegionServerRpcQuotaManager(this);
898     rsSpaceQuotaManager = new RegionServerSpaceQuotaManager(this);
899 
900     if (QuotaUtil.isQuotaEnabled(conf)) {
901       this.fsUtilizationChore = new FileSystemUtilizationChore(this);
902     }
903 
904     // Setup RPC client for master communication
905     rpcClient = RpcClientFactory.createClient(conf, clusterId, new InetSocketAddress(
906         rpcServices.isa.getAddress(), 0));
907 
908     boolean onlyMetaRefresh = false;
909     int storefileRefreshPeriod = conf.getInt(
910         StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD
911       , StorefileRefresherChore.DEFAULT_REGIONSERVER_STOREFILE_REFRESH_PERIOD);
912     if (storefileRefreshPeriod == 0) {
913       storefileRefreshPeriod = conf.getInt(
914           StorefileRefresherChore.REGIONSERVER_META_STOREFILE_REFRESH_PERIOD,
915           StorefileRefresherChore.DEFAULT_REGIONSERVER_STOREFILE_REFRESH_PERIOD);
916       onlyMetaRefresh = true;
917     }
918     if (storefileRefreshPeriod > 0) {
919       this.storefileRefresher = new StorefileRefresherChore(storefileRefreshPeriod,
920           onlyMetaRefresh, this, this);
921     }
922     registerConfigurationObservers();
923   }
924 
925   private void registerConfigurationObservers() {
926     // Registering the compactSplitThread object with the ConfigurationManager.
927     configurationManager.registerObserver(this.compactSplitThread);
928   }
929 
930   /**
931    * The HRegionServer sticks in this loop until closed.
932    */
933   @Override
934   public void run() {
935     try {
936       // Do pre-registration initializations; zookeeper, lease threads, etc.
937       preRegistrationInitialization();
938     } catch (Throwable e) {
939       abort("Fatal exception during initialization", e);
940     }
941 
942     try {
943       if (!isStopped() && !isAborted()) {
944         ShutdownHook.install(conf, fs, this, Thread.currentThread());
945         // Set our ephemeral znode up in zookeeper now we have a name.
946         createMyEphemeralNode();
947         // Initialize the RegionServerCoprocessorHost now that our ephemeral
948         // node was created, in case any coprocessors want to use ZooKeeper
949         this.rsHost = new RegionServerCoprocessorHost(this, this.conf);
950       }
951 
952       // Try and register with the Master; tell it we are here.  Break if
953       // server is stopped or the clusterup flag is down or hdfs went wacky.
954       while (keepLooping()) {
955         RegionServerStartupResponse w = reportForDuty();
956         if (w == null) {
957           LOG.warn("reportForDuty failed; sleeping and then retrying.");
958           this.sleeper.sleep();
959         } else {
960           handleReportForDutyResponse(w);
961           break;
962         }
963       }
964 
965       if (!isStopped() && isHealthy()){
966         // start the snapshot handler and other procedure handlers,
967         // since the server is ready to run
968         rspmHost.start();
969       }
970       
971       // Start the Quota Manager
972       if (this.rsQuotaManager != null) {
973         rsQuotaManager.start(getRpcServer().getScheduler());
974         rsSpaceQuotaManager.start();
975       }
976 
977       // We registered with the Master.  Go into run mode.
978       long lastMsg = System.currentTimeMillis();
979       long oldRequestCount = -1;
980       // The main run loop.
981       while (!isStopped() && isHealthy()) {
982         if (!isClusterUp()) {
983           if (isOnlineRegionsEmpty()) {
984             stop("Exiting; cluster shutdown set and not carrying any regions");
985           } else if (!this.stopping) {
986             this.stopping = true;
987             LOG.info("Closing user regions");
988             closeUserRegions(this.abortRequested);
989           } else if (this.stopping) {
990             boolean allUserRegionsOffline = areAllUserRegionsOffline();
991             if (allUserRegionsOffline) {
992               // Set stopped if no more write requests tp meta tables
993               // since last time we went around the loop.  Any open
994               // meta regions will be closed on our way out.
995               if (oldRequestCount == getWriteRequestCount()) {
996                 stop("Stopped; only catalog regions remaining online");
997                 break;
998               }
999               oldRequestCount = getWriteRequestCount();
1000             } else {
1001               // Make sure all regions have been closed -- some regions may
1002               // have not got it because we were splitting at the time of
1003               // the call to closeUserRegions.
1004               closeUserRegions(this.abortRequested);
1005             }
1006             LOG.debug("Waiting on " + getOnlineRegionsAsPrintableString());
1007           }
1008         }
1009         long now = System.currentTimeMillis();
1010         if ((now - lastMsg) >= msgInterval) {
1011           tryRegionServerReport(lastMsg, now);
1012           lastMsg = System.currentTimeMillis();
1013         }
1014         if (!isStopped() && !isAborted()) {
1015           this.sleeper.sleep();
1016         }
1017       } // for
1018     } catch (Throwable t) {
1019       if (!rpcServices.checkOOME(t)) {
1020         String prefix = t instanceof YouAreDeadException? "": "Unhandled: ";
1021         abort(prefix + t.getMessage(), t);
1022       }
1023     }
1024     // Run shutdown.
1025     if (mxBean != null) {
1026       MBeanUtil.unregisterMBean(mxBean);
1027       mxBean = null;
1028     }
1029     if (this.leases != null) this.leases.closeAfterLeasesExpire();
1030     if (this.splitLogWorker != null) {
1031       splitLogWorker.stop();
1032     }
1033     if (this.infoServer != null) {
1034       LOG.info("Stopping infoServer");
1035       try {
1036         this.infoServer.stop();
1037       } catch (Exception e) {
1038         LOG.error("Failed to stop infoServer", e);
1039       }
1040     }
1041     // Send cache a shutdown.
1042     if (cacheConfig != null && cacheConfig.isBlockCacheEnabled()) {
1043       cacheConfig.getBlockCache().shutdown();
1044     }
1045     mobCacheConfig.getMobFileCache().shutdown();
1046 
1047     if (movedRegionsCleaner != null) {
1048       movedRegionsCleaner.stop("Region Server stopping");
1049     }
1050 
1051     // Send interrupts to wake up threads if sleeping so they notice shutdown.
1052     // TODO: Should we check they are alive? If OOME could have exited already
1053     if (this.hMemManager != null) this.hMemManager.stop();
1054     if (this.cacheFlusher != null) this.cacheFlusher.interruptIfNecessary();
1055     if (this.compactSplitThread != null) this.compactSplitThread.interruptIfNecessary();
1056     if (this.compactionChecker != null) this.compactionChecker.cancel(true);
1057     if (this.healthCheckChore != null) this.healthCheckChore.cancel(true);
1058     if (this.nonceManagerChore != null) this.nonceManagerChore.cancel(true);
1059     if (this.storefileRefresher != null) this.storefileRefresher.cancel(true);
1060     sendShutdownInterrupt();
1061 
1062     // Stop the quota manager
1063     if (rsQuotaManager != null) {
1064       rsQuotaManager.stop();
1065     }
1066     if (rsSpaceQuotaManager != null) {
1067       rsSpaceQuotaManager.stop();
1068     }
1069 
1070     // Stop the snapshot and other procedure handlers, forcefully killing all running tasks
1071     if (rspmHost != null) {
1072       rspmHost.stop(this.abortRequested || this.killed);
1073     }
1074 
1075     if (this.killed) {
1076       // Just skip out w/o closing regions.  Used when testing.
1077     } else if (abortRequested) {
1078       if (this.fsOk) {
1079         closeUserRegions(abortRequested); // Don't leave any open file handles
1080       }
1081       LOG.info("aborting server " + this.serverName);
1082     } else {
1083       closeUserRegions(abortRequested);
1084       LOG.info("stopping server " + this.serverName);
1085     }
1086 
1087     // so callers waiting for meta without timeout can stop
1088     if (this.metaTableLocator != null) this.metaTableLocator.stop();
1089     if (this.clusterConnection != null && !clusterConnection.isClosed()) {
1090       try {
1091         this.clusterConnection.close();
1092       } catch (IOException e) {
1093         // Although the {@link Closeable} interface throws an {@link
1094         // IOException}, in reality, the implementation would never do that.
1095         LOG.warn("Attempt to close server's short circuit HConnection failed.", e);
1096       }
1097     }
1098 
1099     // Closing the compactSplit thread before closing meta regions
1100     if (!this.killed && containsMetaTableRegions()) {
1101       if (!abortRequested || this.fsOk) {
1102         if (this.compactSplitThread != null) {
1103           this.compactSplitThread.join();
1104           this.compactSplitThread = null;
1105         }
1106         closeMetaTableRegions(abortRequested);
1107       }
1108     }
1109 
1110     if (!this.killed && this.fsOk) {
1111       waitOnAllRegionsToClose(abortRequested);
1112       LOG.info("stopping server " + this.serverName +
1113         "; all regions closed.");
1114     }
1115 
1116     //fsOk flag may be changed when closing regions throws exception.
1117     if (this.fsOk) {
1118       shutdownWAL(!abortRequested);
1119     }
1120 
1121     // Make sure the proxy is down.
1122     if (this.rssStub != null) {
1123       this.rssStub = null;
1124     }
1125     if (this.rpcClient != null) {
1126       this.rpcClient.close();
1127     }
1128     if (this.leases != null) {
1129       this.leases.close();
1130     }
1131     if (this.pauseMonitor != null) {
1132       this.pauseMonitor.stop();
1133     }
1134 
1135     if (!killed) {
1136       stopServiceThreads();
1137     }
1138 
1139     if (this.rpcServices != null) {
1140       this.rpcServices.stop();
1141     }
1142 
1143     try {
1144       deleteMyEphemeralNode();
1145     } catch (KeeperException.NoNodeException nn) {
1146     } catch (KeeperException e) {
1147       LOG.warn("Failed deleting my ephemeral node", e);
1148     }
1149     // We may have failed to delete the znode at the previous step, but
1150     //  we delete the file anyway: a second attempt to delete the znode is likely to fail again.
1151     ZNodeClearer.deleteMyEphemeralNodeOnDisk();
1152 
1153     if (this.zooKeeper != null) {
1154       this.zooKeeper.close();
1155     }
1156     LOG.info("stopping server " + this.serverName +
1157       "; zookeeper connection closed.");
1158 
1159     LOG.info(Thread.currentThread().getName() + " exiting");
1160   }
1161 
1162   private boolean containsMetaTableRegions() {
1163     return onlineRegions.containsKey(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName());
1164   }
1165 
1166   private boolean areAllUserRegionsOffline() {
1167     if (getNumberOfOnlineRegions() > 2) return false;
1168     boolean allUserRegionsOffline = true;
1169     for (Map.Entry<String, Region> e: this.onlineRegions.entrySet()) {
1170       if (!e.getValue().getRegionInfo().isMetaTable()) {
1171         allUserRegionsOffline = false;
1172         break;
1173       }
1174     }
1175     return allUserRegionsOffline;
1176   }
1177 
1178   /**
1179    * @return Current write count for all online regions.
1180    */
1181   private long getWriteRequestCount() {
1182     long writeCount = 0;
1183     for (Map.Entry<String, Region> e: this.onlineRegions.entrySet()) {
1184       writeCount += e.getValue().getWriteRequestsCount();
1185     }
1186     return writeCount;
1187   }
1188 
1189   @VisibleForTesting
1190   protected void tryRegionServerReport(long reportStartTime, long reportEndTime)
1191   throws IOException {
1192     RegionServerStatusService.BlockingInterface rss = rssStub;
1193     if (rss == null) {
1194       // the current server could be stopping.
1195       return;
1196     }
1197     ClusterStatusProtos.ServerLoad sl = buildServerLoad(reportStartTime, reportEndTime);
1198     try {
1199       RegionServerReportRequest.Builder request = RegionServerReportRequest.newBuilder();
1200       ServerName sn = ServerName.parseVersionedServerName(
1201         this.serverName.getVersionedBytes());
1202       request.setServer(ProtobufUtil.toServerName(sn));
1203       request.setLoad(sl);
1204       rss.regionServerReport(null, request.build());
1205     } catch (ServiceException se) {
1206       IOException ioe = ProtobufUtil.getRemoteException(se);
1207       if (ioe instanceof YouAreDeadException) {
1208         // This will be caught and handled as a fatal error in run()
1209         throw ioe;
1210       }
1211       if (rssStub == rss) {
1212         rssStub = null;
1213       }
1214       // Couldn't connect to the master, get location from zk and reconnect
1215       // Method blocks until new master is found or we are stopped
1216       createRegionServerStatusStub();
1217     }
1218   }
1219 
1220   /**
1221    * Reports the given map of Regions and their size on the filesystem to the active Master.
1222    *
1223    * @param onlineRegionSizes A map of region info to size in bytes
1224    * @return false if FileSystemUtilizationChore should pause reporting to master. true otherwise
1225    */
1226   public boolean reportRegionSizesForQuotas(final Map<HRegionInfo, Long> onlineRegionSizes) {
1227     RegionServerStatusService.BlockingInterface rss = rssStub;
1228     if (rss == null) {
1229       // the current server could be stopping.
1230       LOG.trace("Skipping Region size report to HMaster as stub is null");
1231       return true;
1232     }
1233     try {
1234       RegionSpaceUseReportRequest request = buildRegionSpaceUseReportRequest(
1235           Objects.requireNonNull(onlineRegionSizes));
1236       rss.reportRegionSpaceUse(null, request);
1237     } catch (ServiceException se) {
1238       IOException ioe = ProtobufUtil.getRemoteException(se);
1239       if (ioe instanceof PleaseHoldException) {
1240         LOG.trace("Failed to report region sizes to Master because it is initializing."
1241             + " This will be retried.", ioe);
1242         // The Master is coming up. Will retry the report later. Avoid re-creating the stub.
1243         return true;
1244       }
1245       if (rssStub == rss) {
1246         rssStub = null;
1247       }
1248       createRegionServerStatusStub();
1249       if (ioe instanceof DoNotRetryIOException) {
1250         DoNotRetryIOException doNotRetryEx = (DoNotRetryIOException) ioe;
1251         if (doNotRetryEx.getCause() != null) {
1252           Throwable t = doNotRetryEx.getCause();
1253           if (t instanceof UnsupportedOperationException) {
1254             LOG.debug("master doesn't support ReportRegionSpaceUse, pause before retrying");
1255             return false;
1256           }
1257         }
1258       }
1259       LOG.debug("Failed to report region sizes to Master. This will be retried.", ioe);
1260     }
1261     return true;
1262   }
1263 
1264   /**
1265    * Builds a {@link RegionSpaceUseReportRequest} protobuf message from the region size map.
1266    *
1267    * @param regionSizes Map of region info to size in bytes.
1268    * @return The corresponding protocol buffer message.
1269    */
1270   RegionSpaceUseReportRequest buildRegionSpaceUseReportRequest(Map<HRegionInfo,Long> regionSizes) {
1271     RegionSpaceUseReportRequest.Builder request = RegionSpaceUseReportRequest.newBuilder();
1272     for (Entry<HRegionInfo, Long> entry : Objects.requireNonNull(regionSizes).entrySet()) {
1273       request.addSpaceUse(convertRegionSize(entry.getKey(), entry.getValue()));
1274     }
1275     return request.build();
1276   }
1277 
1278   /**
1279    * Converts a pair of {@link HRegionInfo} and {@code long} into a {@link RegionSpaceUse}
1280    * protobuf message.
1281    *
1282    * @param regionInfo The HRegionInfo
1283    * @param sizeInBytes The size in bytes of the Region
1284    * @return The protocol buffer
1285    */
1286   RegionSpaceUse convertRegionSize(HRegionInfo regionInfo, Long sizeInBytes) {
1287     return RegionSpaceUse.newBuilder()
1288         .setRegionInfo(HRegionInfo.convert(Objects.requireNonNull(regionInfo)))
1289         .setRegionSize(Objects.requireNonNull(sizeInBytes))
1290         .build();
1291   }
1292 
1293   ClusterStatusProtos.ServerLoad buildServerLoad(long reportStartTime, long reportEndTime)
1294       throws IOException {
1295     // We're getting the MetricsRegionServerWrapper here because the wrapper computes requests
1296     // per second, and other metrics  As long as metrics are part of ServerLoad it's best to use
1297     // the wrapper to compute those numbers in one place.
1298     // In the long term most of these should be moved off of ServerLoad and the heart beat.
1299     // Instead they should be stored in an HBase table so that external visibility into HBase is
1300     // improved; Additionally the load balancer will be able to take advantage of a more complete
1301     // history.
1302     MetricsRegionServerWrapper regionServerWrapper = metricsRegionServer.getRegionServerWrapper();
1303     Collection<Region> regions = getOnlineRegionsLocalContext();
1304     MemoryUsage memory = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
1305 
1306     ClusterStatusProtos.ServerLoad.Builder serverLoad =
1307       ClusterStatusProtos.ServerLoad.newBuilder();
1308     serverLoad.setNumberOfRequests((int) regionServerWrapper.getRequestsPerSecond());
1309     serverLoad.setTotalNumberOfRequests((int) regionServerWrapper.getTotalRequestCount());
1310     serverLoad.setUsedHeapMB((int)(memory.getUsed() / 1024 / 1024));
1311     serverLoad.setMaxHeapMB((int) (memory.getMax() / 1024 / 1024));
1312     Set<String> coprocessors = getWAL(null).getCoprocessorHost().getCoprocessors();
1313     Builder coprocessorBuilder = Coprocessor.newBuilder();
1314     for (String coprocessor : coprocessors) {
1315       serverLoad.addCoprocessors(coprocessorBuilder.setName(coprocessor).build());
1316     }
1317     RegionLoad.Builder regionLoadBldr = RegionLoad.newBuilder();
1318     RegionSpecifier.Builder regionSpecifier = RegionSpecifier.newBuilder();
1319     for (Region region : regions) {
1320       if (region.getCoprocessorHost() != null) {
1321         Set<String> regionCoprocessors = region.getCoprocessorHost().getCoprocessors();
1322         Iterator<String> iterator = regionCoprocessors.iterator();
1323         while (iterator.hasNext()) {
1324           serverLoad.addCoprocessors(coprocessorBuilder.setName(iterator.next()).build());
1325         }
1326       }
1327       serverLoad.addRegionLoads(createRegionLoad(region, regionLoadBldr, regionSpecifier));
1328       for (String coprocessor : getWAL(region.getRegionInfo()).getCoprocessorHost()
1329           .getCoprocessors()) {
1330         serverLoad.addCoprocessors(coprocessorBuilder.setName(coprocessor).build());
1331       }
1332     }
1333     serverLoad.setReportStartTime(reportStartTime);
1334     serverLoad.setReportEndTime(reportEndTime);
1335     if (this.infoServer != null) {
1336       serverLoad.setInfoServerPort(this.infoServer.getPort());
1337     } else {
1338       serverLoad.setInfoServerPort(-1);
1339     }
1340 
1341     // for the replicationLoad purpose. Only need to get from one service
1342     // either source or sink will get the same info
1343     ReplicationSourceService rsources = getReplicationSourceService();
1344 
1345     if (rsources != null) {
1346       // always refresh first to get the latest value
1347       ReplicationLoad rLoad = rsources.refreshAndGetReplicationLoad();
1348       if (rLoad != null) {
1349         serverLoad.setReplLoadSink(rLoad.getReplicationLoadSink());
1350         for (ClusterStatusProtos.ReplicationLoadSource rLS : rLoad.getReplicationLoadSourceList()) {
1351           serverLoad.addReplLoadSource(rLS);
1352         }
1353       }
1354     }
1355 
1356     return serverLoad.build();
1357   }
1358 
1359   String getOnlineRegionsAsPrintableString() {
1360     StringBuilder sb = new StringBuilder();
1361     for (Region r: this.onlineRegions.values()) {
1362       if (sb.length() > 0) sb.append(", ");
1363       sb.append(r.getRegionInfo().getEncodedName());
1364     }
1365     return sb.toString();
1366   }
1367 
1368   /**
1369    * Wait on regions close.
1370    */
1371   private void waitOnAllRegionsToClose(final boolean abort) {
1372     // Wait till all regions are closed before going out.
1373     int lastCount = -1;
1374     long previousLogTime = 0;
1375     Set<String> closedRegions = new HashSet<String>();
1376     boolean interrupted = false;
1377     try {
1378       while (!isOnlineRegionsEmpty()) {
1379         int count = getNumberOfOnlineRegions();
1380         // Only print a message if the count of regions has changed.
1381         if (count != lastCount) {
1382           // Log every second at most
1383           if (System.currentTimeMillis() > (previousLogTime + 1000)) {
1384             previousLogTime = System.currentTimeMillis();
1385             lastCount = count;
1386             LOG.info("Waiting on " + count + " regions to close");
1387             // Only print out regions still closing if a small number else will
1388             // swamp the log.
1389             if (count < 10 && LOG.isDebugEnabled()) {
1390               LOG.debug(this.onlineRegions);
1391             }
1392           }
1393         }
1394         // Ensure all user regions have been sent a close. Use this to
1395         // protect against the case where an open comes in after we start the
1396         // iterator of onlineRegions to close all user regions.
1397         for (Map.Entry<String, Region> e : this.onlineRegions.entrySet()) {
1398           HRegionInfo hri = e.getValue().getRegionInfo();
1399           if (!this.regionsInTransitionInRS.containsKey(hri.getEncodedNameAsBytes())
1400               && !closedRegions.contains(hri.getEncodedName())) {
1401             closedRegions.add(hri.getEncodedName());
1402             // Don't update zk with this close transition; pass false.
1403             closeRegionIgnoreErrors(hri, abort);
1404               }
1405         }
1406         // No regions in RIT, we could stop waiting now.
1407         if (this.regionsInTransitionInRS.isEmpty()) {
1408           if (!isOnlineRegionsEmpty()) {
1409             LOG.info("We were exiting though online regions are not empty," +
1410                 " because some regions failed closing");
1411           }
1412           break;
1413         }
1414         if (sleep(200)) {
1415           interrupted = true;
1416         }
1417       }
1418     } finally {
1419       if (interrupted) {
1420         Thread.currentThread().interrupt();
1421       }
1422     }
1423   }
1424 
1425   private boolean sleep(long millis) {
1426     boolean interrupted = false;
1427     try {
1428       Thread.sleep(millis);
1429     } catch (InterruptedException e) {
1430       LOG.warn("Interrupted while sleeping");
1431       interrupted = true;
1432     }
1433     return interrupted;
1434   }
1435 
1436   private void shutdownWAL(final boolean close) {
1437     if (this.walFactory != null) {
1438       try {
1439         if (close) {
1440           walFactory.close();
1441         } else {
1442           walFactory.shutdown();
1443         }
1444       } catch (Throwable e) {
1445         e = RemoteExceptionHandler.checkThrowable(e);
1446         LOG.error("Shutdown / close of WAL failed: " + e);
1447         LOG.debug("Shutdown / close exception details:", e);
1448       }
1449     }
1450   }
1451 
1452   /*
1453    * Run init. Sets up wal and starts up all server threads.
1454    *
1455    * @param c Extra configuration.
1456    */
1457   protected void handleReportForDutyResponse(final RegionServerStartupResponse c)
1458   throws IOException {
1459     try {
1460       boolean updateRootDir = false;
1461       for (NameStringPair e : c.getMapEntriesList()) {
1462         String key = e.getName();
1463         // The hostname the master sees us as.
1464         if (key.equals(HConstants.KEY_FOR_HOSTNAME_SEEN_BY_MASTER)) {
1465           String hostnameFromMasterPOV = e.getValue();
1466           this.serverName = ServerName.valueOf(hostnameFromMasterPOV,
1467             rpcServices.isa.getPort(), this.startcode);
1468           if (shouldUseThisHostnameInstead() &&
1469               !hostnameFromMasterPOV.equals(useThisHostnameInstead)) {
1470             String msg = "Master passed us a different hostname to use; was=" +
1471                 this.useThisHostnameInstead + ", but now=" + hostnameFromMasterPOV;
1472             LOG.error(msg);
1473             throw new IOException(msg);
1474           }
1475           if (!shouldUseThisHostnameInstead() &&
1476               !hostnameFromMasterPOV.equals(rpcServices.isa.getHostName())) {
1477             String msg = "Master passed us a different hostname to use; was=" +
1478                 rpcServices.isa.getHostName() + ", but now=" + hostnameFromMasterPOV;
1479             LOG.error(msg);
1480           }
1481           continue;
1482         }
1483 
1484         String value = e.getValue();
1485         if (key.equals(HConstants.HBASE_DIR)) {
1486           if (value != null && !value.equals(conf.get(HConstants.HBASE_DIR))) {
1487             updateRootDir = true;
1488           }
1489         }
1490 
1491         if (LOG.isDebugEnabled()) {
1492           LOG.debug("Config from master: " + key + "=" + value);
1493         }
1494         this.conf.set(key, value);
1495       }
1496 
1497       if (updateRootDir) {
1498         // initialize file system by the config fs.defaultFS and hbase.rootdir from master
1499         initializeFileSystem();
1500       }
1501 
1502       // hack! Maps DFSClient => RegionServer for logs.  HDFS made this
1503       // config param for task trackers, but we can piggyback off of it.
1504       if (this.conf.get("mapreduce.task.attempt.id") == null) {
1505         this.conf.set("mapreduce.task.attempt.id", "hb_rs_" +
1506           this.serverName.toString());
1507       }
1508 
1509       // Save it in a file, this will allow to see if we crash
1510       ZNodeClearer.writeMyEphemeralNodeOnDisk(getMyEphemeralNodePath());
1511 
1512       this.cacheConfig = new CacheConfig(conf);
1513       this.walFactory = setupWALAndReplication();
1514       // Init in here rather than in constructor after thread name has been set
1515       this.metricsTable = new MetricsTable(new MetricsTableWrapperAggregateImpl(this));
1516       this.metricsRegionServer = new MetricsRegionServer(getConfiguration(),
1517         new MetricsRegionServerWrapperImpl(this), metricsTable);
1518       // Now that we have a metrics source, start the pause monitor
1519       this.pauseMonitor = new JvmPauseMonitor(conf, getMetrics().getMetricsSource());
1520       pauseMonitor.start();
1521 
1522       startServiceThreads();
1523       startHeapMemoryManager();
1524       LOG.info("Serving as " + this.serverName +
1525         ", RpcServer on " + rpcServices.isa +
1526         ", sessionid=0x" +
1527         Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()));
1528 
1529       // Wake up anyone waiting for this server to online
1530       synchronized (online) {
1531         online.set(true);
1532         online.notifyAll();
1533       }
1534     } catch (Throwable e) {
1535       stop("Failed initialization");
1536       throw convertThrowableToIOE(cleanup(e, "Failed init"),
1537           "Region server startup failed");
1538     } finally {
1539       sleeper.skipSleepCycle();
1540     }
1541   }
1542 
1543   private void startHeapMemoryManager() {
1544     this.hMemManager = HeapMemoryManager.create(this.conf, this.cacheFlusher, this);
1545     if (this.hMemManager != null) {
1546       this.hMemManager.start(getChoreService());
1547     }
1548   }
1549 
1550   private void createMyEphemeralNode() throws KeeperException, IOException {
1551     RegionServerInfo.Builder rsInfo = RegionServerInfo.newBuilder();
1552     rsInfo.setInfoPort(infoServer != null ? infoServer.getPort() : -1);
1553     byte[] data = ProtobufUtil.prependPBMagic(rsInfo.build().toByteArray());
1554     ZKUtil.createEphemeralNodeAndWatch(this.zooKeeper,
1555       getMyEphemeralNodePath(), data);
1556   }
1557 
1558   private void deleteMyEphemeralNode() throws KeeperException {
1559     ZKUtil.deleteNode(this.zooKeeper, getMyEphemeralNodePath());
1560   }
1561 
1562   @Override
1563   public RegionServerAccounting getRegionServerAccounting() {
1564     return regionServerAccounting;
1565   }
1566 
1567   @Override
1568   public TableLockManager getTableLockManager() {
1569     return tableLockManager;
1570   }
1571 
1572   /*
1573    * @param r Region to get RegionLoad for.
1574    * @param regionLoadBldr the RegionLoad.Builder, can be null
1575    * @param regionSpecifier the RegionSpecifier.Builder, can be null
1576    * @return RegionLoad instance.
1577    *
1578    * @throws IOException
1579    */
1580   private RegionLoad createRegionLoad(final Region r, RegionLoad.Builder regionLoadBldr,
1581       RegionSpecifier.Builder regionSpecifier) throws IOException {
1582     byte[] name = r.getRegionInfo().getRegionName();
1583     int stores = 0;
1584     int storefiles = 0;
1585     int storeUncompressedSizeMB = 0;
1586     int storefileSizeMB = 0;
1587     int memstoreSizeMB = (int) (r.getMemstoreSize() / 1024 / 1024);
1588     int storefileIndexSizeMB = 0;
1589     int rootIndexSizeKB = 0;
1590     int totalStaticIndexSizeKB = 0;
1591     int totalStaticBloomSizeKB = 0;
1592     long totalCompactingKVs = 0;
1593     long currentCompactedKVs = 0;
1594     List<Store> storeList = r.getStores();
1595     stores += storeList.size();
1596     for (Store store : storeList) {
1597       storefiles += store.getStorefilesCount();
1598       storeUncompressedSizeMB += (int) (store.getStoreSizeUncompressed() / 1024 / 1024);
1599       storefileSizeMB += (int) (store.getStorefilesSize() / 1024 / 1024);
1600       storefileIndexSizeMB += (int) (store.getStorefilesIndexSize() / 1024 / 1024);
1601       CompactionProgress progress = store.getCompactionProgress();
1602       if (progress != null) {
1603         totalCompactingKVs += progress.totalCompactingKVs;
1604         currentCompactedKVs += progress.currentCompactedKVs;
1605       }
1606       rootIndexSizeKB += (int) (store.getStorefilesIndexSize() / 1024);
1607       totalStaticIndexSizeKB += (int) (store.getTotalStaticIndexSize() / 1024);
1608       totalStaticBloomSizeKB += (int) (store.getTotalStaticBloomSize() / 1024);
1609     }
1610 
1611     float dataLocality =
1612         r.getHDFSBlocksDistribution().getBlockLocalityIndex(serverName.getHostname());
1613     if (regionLoadBldr == null) {
1614       regionLoadBldr = RegionLoad.newBuilder();
1615     }
1616     if (regionSpecifier == null) {
1617       regionSpecifier = RegionSpecifier.newBuilder();
1618     }
1619     regionSpecifier.setType(RegionSpecifierType.REGION_NAME);
1620     regionSpecifier.setValue(ByteStringer.wrap(name));
1621     regionLoadBldr.setRegionSpecifier(regionSpecifier.build())
1622       .setStores(stores)
1623       .setStorefiles(storefiles)
1624       .setStoreUncompressedSizeMB(storeUncompressedSizeMB)
1625       .setStorefileSizeMB(storefileSizeMB)
1626       .setMemstoreSizeMB(memstoreSizeMB)
1627       .setStorefileIndexSizeMB(storefileIndexSizeMB)
1628       .setRootIndexSizeKB(rootIndexSizeKB)
1629       .setTotalStaticIndexSizeKB(totalStaticIndexSizeKB)
1630       .setTotalStaticBloomSizeKB(totalStaticBloomSizeKB)
1631       .setReadRequestsCount(r.getReadRequestsCount())
1632       .setWriteRequestsCount(r.getWriteRequestsCount())
1633       .setTotalCompactingKVs(totalCompactingKVs)
1634       .setCurrentCompactedKVs(currentCompactedKVs)
1635       .setDataLocality(dataLocality)
1636       .setLastMajorCompactionTs(r.getOldestHfileTs(true));
1637     ((HRegion)r).setCompleteSequenceId(regionLoadBldr);
1638 
1639     return regionLoadBldr.build();
1640   }
1641 
1642   /**
1643    * @param encodedRegionName
1644    * @return An instance of RegionLoad.
1645    */
1646   public RegionLoad createRegionLoad(final String encodedRegionName) throws IOException {
1647     Region r = onlineRegions.get(encodedRegionName);
1648     return r != null ? createRegionLoad(r, null, null) : null;
1649   }
1650 
1651   /*
1652    * Inner class that runs on a long period checking if regions need compaction.
1653    */
1654   private static class CompactionChecker extends ScheduledChore {
1655     private final HRegionServer instance;
1656     private final int majorCompactPriority;
1657     private final static int DEFAULT_PRIORITY = Integer.MAX_VALUE;
1658     private long iteration = 0;
1659 
1660     CompactionChecker(final HRegionServer h, final int sleepTime,
1661         final Stoppable stopper) {
1662       super("CompactionChecker", stopper, sleepTime);
1663       this.instance = h;
1664       LOG.info(this.getName() + " runs every " + StringUtils.formatTime(sleepTime));
1665 
1666       /* MajorCompactPriority is configurable.
1667        * If not set, the compaction will use default priority.
1668        */
1669       this.majorCompactPriority = this.instance.conf.
1670         getInt("hbase.regionserver.compactionChecker.majorCompactPriority",
1671         DEFAULT_PRIORITY);
1672     }
1673 
1674     @Override
1675     protected void chore() {
1676       for (Region r : this.instance.onlineRegions.values()) {
1677         if (r == null)
1678           continue;
1679         for (Store s : r.getStores()) {
1680           try {
1681             long multiplier = s.getCompactionCheckMultiplier();
1682             assert multiplier > 0;
1683             if (iteration % multiplier != 0) continue;
1684             if (s.needsCompaction()) {
1685               // Queue a compaction. Will recognize if major is needed.
1686               this.instance.compactSplitThread.requestSystemCompaction(r, s, getName()
1687                   + " requests compaction");
1688             } else if (s.isMajorCompaction()) {
1689               if (majorCompactPriority == DEFAULT_PRIORITY
1690                   || majorCompactPriority > ((HRegion)r).getCompactPriority()) {
1691                 this.instance.compactSplitThread.requestCompaction(r, s, getName()
1692                     + " requests major compaction; use default priority", null);
1693               } else {
1694                 this.instance.compactSplitThread.requestCompaction(r, s, getName()
1695                     + " requests major compaction; use configured priority",
1696                   this.majorCompactPriority, null, null);
1697               }
1698             }
1699           } catch (IOException e) {
1700             LOG.warn("Failed major compaction check on " + r, e);
1701           }
1702         }
1703       }
1704       iteration = (iteration == Long.MAX_VALUE) ? 0 : (iteration + 1);
1705     }
1706   }
1707 
1708   static class PeriodicMemstoreFlusher extends ScheduledChore {
1709     final HRegionServer server;
1710     final static int RANGE_OF_DELAY = 20000; //millisec
1711     final static int MIN_DELAY_TIME = 3000; //millisec
1712     public PeriodicMemstoreFlusher(int cacheFlushInterval, final HRegionServer server) {
1713       super(server.getServerName() + "-MemstoreFlusherChore", server, cacheFlushInterval);
1714       this.server = server;
1715     }
1716 
1717     @Override
1718     protected void chore() {
1719       for (Region r : this.server.onlineRegions.values()) {
1720         if (r == null)
1721           continue;
1722         if (((HRegion)r).shouldFlush()) {
1723           FlushRequester requester = server.getFlushRequester();
1724           if (requester != null) {
1725             long randomDelay = RandomUtils.nextInt(RANGE_OF_DELAY) + MIN_DELAY_TIME;
1726             LOG.info(getName() + " requesting flush for region " +
1727               r.getRegionInfo().getRegionNameAsString() + " after a delay of " + randomDelay);
1728             //Throttle the flushes by putting a delay. If we don't throttle, and there
1729             //is a balanced write-load on the regions in a table, we might end up
1730             //overwhelming the filesystem with too many flushes at once.
1731             requester.requestDelayedFlush(r, randomDelay, false);
1732           }
1733         }
1734       }
1735     }
1736   }
1737 
1738   /**
1739    * Report the status of the server. A server is online once all the startup is
1740    * completed (setting up filesystem, starting service threads, etc.). This
1741    * method is designed mostly to be useful in tests.
1742    *
1743    * @return true if online, false if not.
1744    */
1745   public boolean isOnline() {
1746     return online.get();
1747   }
1748 
1749   /**
1750    * Setup WAL log and replication if enabled.
1751    * Replication setup is done in here because it wants to be hooked up to WAL.
1752    * @return A WAL instance.
1753    * @throws IOException
1754    */
1755   private WALFactory setupWALAndReplication() throws IOException {
1756     // TODO Replication make assumptions here based on the default filesystem impl
1757     final Path oldLogDir = new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
1758     final String logName = DefaultWALProvider.getWALDirectoryName(this.serverName.toString());
1759 
1760     Path logDir = new Path(walRootDir, logName);
1761     if (LOG.isDebugEnabled()) LOG.debug("logDir=" + logDir);
1762     if (this.walFs.exists(logDir)) {
1763       throw new RegionServerRunningException("Region server has already " +
1764         "created directory at " + this.serverName.toString());
1765     }
1766 
1767     // Instantiate replication manager if replication enabled.  Pass it the
1768     // log directories.
1769     createNewReplicationInstance(conf, this, this.walFs, logDir, oldLogDir);
1770 
1771     // listeners the wal factory will add to wals it creates.
1772     final List<WALActionsListener> listeners = new ArrayList<WALActionsListener>();
1773     listeners.add(new MetricsWAL());
1774     /* loader = new BulkLoadHandler(getConnection());
1775     listeners.add(loader); */
1776     if (this.replicationSourceHandler != null &&
1777         this.replicationSourceHandler.getWALActionsListener() != null) {
1778       // Replication handler is an implementation of WALActionsListener.
1779       listeners.add(this.replicationSourceHandler.getWALActionsListener());
1780     }
1781 
1782     return new WALFactory(conf, listeners, serverName.toString());
1783   }
1784 
1785   /**
1786    * We initialize the roller for the wal that handles meta lazily
1787    * since we don't know if this regionserver will handle it. All calls to
1788    * this method return a reference to the that same roller. As newly referenced
1789    * meta regions are brought online, they will be offered to the roller for maintenance.
1790    * As a part of that registration process, the roller will add itself as a
1791    * listener on the wal.
1792    */
1793   protected LogRoller ensureMetaWALRoller() {
1794     // Using a tmp log roller to ensure metaLogRoller is alive once it is not
1795     // null
1796     LogRoller roller = metawalRoller.get();
1797     if (null == roller) {
1798       LogRoller tmpLogRoller = new LogRoller(this, this);
1799       String n = Thread.currentThread().getName();
1800       Threads.setDaemonThreadRunning(tmpLogRoller.getThread(),
1801           n + "-MetaLogRoller", uncaughtExceptionHandler);
1802       if (metawalRoller.compareAndSet(null, tmpLogRoller)) {
1803         roller = tmpLogRoller;
1804       } else {
1805         // Another thread won starting the roller
1806         Threads.shutdown(tmpLogRoller.getThread());
1807         roller = metawalRoller.get();
1808       }
1809     }
1810     return roller;
1811   }
1812 
1813   public MetricsRegionServer getRegionServerMetrics() {
1814     return this.metricsRegionServer;
1815   }
1816 
1817   /**
1818    * @return Master address tracker instance.
1819    */
1820   public MasterAddressTracker getMasterAddressTracker() {
1821     return this.masterAddressTracker;
1822   }
1823 
1824   /*
1825    * Start maintenance Threads, Server, Worker and lease checker threads.
1826    * Install an UncaughtExceptionHandler that calls abort of RegionServer if we
1827    * get an unhandled exception. We cannot set the handler on all threads.
1828    * Server's internal Listener thread is off limits. For Server, if an OOME, it
1829    * waits a while then retries. Meantime, a flush or a compaction that tries to
1830    * run should trigger same critical condition and the shutdown will run. On
1831    * its way out, this server will shut down Server. Leases are sort of
1832    * inbetween. It has an internal thread that while it inherits from Chore, it
1833    * keeps its own internal stop mechanism so needs to be stopped by this
1834    * hosting server. Worker logs the exception and exits.
1835    */
1836   private void startServiceThreads() throws IOException {
1837     // Start executor services
1838     this.service.startExecutorService(ExecutorType.RS_OPEN_REGION,
1839       conf.getInt("hbase.regionserver.executor.openregion.threads", 3));
1840     this.service.startExecutorService(ExecutorType.RS_OPEN_META,
1841       conf.getInt("hbase.regionserver.executor.openmeta.threads", 1));
1842     this.service.startExecutorService(ExecutorType.RS_OPEN_PRIORITY_REGION,
1843       conf.getInt("hbase.regionserver.executor.openpriorityregion.threads", 3));
1844     this.service.startExecutorService(ExecutorType.RS_CLOSE_REGION,
1845       conf.getInt("hbase.regionserver.executor.closeregion.threads", 3));
1846     this.service.startExecutorService(ExecutorType.RS_CLOSE_META,
1847       conf.getInt("hbase.regionserver.executor.closemeta.threads", 1));
1848     if (conf.getBoolean(StoreScanner.STORESCANNER_PARALLEL_SEEK_ENABLE, false)) {
1849       this.service.startExecutorService(ExecutorType.RS_PARALLEL_SEEK,
1850         conf.getInt("hbase.storescanner.parallel.seek.threads", 10));
1851     }
1852     this.service.startExecutorService(ExecutorType.RS_LOG_REPLAY_OPS, conf.getInt(
1853        "hbase.regionserver.wal.max.splitters", SplitLogWorkerCoordination.DEFAULT_MAX_SPLITTERS));
1854 
1855     if (ServerRegionReplicaUtil.isRegionReplicaWaitForPrimaryFlushEnabled(conf)) {
1856       this.service.startExecutorService(ExecutorType.RS_REGION_REPLICA_FLUSH_OPS,
1857         conf.getInt("hbase.regionserver.region.replica.flusher.threads",
1858           conf.getInt("hbase.regionserver.executor.openregion.threads", 3)));
1859     }
1860 
1861     Threads.setDaemonThreadRunning(this.walRoller.getThread(), getName() + ".logRoller",
1862         uncaughtExceptionHandler);
1863     this.cacheFlusher.start(uncaughtExceptionHandler);
1864 
1865     if (this.compactionChecker != null) choreService.scheduleChore(compactionChecker);
1866     if (this.periodicFlusher != null) choreService.scheduleChore(periodicFlusher);
1867     if (this.healthCheckChore != null) choreService.scheduleChore(healthCheckChore);
1868     if (this.nonceManagerChore != null) choreService.scheduleChore(nonceManagerChore);
1869     if (this.storefileRefresher != null) choreService.scheduleChore(storefileRefresher);
1870     if (this.movedRegionsCleaner != null) choreService.scheduleChore(movedRegionsCleaner);
1871     if (this.fsUtilizationChore != null) choreService.scheduleChore(fsUtilizationChore);
1872 
1873     // Leases is not a Thread. Internally it runs a daemon thread. If it gets
1874     // an unhandled exception, it will just exit.
1875     Threads.setDaemonThreadRunning(this.leases.getThread(), getName() + ".leaseChecker",
1876       uncaughtExceptionHandler);
1877 
1878     if (this.replicationSourceHandler == this.replicationSinkHandler &&
1879         this.replicationSourceHandler != null) {
1880       this.replicationSourceHandler.startReplicationService();
1881     } else {
1882       if (this.replicationSourceHandler != null) {
1883         this.replicationSourceHandler.startReplicationService();
1884       }
1885       if (this.replicationSinkHandler != null) {
1886         this.replicationSinkHandler.startReplicationService();
1887       }
1888     }
1889 
1890     // Create the log splitting worker and start it
1891     // set a smaller retries to fast fail otherwise splitlogworker could be blocked for
1892     // quite a while inside HConnection layer. The worker won't be available for other
1893     // tasks even after current task is preempted after a split task times out.
1894     Configuration sinkConf = HBaseConfiguration.create(conf);
1895     sinkConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
1896       conf.getInt("hbase.log.replay.retries.number", 8)); // 8 retries take about 23 seconds
1897     sinkConf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY,
1898       conf.getInt("hbase.log.replay.rpc.timeout", 30000)); // default 30 seconds
1899     sinkConf.setInt("hbase.client.serverside.retries.multiplier", 1);
1900     this.splitLogWorker = new SplitLogWorker(this, sinkConf, this, this, walFactory);
1901     splitLogWorker.start();
1902   }
1903 
1904   /**
1905    * Puts up the webui.
1906    * @return Returns final port -- maybe different from what we started with.
1907    * @throws IOException
1908    */
1909   private int putUpWebUI() throws IOException {
1910     int port = this.conf.getInt(HConstants.REGIONSERVER_INFO_PORT,
1911       HConstants.DEFAULT_REGIONSERVER_INFOPORT);
1912     String addr = this.conf.get("hbase.regionserver.info.bindAddress", "0.0.0.0");
1913 
1914     if(this instanceof HMaster) {
1915       port = conf.getInt(HConstants.MASTER_INFO_PORT,
1916           HConstants.DEFAULT_MASTER_INFOPORT);
1917       addr = this.conf.get("hbase.master.info.bindAddress", "0.0.0.0");
1918     }
1919     // -1 is for disabling info server
1920     if (port < 0) return port;
1921 
1922     if (!Addressing.isLocalAddress(InetAddress.getByName(addr))) {
1923       String msg =
1924           "Failed to start http info server. Address " + addr
1925               + " does not belong to this host. Correct configuration parameter: "
1926               + "hbase.regionserver.info.bindAddress";
1927       LOG.error(msg);
1928       throw new IOException(msg);
1929     }
1930     // check if auto port bind enabled
1931     boolean auto = this.conf.getBoolean(HConstants.REGIONSERVER_INFO_PORT_AUTO,
1932         false);
1933     while (true) {
1934       try {
1935         this.infoServer = new InfoServer(getProcessName(), addr, port, false, this.conf);
1936         infoServer.addServlet("dump", "/dump", getDumpServlet());
1937         configureInfoServer();
1938         this.infoServer.start();
1939         break;
1940       } catch (BindException e) {
1941         if (!auto) {
1942           // auto bind disabled throw BindException
1943           LOG.error("Failed binding http info server to port: " + port);
1944           throw e;
1945         }
1946         // auto bind enabled, try to use another port
1947         LOG.info("Failed binding http info server to port: " + port);
1948         port++;
1949       }
1950     }
1951     port = this.infoServer.getPort();
1952     conf.setInt(HConstants.REGIONSERVER_INFO_PORT, port);
1953     int masterInfoPort = conf.getInt(HConstants.MASTER_INFO_PORT,
1954       HConstants.DEFAULT_MASTER_INFOPORT);
1955     conf.setInt("hbase.master.info.port.orig", masterInfoPort);
1956     conf.setInt(HConstants.MASTER_INFO_PORT, port);
1957     return port;
1958   }
1959 
1960   /*
1961    * Verify that server is healthy
1962    */
1963   private boolean isHealthy() {
1964     if (!fsOk) {
1965       // File system problem
1966       return false;
1967     }
1968     // Verify that all threads are alive
1969     if (!(leases.isAlive()
1970         && cacheFlusher.isAlive() && walRoller.isAlive()
1971         && this.compactionChecker.isScheduled()
1972         && this.periodicFlusher.isScheduled())) {
1973       stop("One or more threads are no longer alive -- stop");
1974       return false;
1975     }
1976     final LogRoller metawalRoller = this.metawalRoller.get();
1977     if (metawalRoller != null && !metawalRoller.isAlive()) {
1978       stop("Meta WAL roller thread is no longer alive -- stop");
1979       return false;
1980     }
1981     return true;
1982   }
1983 
1984   private static final byte[] UNSPECIFIED_REGION = new byte[]{};
1985 
1986   @Override
1987   public WAL getWAL(HRegionInfo regionInfo) throws IOException {
1988     WAL wal;
1989     LogRoller roller = walRoller;
1990     //_ROOT_ and hbase:meta regions have separate WAL.
1991     if (regionInfo != null && regionInfo.isMetaTable() &&
1992         regionInfo.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
1993       roller = ensureMetaWALRoller();
1994       wal = walFactory.getMetaWAL(regionInfo.getEncodedNameAsBytes());
1995     } else if (regionInfo == null) {
1996       wal = walFactory.getWAL(UNSPECIFIED_REGION);
1997     } else {
1998       wal = walFactory.getWAL(regionInfo.getEncodedNameAsBytes());
1999     }
2000     roller.addWAL(wal);
2001     return wal;
2002   }
2003 
2004   @Override
2005   public ClusterConnection getConnection() {
2006     return this.clusterConnection;
2007   }
2008 
2009   @Override
2010   public MetaTableLocator getMetaTableLocator() {
2011     return this.metaTableLocator;
2012   }
2013 
2014   @Override
2015   public void stop(final String msg) {
2016     if (!this.stopped) {
2017       try {
2018         if (this.rsHost != null) {
2019           this.rsHost.preStop(msg);
2020         }
2021         this.stopped = true;
2022         LOG.info("STOPPED: " + msg);
2023         // Wakes run() if it is sleeping
2024         sleeper.skipSleepCycle();
2025       } catch (IOException exp) {
2026         LOG.warn("The region server did not stop", exp);
2027       }
2028     }
2029   }
2030 
2031   public void waitForServerOnline(){
2032     while (!isStopped() && !isOnline()) {
2033       synchronized (online) {
2034         try {
2035           online.wait(msgInterval);
2036         } catch (InterruptedException ie) {
2037           Thread.currentThread().interrupt();
2038           break;
2039         }
2040       }
2041     }
2042   }
2043 
2044   @Override
2045   public void postOpenDeployTasks(final Region r) throws KeeperException, IOException {
2046     postOpenDeployTasks(new PostOpenDeployContext(r, -1));
2047   }
2048 
2049   @Override
2050   public void postOpenDeployTasks(final PostOpenDeployContext context)
2051       throws KeeperException, IOException {
2052     Region r = context.getRegion();
2053     long masterSystemTime = context.getMasterSystemTime();
2054     Preconditions.checkArgument(r instanceof HRegion, "r must be an HRegion");
2055     rpcServices.checkOpen();
2056     LOG.info("Post open deploy tasks for " + r.getRegionInfo().getRegionNameAsString());
2057     // Do checks to see if we need to compact (references or too many files)
2058     for (Store s : r.getStores()) {
2059       if (s.hasReferences() || s.needsCompaction()) {
2060        this.compactSplitThread.requestSystemCompaction(r, s, "Opening Region");
2061       }
2062     }
2063     long openSeqNum = r.getOpenSeqNum();
2064     if (openSeqNum == HConstants.NO_SEQNUM) {
2065       // If we opened a region, we should have read some sequence number from it.
2066       LOG.error("No sequence number found when opening " +
2067         r.getRegionInfo().getRegionNameAsString());
2068       openSeqNum = 0;
2069     }
2070 
2071     // Update flushed sequence id of a recovering region in ZK
2072     updateRecoveringRegionLastFlushedSequenceId(r);
2073 
2074     // Update ZK, or META
2075     if (r.getRegionInfo().isMetaRegion()) {
2076       MetaTableLocator.setMetaLocation(getZooKeeper(), serverName, r.getRegionInfo().getReplicaId(),
2077          State.OPEN);
2078     } else if (useZKForAssignment) {
2079       MetaTableAccessor.updateRegionLocation(getConnection(), r.getRegionInfo(),
2080         this.serverName, openSeqNum, masterSystemTime);
2081     }
2082     if (!useZKForAssignment && !reportRegionStateTransition(new RegionStateTransitionContext(
2083         TransitionCode.OPENED, openSeqNum, masterSystemTime, r.getRegionInfo()))) {
2084       throw new IOException("Failed to report opened region to master: "
2085         + r.getRegionInfo().getRegionNameAsString());
2086     }
2087 
2088     triggerFlushInPrimaryRegion((HRegion)r);
2089 
2090     LOG.debug("Finished post open deploy task for " + r.getRegionInfo().getRegionNameAsString());
2091   }
2092 
2093   @Override
2094   public boolean reportRegionStateTransition(TransitionCode code, HRegionInfo... hris) {
2095     return reportRegionStateTransition(code, HConstants.NO_SEQNUM, hris);
2096   }
2097 
2098   @Override
2099   public boolean reportRegionStateTransition(
2100       TransitionCode code, long openSeqNum, HRegionInfo... hris) {
2101     return reportRegionStateTransition(
2102       new RegionStateTransitionContext(code, HConstants.NO_SEQNUM, -1, hris));
2103   }
2104 
2105   @Override
2106   public boolean reportRegionStateTransition(final RegionStateTransitionContext context) {
2107     TransitionCode code = context.getCode();
2108     long openSeqNum = context.getOpenSeqNum();
2109     long masterSystemTime = context.getMasterSystemTime();
2110     HRegionInfo[] hris = context.getHris();
2111 
2112     ReportRegionStateTransitionRequest.Builder builder =
2113       ReportRegionStateTransitionRequest.newBuilder();
2114     builder.setServer(ProtobufUtil.toServerName(serverName));
2115     RegionStateTransition.Builder transition = builder.addTransitionBuilder();
2116     transition.setTransitionCode(code);
2117     if (code == TransitionCode.OPENED && openSeqNum >= 0) {
2118       transition.setOpenSeqNum(openSeqNum);
2119     }
2120     for (HRegionInfo hri: hris) {
2121       transition.addRegionInfo(HRegionInfo.convert(hri));
2122     }
2123     ReportRegionStateTransitionRequest request = builder.build();
2124     while (keepLooping()) {
2125       RegionServerStatusService.BlockingInterface rss = rssStub;
2126       try {
2127         if (rss == null) {
2128           createRegionServerStatusStub();
2129           continue;
2130         }
2131         ReportRegionStateTransitionResponse response =
2132           rss.reportRegionStateTransition(null, request);
2133         if (response.hasErrorMessage()) {
2134           LOG.info("Failed to transition " + hris[0]
2135             + " to " + code + ": " + response.getErrorMessage());
2136           return false;
2137         }
2138         return true;
2139       } catch (ServiceException se) {
2140         IOException ioe = ProtobufUtil.getRemoteException(se);
2141         LOG.info("Failed to report region transition, will retry", ioe);
2142         if (rssStub == rss) {
2143           rssStub = null;
2144         }
2145       }
2146     }
2147     return false;
2148   }
2149 
2150   /**
2151    * Trigger a flush in the primary region replica if this region is a secondary replica. Does not
2152    * block this thread. See RegionReplicaFlushHandler for details.
2153    */
2154   void triggerFlushInPrimaryRegion(final HRegion region) {
2155     if (ServerRegionReplicaUtil.isDefaultReplica(region.getRegionInfo())) {
2156       return;
2157     }
2158     if (!ServerRegionReplicaUtil.isRegionReplicaReplicationEnabled(region.conf) ||
2159         !ServerRegionReplicaUtil.isRegionReplicaWaitForPrimaryFlushEnabled(
2160           region.conf)) {
2161       region.setReadsEnabled(true);
2162       return;
2163     }
2164 
2165     region.setReadsEnabled(false); // disable reads before marking the region as opened.
2166     // RegionReplicaFlushHandler might reset this.
2167 
2168     // submit it to be handled by one of the handlers so that we do not block OpenRegionHandler
2169     this.service.submit(
2170       new RegionReplicaFlushHandler(this, clusterConnection,
2171         rpcRetryingCallerFactory, rpcControllerFactory, operationTimeout, region));
2172   }
2173 
2174   @Override
2175   public RpcServerInterface getRpcServer() {
2176     return rpcServices.rpcServer;
2177   }
2178 
2179   @VisibleForTesting
2180   public RSRpcServices getRSRpcServices() {
2181     return rpcServices;
2182   }
2183 
2184   /**
2185    * Cause the server to exit without closing the regions it is serving, the log
2186    * it is using and without notifying the master. Used unit testing and on
2187    * catastrophic events such as HDFS is yanked out from under hbase or we OOME.
2188    *
2189    * @param reason
2190    *          the reason we are aborting
2191    * @param cause
2192    *          the exception that caused the abort, or null
2193    */
2194   @Override
2195   public void abort(String reason, Throwable cause) {
2196     String msg = "ABORTING region server " + this + ": " + reason;
2197     if (cause != null) {
2198       LOG.fatal(msg, cause);
2199     } else {
2200       LOG.fatal(msg);
2201     }
2202     this.abortRequested = true;
2203     // HBASE-4014: show list of coprocessors that were loaded to help debug
2204     // regionserver crashes.Note that we're implicitly using
2205     // java.util.HashSet's toString() method to print the coprocessor names.
2206     LOG.fatal("RegionServer abort: loaded coprocessors are: " +
2207         CoprocessorHost.getLoadedCoprocessors());
2208     // Try and dump metrics if abort -- might give clue as to how fatal came about....
2209     try {
2210       LOG.info("Dump of metrics as JSON on abort: " + JSONBean.dumpRegionServerMetrics());
2211     } catch (MalformedObjectNameException | IOException e) {
2212       LOG.warn("Failed dumping metrics", e);
2213     }
2214 
2215     // Do our best to report our abort to the master, but this may not work
2216     try {
2217       if (cause != null) {
2218         msg += "\nCause:\n" + StringUtils.stringifyException(cause);
2219       }
2220       // Report to the master but only if we have already registered with the master.
2221       if (rssStub != null && this.serverName != null) {
2222         ReportRSFatalErrorRequest.Builder builder =
2223           ReportRSFatalErrorRequest.newBuilder();
2224         ServerName sn =
2225           ServerName.parseVersionedServerName(this.serverName.getVersionedBytes());
2226         builder.setServer(ProtobufUtil.toServerName(sn));
2227         builder.setErrorMessage(msg);
2228         rssStub.reportRSFatalError(null, builder.build());
2229       }
2230     } catch (Throwable t) {
2231       LOG.warn("Unable to report fatal error to master", t);
2232     }
2233     stop(reason);
2234   }
2235 
2236   /**
2237    * @see HRegionServer#abort(String, Throwable)
2238    */
2239   public void abort(String reason) {
2240     abort(reason, null);
2241   }
2242 
2243   @Override
2244   public boolean isAborted() {
2245     return this.abortRequested;
2246   }
2247 
2248   /*
2249    * Simulate a kill -9 of this server. Exits w/o closing regions or cleaninup
2250    * logs but it does close socket in case want to bring up server on old
2251    * hostname+port immediately.
2252    */
2253   protected void kill() {
2254     this.killed = true;
2255     abort("Simulated kill");
2256   }
2257 
2258   /**
2259    * Called on stop/abort before closing the cluster connection and meta locator.
2260    */
2261   protected void sendShutdownInterrupt() {
2262   }
2263 
2264   /**
2265    * Wait on all threads to finish. Presumption is that all closes and stops
2266    * have already been called.
2267    */
2268   protected void stopServiceThreads() {
2269     //loader.close();
2270     // clean up the scheduled chores
2271     if (this.choreService != null) choreService.shutdown();
2272     if (this.nonceManagerChore != null) nonceManagerChore.cancel(true);
2273     if (this.compactionChecker != null) compactionChecker.cancel(true);
2274     if (this.periodicFlusher != null) periodicFlusher.cancel(true);
2275     if (this.healthCheckChore != null) healthCheckChore.cancel(true);
2276     if (this.storefileRefresher != null) storefileRefresher.cancel(true);
2277     if (this.movedRegionsCleaner != null) movedRegionsCleaner.cancel(true);
2278     if (this.fsUtilizationChore != null) fsUtilizationChore.cancel(true);
2279 
2280     if (this.cacheFlusher != null) {
2281       this.cacheFlusher.join();
2282     }
2283 
2284     if (this.spanReceiverHost != null) {
2285       this.spanReceiverHost.closeReceivers();
2286     }
2287     if (this.walRoller != null) {
2288       Threads.shutdown(this.walRoller.getThread());
2289     }
2290     final LogRoller metawalRoller = this.metawalRoller.get();
2291     if (metawalRoller != null) {
2292       Threads.shutdown(metawalRoller.getThread());
2293     }
2294     if (this.compactSplitThread != null) {
2295       this.compactSplitThread.join();
2296     }
2297     if (this.service != null) this.service.shutdown();
2298     if (this.replicationSourceHandler != null &&
2299         this.replicationSourceHandler == this.replicationSinkHandler) {
2300       this.replicationSourceHandler.stopReplicationService();
2301     } else {
2302       if (this.replicationSourceHandler != null) {
2303         this.replicationSourceHandler.stopReplicationService();
2304       }
2305       if (this.replicationSinkHandler != null) {
2306         this.replicationSinkHandler.stopReplicationService();
2307       }
2308     }
2309   }
2310 
2311   /**
2312    * @return Return the object that implements the replication
2313    * source service.
2314    */
2315   public ReplicationSourceService getReplicationSourceService() {
2316     return replicationSourceHandler;
2317   }
2318 
2319   /**
2320    * @return Return the object that implements the replication
2321    * sink service.
2322    */
2323   ReplicationSinkService getReplicationSinkService() {
2324     return replicationSinkHandler;
2325   }
2326 
2327   /**
2328    * Get the current master from ZooKeeper and open the RPC connection to it.
2329    * To get a fresh connection, the current rssStub must be null.
2330    * Method will block until a master is available. You can break from this
2331    * block by requesting the server stop.
2332    *
2333    * @return master + port, or null if server has been stopped
2334    */
2335   @VisibleForTesting
2336   protected synchronized ServerName createRegionServerStatusStub() {
2337     if (rssStub != null) {
2338       return masterAddressTracker.getMasterAddress();
2339     }
2340     ServerName sn = null;
2341     long previousLogTime = 0;
2342     boolean refresh = false; // for the first time, use cached data
2343     RegionServerStatusService.BlockingInterface intf = null;
2344     boolean interrupted = false;
2345     try {
2346       while (keepLooping()) {
2347         sn = this.masterAddressTracker.getMasterAddress(refresh);
2348         if (sn == null) {
2349           if (!keepLooping()) {
2350             // give up with no connection.
2351             LOG.debug("No master found and cluster is stopped; bailing out");
2352             return null;
2353           }
2354           if (System.currentTimeMillis() > (previousLogTime + 1000)) {
2355             LOG.debug("No master found; retry");
2356             previousLogTime = System.currentTimeMillis();
2357           }
2358           refresh = true; // let's try pull it from ZK directly
2359           if (sleep(200)) {
2360             interrupted = true;
2361           }
2362           continue;
2363         }
2364 
2365         // If we are on the active master, use the shortcut
2366         if (this instanceof HMaster && sn.equals(getServerName())) {
2367           intf = ((HMaster)this).getMasterRpcServices();
2368           break;
2369         }
2370         try {
2371           BlockingRpcChannel channel =
2372             this.rpcClient.createBlockingRpcChannel(sn, userProvider.getCurrent(),
2373               shortOperationTimeout);
2374           intf = RegionServerStatusService.newBlockingStub(channel);
2375           break;
2376         } catch (IOException e) {
2377           if (System.currentTimeMillis() > (previousLogTime + 1000)) {
2378             e = e instanceof RemoteException ?
2379               ((RemoteException)e).unwrapRemoteException() : e;
2380             if (e instanceof ServerNotRunningYetException) {
2381               LOG.info("Master isn't available yet, retrying");
2382             } else {
2383               LOG.warn("Unable to connect to master. Retrying. Error was:", e);
2384             }
2385             previousLogTime = System.currentTimeMillis();
2386           }
2387           if (sleep(200)) {
2388             interrupted = true;
2389           }
2390         }
2391       }
2392     } finally {
2393       if (interrupted) {
2394         Thread.currentThread().interrupt();
2395       }
2396     }
2397     rssStub = intf;
2398     return sn;
2399   }
2400 
2401   /**
2402    * @return True if we should break loop because cluster is going down or
2403    * this server has been stopped or hdfs has gone bad.
2404    */
2405   private boolean keepLooping() {
2406     return !this.stopped && isClusterUp();
2407   }
2408 
2409   /*
2410    * Let the master know we're here Run initialization using parameters passed
2411    * us by the master.
2412    * @return A Map of key/value configurations we got from the Master else
2413    * null if we failed to register.
2414    * @throws IOException
2415    */
2416   private RegionServerStartupResponse reportForDuty() throws IOException {
2417     ServerName masterServerName = createRegionServerStatusStub();
2418     if (masterServerName == null) return null;
2419     RegionServerStartupResponse result = null;
2420     try {
2421       rpcServices.requestCount.set(0);
2422       rpcServices.rpcGetRequestCount.set(0);
2423       rpcServices.rpcScanRequestCount.set(0);
2424       rpcServices.rpcMultiRequestCount.set(0);
2425       rpcServices.rpcMutateRequestCount.set(0);
2426       LOG.info("reportForDuty to master=" + masterServerName + " with port="
2427         + rpcServices.isa.getPort() + ", startcode=" + this.startcode);
2428       long now = EnvironmentEdgeManager.currentTime();
2429       int port = rpcServices.isa.getPort();
2430       RegionServerStartupRequest.Builder request = RegionServerStartupRequest.newBuilder();
2431       if (shouldUseThisHostnameInstead()) {
2432         request.setUseThisHostnameInstead(useThisHostnameInstead);
2433       }
2434       request.setPort(port);
2435       request.setServerStartCode(this.startcode);
2436       request.setServerCurrentTime(now);
2437       result = this.rssStub.regionServerStartup(null, request.build());
2438     } catch (ServiceException se) {
2439       IOException ioe = ProtobufUtil.getRemoteException(se);
2440       if (ioe instanceof ClockOutOfSyncException) {
2441         LOG.fatal("Master rejected startup because clock is out of sync", ioe);
2442         // Re-throw IOE will cause RS to abort
2443         throw ioe;
2444       } else if (ioe instanceof ServerNotRunningYetException) {
2445         LOG.debug("Master is not running yet");
2446       } else {
2447         LOG.warn("error telling master we are up", se);
2448       }
2449       rssStub = null;
2450     }
2451     return result;
2452   }
2453 
2454   @Override
2455   public RegionStoreSequenceIds getLastSequenceId(byte[] encodedRegionName) {
2456     try {
2457       GetLastFlushedSequenceIdRequest req =
2458           RequestConverter.buildGetLastFlushedSequenceIdRequest(encodedRegionName);
2459       RegionServerStatusService.BlockingInterface rss = rssStub;
2460       if (rss == null) { // Try to connect one more time
2461         createRegionServerStatusStub();
2462         rss = rssStub;
2463         if (rss == null) {
2464           // Still no luck, we tried
2465           LOG.warn("Unable to connect to the master to check " + "the last flushed sequence id");
2466           return RegionStoreSequenceIds.newBuilder().setLastFlushedSequenceId(HConstants.NO_SEQNUM)
2467               .build();
2468         }
2469       }
2470       GetLastFlushedSequenceIdResponse resp = rss.getLastFlushedSequenceId(null, req);
2471       return RegionStoreSequenceIds.newBuilder()
2472           .setLastFlushedSequenceId(resp.getLastFlushedSequenceId())
2473           .addAllStoreSequenceId(resp.getStoreLastFlushedSequenceIdList()).build();
2474     } catch (ServiceException e) {
2475       LOG.warn("Unable to connect to the master to check the last flushed sequence id", e);
2476       return RegionStoreSequenceIds.newBuilder().setLastFlushedSequenceId(HConstants.NO_SEQNUM)
2477           .build();
2478     }
2479   }
2480 
2481   /**
2482    * Closes all regions.  Called on our way out.
2483    * Assumes that its not possible for new regions to be added to onlineRegions
2484    * while this method runs.
2485    */
2486   protected void closeAllRegions(final boolean abort) {
2487     closeUserRegions(abort);
2488     closeMetaTableRegions(abort);
2489   }
2490 
2491   /**
2492    * Close meta region if we carry it
2493    * @param abort Whether we're running an abort.
2494    */
2495   void closeMetaTableRegions(final boolean abort) {
2496     Region meta = null;
2497     this.lock.writeLock().lock();
2498     try {
2499       for (Map.Entry<String, Region> e: onlineRegions.entrySet()) {
2500         HRegionInfo hri = e.getValue().getRegionInfo();
2501         if (hri.isMetaRegion()) {
2502           meta = e.getValue();
2503         }
2504         if (meta != null) break;
2505       }
2506     } finally {
2507       this.lock.writeLock().unlock();
2508     }
2509     if (meta != null) closeRegionIgnoreErrors(meta.getRegionInfo(), abort);
2510   }
2511 
2512   /**
2513    * Schedule closes on all user regions.
2514    * Should be safe calling multiple times because it wont' close regions
2515    * that are already closed or that are closing.
2516    * @param abort Whether we're running an abort.
2517    */
2518   void closeUserRegions(final boolean abort) {
2519     this.lock.writeLock().lock();
2520     try {
2521       for (Map.Entry<String, Region> e: this.onlineRegions.entrySet()) {
2522         Region r = e.getValue();
2523         if (!r.getRegionInfo().isMetaTable() && r.isAvailable()) {
2524           // Don't update zk with this close transition; pass false.
2525           closeRegionIgnoreErrors(r.getRegionInfo(), abort);
2526         }
2527       }
2528     } finally {
2529       this.lock.writeLock().unlock();
2530     }
2531   }
2532 
2533   /** @return the info server */
2534   public InfoServer getInfoServer() {
2535     return infoServer;
2536   }
2537 
2538   /**
2539    * @return true if a stop has been requested.
2540    */
2541   @Override
2542   public boolean isStopped() {
2543     return this.stopped;
2544   }
2545 
2546   @Override
2547   public boolean isStopping() {
2548     return this.stopping;
2549   }
2550 
2551   @Override
2552   public Map<String, Region> getRecoveringRegions() {
2553     return this.recoveringRegions;
2554   }
2555 
2556   /**
2557    *
2558    * @return the configuration
2559    */
2560   @Override
2561   public Configuration getConfiguration() {
2562     return conf;
2563   }
2564 
2565   /** @return the write lock for the server */
2566   ReentrantReadWriteLock.WriteLock getWriteLock() {
2567     return lock.writeLock();
2568   }
2569 
2570   public int getNumberOfOnlineRegions() {
2571     return this.onlineRegions.size();
2572   }
2573 
2574   boolean isOnlineRegionsEmpty() {
2575     return this.onlineRegions.isEmpty();
2576   }
2577 
2578   /**
2579    * For tests, web ui and metrics.
2580    * This method will only work if HRegionServer is in the same JVM as client;
2581    * HRegion cannot be serialized to cross an rpc.
2582    */
2583   public Collection<Region> getOnlineRegionsLocalContext() {
2584     Collection<Region> regions = this.onlineRegions.values();
2585     return Collections.unmodifiableCollection(regions);
2586   }
2587 
2588   @Override
2589   public void addToOnlineRegions(Region region) {
2590     this.onlineRegions.put(region.getRegionInfo().getEncodedName(), region);
2591     configurationManager.registerObserver(region);
2592   }
2593 
2594   /**
2595    * @return A new Map of online regions sorted by region size with the first entry being the
2596    * biggest.  If two regions are the same size, then the last one found wins; i.e. this method
2597    * may NOT return all regions.
2598    */
2599   SortedMap<Long, Region> getCopyOfOnlineRegionsSortedBySize() {
2600     // we'll sort the regions in reverse
2601     SortedMap<Long, Region> sortedRegions = new TreeMap<Long, Region>(
2602         new Comparator<Long>() {
2603           @Override
2604           public int compare(Long a, Long b) {
2605             return -1 * a.compareTo(b);
2606           }
2607         });
2608     // Copy over all regions. Regions are sorted by size with biggest first.
2609     for (Region region : this.onlineRegions.values()) {
2610       sortedRegions.put(region.getMemstoreSize(), region);
2611     }
2612     return sortedRegions;
2613   }
2614 
2615   /**
2616    * @return time stamp in millis of when this region server was started
2617    */
2618   public long getStartcode() {
2619     return this.startcode;
2620   }
2621 
2622   /** @return reference to FlushRequester */
2623   @Override
2624   public FlushRequester getFlushRequester() {
2625     return this.cacheFlusher;
2626   }
2627 
2628   /**
2629    * Get the top N most loaded regions this server is serving so we can tell the
2630    * master which regions it can reallocate if we're overloaded. TODO: actually
2631    * calculate which regions are most loaded. (Right now, we're just grabbing
2632    * the first N regions being served regardless of load.)
2633    */
2634   protected HRegionInfo[] getMostLoadedRegions() {
2635     ArrayList<HRegionInfo> regions = new ArrayList<HRegionInfo>();
2636     for (Region r : onlineRegions.values()) {
2637       if (!r.isAvailable()) {
2638         continue;
2639       }
2640       if (regions.size() < numRegionsToReport) {
2641         regions.add(r.getRegionInfo());
2642       } else {
2643         break;
2644       }
2645     }
2646     return regions.toArray(new HRegionInfo[regions.size()]);
2647   }
2648 
2649   @Override
2650   public Leases getLeases() {
2651     return leases;
2652   }
2653 
2654   /**
2655    * @return Return the rootDir.
2656    */
2657   protected Path getRootDir() {
2658     return rootDir;
2659   }
2660 
2661   /**
2662    * @return Return the fs.
2663    */
2664   @Override
2665   public FileSystem getFileSystem() {
2666     return fs;
2667   }
2668 
2669   /**
2670    * @return Return the walRootDir.
2671    */
2672   protected Path getWALRootDir() {
2673     return walRootDir;
2674   }
2675 
2676   /**
2677    * @return Return the walFs.
2678    */
2679   protected FileSystem getWALFileSystem() {
2680     return walFs;
2681   }
2682 
2683   @Override
2684   public String toString() {
2685     return getServerName().toString();
2686   }
2687 
2688   /**
2689    * Interval at which threads should run
2690    *
2691    * @return the interval
2692    */
2693   public int getThreadWakeFrequency() {
2694     return threadWakeFrequency;
2695   }
2696 
2697   @Override
2698   public ZooKeeperWatcher getZooKeeper() {
2699     return zooKeeper;
2700   }
2701 
2702   @Override
2703   public BaseCoordinatedStateManager getCoordinatedStateManager() {
2704     return csm;
2705   }
2706 
2707   @Override
2708   public ServerName getServerName() {
2709     return serverName;
2710   }
2711 
2712   @Override
2713   public CompactionRequestor getCompactionRequester() {
2714     return this.compactSplitThread;
2715   }
2716 
2717   public RegionServerCoprocessorHost getRegionServerCoprocessorHost(){
2718     return this.rsHost;
2719   }
2720 
2721   @Override
2722   public ConcurrentMap<byte[], Boolean> getRegionsInTransitionInRS() {
2723     return this.regionsInTransitionInRS;
2724   }
2725 
2726   @Override
2727   public ExecutorService getExecutorService() {
2728     return service;
2729   }
2730 
2731   @Override
2732   public ChoreService getChoreService() {
2733     return choreService;
2734   }
2735   
2736   @Override
2737   public RegionServerRpcQuotaManager getRegionServerRpcQuotaManager() {
2738     return rsQuotaManager;
2739   }
2740 
2741   //
2742   // Main program and support routines
2743   //
2744 
2745   /**
2746    * Load the replication service objects, if any
2747    */
2748   static private void createNewReplicationInstance(Configuration conf,
2749     HRegionServer server, FileSystem walFs, Path walDir, Path oldWALDir) throws IOException{
2750 
2751     // If replication is not enabled, then return immediately.
2752     if (!conf.getBoolean(HConstants.REPLICATION_ENABLE_KEY,
2753         HConstants.REPLICATION_ENABLE_DEFAULT)) {
2754       return;
2755     }
2756 
2757     // read in the name of the source replication class from the config file.
2758     String sourceClassname = conf.get(HConstants.REPLICATION_SOURCE_SERVICE_CLASSNAME,
2759                                HConstants.REPLICATION_SERVICE_CLASSNAME_DEFAULT);
2760 
2761     // read in the name of the sink replication class from the config file.
2762     String sinkClassname = conf.get(HConstants.REPLICATION_SINK_SERVICE_CLASSNAME,
2763                              HConstants.REPLICATION_SERVICE_CLASSNAME_DEFAULT);
2764 
2765     // If both the sink and the source class names are the same, then instantiate
2766     // only one object.
2767     if (sourceClassname.equals(sinkClassname)) {
2768       server.replicationSourceHandler = (ReplicationSourceService)
2769                                          newReplicationInstance(sourceClassname,
2770                                          conf, server, walFs, walDir, oldWALDir);
2771       server.replicationSinkHandler = (ReplicationSinkService)
2772                                          server.replicationSourceHandler;
2773     } else {
2774       server.replicationSourceHandler = (ReplicationSourceService)
2775                                          newReplicationInstance(sourceClassname,
2776                                          conf, server, walFs, walDir, oldWALDir);
2777       server.replicationSinkHandler = (ReplicationSinkService)
2778                                          newReplicationInstance(sinkClassname,
2779                                          conf, server, walFs, walDir, oldWALDir);
2780     }
2781   }
2782 
2783   static private ReplicationService newReplicationInstance(String classname,
2784     Configuration conf, HRegionServer server, FileSystem walFs, Path walDir,
2785     Path oldLogDir) throws IOException{
2786 
2787     Class<?> clazz = null;
2788     try {
2789       ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
2790       clazz = Class.forName(classname, true, classLoader);
2791     } catch (java.lang.ClassNotFoundException nfe) {
2792       throw new IOException("Could not find class for " + classname);
2793     }
2794 
2795     // create an instance of the replication object.
2796     ReplicationService service = (ReplicationService)
2797                               ReflectionUtils.newInstance(clazz, conf);
2798     service.initialize(server, walFs, walDir, oldLogDir);
2799     return service;
2800   }
2801 
2802   /**
2803    * Utility for constructing an instance of the passed HRegionServer class.
2804    *
2805    * @param regionServerClass
2806    * @param conf2
2807    * @return HRegionServer instance.
2808    */
2809   public static HRegionServer constructRegionServer(
2810       Class<? extends HRegionServer> regionServerClass,
2811       final Configuration conf2, CoordinatedStateManager cp) {
2812     try {
2813       Constructor<? extends HRegionServer> c = regionServerClass
2814           .getConstructor(Configuration.class, CoordinatedStateManager.class);
2815       return c.newInstance(conf2, cp);
2816     } catch (Exception e) {
2817       throw new RuntimeException("Failed construction of " + "Regionserver: "
2818           + regionServerClass.toString(), e);
2819     }
2820   }
2821 
2822   /**
2823    * @see org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine
2824    */
2825   public static void main(String[] args) throws Exception {
2826     VersionInfo.logVersion();
2827     Configuration conf = HBaseConfiguration.create();
2828     @SuppressWarnings("unchecked")
2829     Class<? extends HRegionServer> regionServerClass = (Class<? extends HRegionServer>) conf
2830         .getClass(HConstants.REGION_SERVER_IMPL, HRegionServer.class);
2831 
2832     new HRegionServerCommandLine(regionServerClass).doMain(args);
2833   }
2834 
2835   /**
2836    * Gets the online regions of the specified table.
2837    * This method looks at the in-memory onlineRegions.  It does not go to <code>hbase:meta</code>.
2838    * Only returns <em>online</em> regions.  If a region on this table has been
2839    * closed during a disable, etc., it will not be included in the returned list.
2840    * So, the returned list may not necessarily be ALL regions in this table, its
2841    * all the ONLINE regions in the table.
2842    * @param tableName
2843    * @return Online regions from <code>tableName</code>
2844    */
2845   @Override
2846   public List<Region> getOnlineRegions(TableName tableName) {
2847      List<Region> tableRegions = new ArrayList<Region>();
2848      synchronized (this.onlineRegions) {
2849        for (Region region: this.onlineRegions.values()) {
2850          HRegionInfo regionInfo = region.getRegionInfo();
2851          if(regionInfo.getTable().equals(tableName)) {
2852            tableRegions.add(region);
2853          }
2854        }
2855      }
2856      return tableRegions;
2857    }
2858 
2859   // Backport from HBASE-15027
2860   /**
2861    * Get all online regions in this RS.
2862    * @return List of online Region
2863    */
2864   public List<Region> getOnlineRegions() {
2865     List<Region> allRegions = new ArrayList<Region>();
2866     synchronized (this.onlineRegions) {
2867       // Return a clone copy of the onlineRegions
2868       allRegions.addAll(onlineRegions.values());
2869     }
2870     return allRegions;
2871   }
2872   
2873   /**
2874    * Gets the online tables in this RS.
2875    * This method looks at the in-memory onlineRegions.
2876    * @return all the online tables in this RS
2877    */
2878   @Override
2879   public Set<TableName> getOnlineTables() {
2880     Set<TableName> tables = new HashSet<TableName>();
2881     synchronized (this.onlineRegions) {
2882       for (Region region: this.onlineRegions.values()) {
2883         tables.add(region.getTableDesc().getTableName());
2884       }
2885     }
2886     return tables;
2887   }
2888 
2889   // used by org/apache/hbase/tmpl/regionserver/RSStatusTmpl.jamon (HBASE-4070).
2890   public String[] getRegionServerCoprocessors() {
2891     TreeSet<String> coprocessors = new TreeSet<String>();
2892     try {
2893       coprocessors.addAll(getWAL(null).getCoprocessorHost().getCoprocessors());
2894     } catch (IOException exception) {
2895       LOG.warn("Exception attempting to fetch wal coprocessor information for the common wal; " +
2896           "skipping.");
2897       LOG.debug("Exception details for failure to fetch wal coprocessor information.", exception);
2898     }
2899     Collection<Region> regions = getOnlineRegionsLocalContext();
2900     for (Region region: regions) {
2901       coprocessors.addAll(region.getCoprocessorHost().getCoprocessors());
2902       try {
2903         coprocessors.addAll(getWAL(region.getRegionInfo()).getCoprocessorHost().getCoprocessors());
2904       } catch (IOException exception) {
2905         LOG.warn("Exception attempting to fetch wal coprocessor information for region " + region +
2906             "; skipping.");
2907         LOG.debug("Exception details for failure to fetch wal coprocessor information.", exception);
2908       }
2909     }
2910     return coprocessors.toArray(new String[coprocessors.size()]);
2911   }
2912 
2913   /**
2914    * Try to close the region, logs a warning on failure but continues.
2915    * @param region Region to close
2916    */
2917   private void closeRegionIgnoreErrors(HRegionInfo region, final boolean abort) {
2918     try {
2919       CloseRegionCoordination.CloseRegionDetails details =
2920         csm.getCloseRegionCoordination().getDetaultDetails();
2921       if (!closeRegion(region.getEncodedName(), abort, details, null)) {
2922         LOG.warn("Failed to close " + region.getRegionNameAsString() +
2923             " - ignoring and continuing");
2924       }
2925     } catch (IOException e) {
2926       LOG.warn("Failed to close " + region.getRegionNameAsString() +
2927           " - ignoring and continuing", e);
2928     }
2929   }
2930 
2931   /**
2932    * Close asynchronously a region, can be called from the master or internally by the regionserver
2933    * when stopping. If called from the master, the region will update the znode status.
2934    *
2935    * <p>
2936    * If an opening was in progress, this method will cancel it, but will not start a new close. The
2937    * coprocessors are not called in this case. A NotServingRegionException exception is thrown.
2938    * </p>
2939 
2940    * <p>
2941    *   If a close was in progress, this new request will be ignored, and an exception thrown.
2942    * </p>
2943    *
2944    * @param encodedName Region to close
2945    * @param abort True if we are aborting
2946    * @param crd details about closing region coordination-coordinated task
2947    * @return True if closed a region.
2948    * @throws NotServingRegionException if the region is not online
2949    * @throws RegionAlreadyInTransitionException if the region is already closing
2950    */
2951   protected boolean closeRegion(String encodedName, final boolean abort,
2952       CloseRegionCoordination.CloseRegionDetails crd, final ServerName sn)
2953       throws NotServingRegionException, RegionAlreadyInTransitionException {
2954     //Check for permissions to close.
2955     Region actualRegion = this.getFromOnlineRegions(encodedName);
2956     if ((actualRegion != null) && (actualRegion.getCoprocessorHost() != null)) {
2957       try {
2958         actualRegion.getCoprocessorHost().preClose(false);
2959       } catch (IOException exp) {
2960         LOG.warn("Unable to close region: the coprocessor launched an error ", exp);
2961         return false;
2962       }
2963     }
2964 
2965     final Boolean previous = this.regionsInTransitionInRS.putIfAbsent(encodedName.getBytes(),
2966         Boolean.FALSE);
2967 
2968     if (Boolean.TRUE.equals(previous)) {
2969       LOG.info("Received CLOSE for the region:" + encodedName + " , which we are already " +
2970           "trying to OPEN. Cancelling OPENING.");
2971       if (!regionsInTransitionInRS.replace(encodedName.getBytes(), previous, Boolean.FALSE)){
2972         // The replace failed. That should be an exceptional case, but theoretically it can happen.
2973         // We're going to try to do a standard close then.
2974         LOG.warn("The opening for region " + encodedName + " was done before we could cancel it." +
2975             " Doing a standard close now");
2976         return closeRegion(encodedName, abort, crd, sn);
2977       }
2978       // Let's get the region from the online region list again
2979       actualRegion = this.getFromOnlineRegions(encodedName);
2980       if (actualRegion == null) { // If already online, we still need to close it.
2981         LOG.info("The opening previously in progress has been cancelled by a CLOSE request.");
2982         // The master deletes the znode when it receives this exception.
2983         throw new RegionAlreadyInTransitionException("The region " + encodedName +
2984           " was opening but not yet served. Opening is cancelled.");
2985       }
2986     } else if (Boolean.FALSE.equals(previous)) {
2987       LOG.info("Received CLOSE for the region: " + encodedName +
2988         ", which we are already trying to CLOSE, but not completed yet");
2989       // The master will retry till the region is closed. We need to do this since
2990       // the region could fail to close somehow. If we mark the region closed in master
2991       // while it is not, there could be data loss.
2992       // If the region stuck in closing for a while, and master runs out of retries,
2993       // master will move the region to failed_to_close. Later on, if the region
2994       // is indeed closed, master can properly re-assign it.
2995       throw new RegionAlreadyInTransitionException("The region " + encodedName +
2996         " was already closing. New CLOSE request is ignored.");
2997     }
2998 
2999     if (actualRegion == null) {
3000       LOG.error("Received CLOSE for a region which is not online, and we're not opening.");
3001       this.regionsInTransitionInRS.remove(encodedName.getBytes());
3002       // The master deletes the znode when it receives this exception.
3003       throw new NotServingRegionException("The region " + encodedName +
3004           " is not online, and is not opening.");
3005     }
3006 
3007     CloseRegionHandler crh;
3008     final HRegionInfo hri = actualRegion.getRegionInfo();
3009     if (hri.isMetaRegion()) {
3010       crh = new CloseMetaHandler(this, this, hri, abort,
3011         csm.getCloseRegionCoordination(), crd);
3012     } else {
3013       crh = new CloseRegionHandler(this, this, hri, abort,
3014         csm.getCloseRegionCoordination(), crd, sn);
3015     }
3016     this.service.submit(crh);
3017     return true;
3018   }
3019 
3020    /**
3021    * @param regionName
3022    * @return HRegion for the passed binary <code>regionName</code> or null if
3023    *         named region is not member of the online regions.
3024    */
3025   public Region getOnlineRegion(final byte[] regionName) {
3026     String encodedRegionName = HRegionInfo.encodeRegionName(regionName);
3027     return this.onlineRegions.get(encodedRegionName);
3028   }
3029 
3030   public InetSocketAddress[] getRegionBlockLocations(final String encodedRegionName) {
3031     return this.regionFavoredNodesMap.get(encodedRegionName);
3032   }
3033 
3034   @Override
3035   public Region getFromOnlineRegions(final String encodedRegionName) {
3036     return this.onlineRegions.get(encodedRegionName);
3037   }
3038 
3039 
3040   @Override
3041   public boolean removeFromOnlineRegions(final Region r, ServerName destination) {
3042     Region toReturn = this.onlineRegions.remove(r.getRegionInfo().getEncodedName());
3043 
3044     if (destination != null) {
3045       long closeSeqNum = r.getMaxFlushedSeqId();
3046       if (closeSeqNum == HConstants.NO_SEQNUM) {
3047         // No edits in WAL for this region; get the sequence number when the region was opened.
3048         closeSeqNum = r.getOpenSeqNum();
3049         if (closeSeqNum == HConstants.NO_SEQNUM) {
3050           closeSeqNum = 0;
3051         }
3052       }
3053       addToMovedRegions(r.getRegionInfo().getEncodedName(), destination, closeSeqNum);
3054     }
3055     this.regionFavoredNodesMap.remove(r.getRegionInfo().getEncodedName());
3056     return toReturn != null;
3057   }
3058 
3059   /**
3060    * Protected utility method for safely obtaining an HRegion handle.
3061    *
3062    * @param regionName
3063    *          Name of online {@link Region} to return
3064    * @return {@link Region} for <code>regionName</code>
3065    * @throws NotServingRegionException
3066    */
3067   protected Region getRegion(final byte[] regionName)
3068       throws NotServingRegionException {
3069     String encodedRegionName = HRegionInfo.encodeRegionName(regionName);
3070     return getRegionByEncodedName(regionName, encodedRegionName);
3071   }
3072 
3073   public Region getRegionByEncodedName(String encodedRegionName)
3074       throws NotServingRegionException {
3075     return getRegionByEncodedName(null, encodedRegionName);
3076   }
3077 
3078   protected Region getRegionByEncodedName(byte[] regionName, String encodedRegionName)
3079     throws NotServingRegionException {
3080     Region region = this.onlineRegions.get(encodedRegionName);
3081     if (region == null) {
3082       MovedRegionInfo moveInfo = getMovedRegion(encodedRegionName);
3083       if (moveInfo != null) {
3084         throw new RegionMovedException(moveInfo.getServerName(), moveInfo.getSeqNum());
3085       }
3086       Boolean isOpening = this.regionsInTransitionInRS.get(Bytes.toBytes(encodedRegionName));
3087       String regionNameStr = regionName == null?
3088         encodedRegionName: Bytes.toStringBinary(regionName);
3089       if (isOpening != null && isOpening.booleanValue()) {
3090         throw new RegionOpeningException("Region " + regionNameStr +
3091           " is opening on " + this.serverName);
3092       }
3093       throw new NotServingRegionException("Region " + regionNameStr +
3094         " is not online on " + this.serverName);
3095     }
3096     return region;
3097   }
3098 
3099   /*
3100    * Cleanup after Throwable caught invoking method. Converts <code>t</code> to
3101    * IOE if it isn't already.
3102    *
3103    * @param t Throwable
3104    *
3105    * @param msg Message to log in error. Can be null.
3106    *
3107    * @return Throwable converted to an IOE; methods can only let out IOEs.
3108    */
3109   private Throwable cleanup(final Throwable t, final String msg) {
3110     // Don't log as error if NSRE; NSRE is 'normal' operation.
3111     if (t instanceof NotServingRegionException) {
3112       LOG.debug("NotServingRegionException; " + t.getMessage());
3113       return t;
3114     }
3115     if (msg == null) {
3116       LOG.error("", RemoteExceptionHandler.checkThrowable(t));
3117     } else {
3118       LOG.error(msg, RemoteExceptionHandler.checkThrowable(t));
3119     }
3120     if (!rpcServices.checkOOME(t)) {
3121       checkFileSystem();
3122     }
3123     return t;
3124   }
3125 
3126   /*
3127    * @param t
3128    *
3129    * @param msg Message to put in new IOE if passed <code>t</code> is not an IOE
3130    *
3131    * @return Make <code>t</code> an IOE if it isn't already.
3132    */
3133   protected IOException convertThrowableToIOE(final Throwable t, final String msg) {
3134     return (t instanceof IOException ? (IOException) t : msg == null
3135         || msg.length() == 0 ? new IOException(t) : new IOException(msg, t));
3136   }
3137 
3138   /**
3139    * Checks to see if the file system is still accessible. If not, sets
3140    * abortRequested and stopRequested
3141    *
3142    * @return false if file system is not available
3143    */
3144   public boolean checkFileSystem() {
3145     if (this.fsOk && this.fs != null) {
3146       try {
3147         FSUtils.checkFileSystemAvailable(this.fs);
3148       } catch (IOException e) {
3149         abort("File System not available", e);
3150         this.fsOk = false;
3151       }
3152     }
3153     return this.fsOk;
3154   }
3155 
3156   @Override
3157   public void updateRegionFavoredNodesMapping(String encodedRegionName,
3158       List<org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName> favoredNodes) {
3159     InetSocketAddress[] addr = new InetSocketAddress[favoredNodes.size()];
3160     // Refer to the comment on the declaration of regionFavoredNodesMap on why
3161     // it is a map of region name to InetSocketAddress[]
3162     for (int i = 0; i < favoredNodes.size(); i++) {
3163       addr[i] = InetSocketAddress.createUnresolved(favoredNodes.get(i).getHostName(),
3164           favoredNodes.get(i).getPort());
3165     }
3166     regionFavoredNodesMap.put(encodedRegionName, addr);
3167   }
3168 
3169   /**
3170    * Return the favored nodes for a region given its encoded name. Look at the
3171    * comment around {@link #regionFavoredNodesMap} on why it is InetSocketAddress[]
3172    * @param encodedRegionName
3173    * @return array of favored locations
3174    */
3175   @Override
3176   public InetSocketAddress[] getFavoredNodesForRegion(String encodedRegionName) {
3177     return regionFavoredNodesMap.get(encodedRegionName);
3178   }
3179 
3180   @Override
3181   public ServerNonceManager getNonceManager() {
3182     return this.nonceManager;
3183   }
3184 
3185   private static class MovedRegionInfo {
3186     private final ServerName serverName;
3187     private final long seqNum;
3188     private final long ts;
3189 
3190     public MovedRegionInfo(ServerName serverName, long closeSeqNum) {
3191       this.serverName = serverName;
3192       this.seqNum = closeSeqNum;
3193       ts = EnvironmentEdgeManager.currentTime();
3194      }
3195 
3196     public ServerName getServerName() {
3197       return serverName;
3198     }
3199 
3200     public long getSeqNum() {
3201       return seqNum;
3202     }
3203 
3204     public long getMoveTime() {
3205       return ts;
3206     }
3207   }
3208 
3209   // This map will contains all the regions that we closed for a move.
3210   //  We add the time it was moved as we don't want to keep too old information
3211   protected Map<String, MovedRegionInfo> movedRegions =
3212       new ConcurrentHashMap<String, MovedRegionInfo>(3000);
3213 
3214   // We need a timeout. If not there is a risk of giving a wrong information: this would double
3215   //  the number of network calls instead of reducing them.
3216   private static final int TIMEOUT_REGION_MOVED = (2 * 60 * 1000);
3217 
3218   protected void addToMovedRegions(String encodedName, ServerName destination, long closeSeqNum) {
3219     if (ServerName.isSameHostnameAndPort(destination, this.getServerName())) {
3220       LOG.warn("Not adding moved region record: " + encodedName + " to self.");
3221       return;
3222     }
3223     LOG.info("Adding moved region record: "
3224       + encodedName + " to " + destination + " as of " + closeSeqNum);
3225     movedRegions.put(encodedName, new MovedRegionInfo(destination, closeSeqNum));
3226   }
3227 
3228   void removeFromMovedRegions(String encodedName) {
3229     movedRegions.remove(encodedName);
3230   }
3231 
3232   private MovedRegionInfo getMovedRegion(final String encodedRegionName) {
3233     MovedRegionInfo dest = movedRegions.get(encodedRegionName);
3234 
3235     long now = EnvironmentEdgeManager.currentTime();
3236     if (dest != null) {
3237       if (dest.getMoveTime() > (now - TIMEOUT_REGION_MOVED)) {
3238         return dest;
3239       } else {
3240         movedRegions.remove(encodedRegionName);
3241       }
3242     }
3243 
3244     return null;
3245   }
3246 
3247   /**
3248    * Remove the expired entries from the moved regions list.
3249    */
3250   protected void cleanMovedRegions() {
3251     final long cutOff = System.currentTimeMillis() - TIMEOUT_REGION_MOVED;
3252     Iterator<Entry<String, MovedRegionInfo>> it = movedRegions.entrySet().iterator();
3253 
3254     while (it.hasNext()){
3255       Map.Entry<String, MovedRegionInfo> e = it.next();
3256       if (e.getValue().getMoveTime() < cutOff) {
3257         it.remove();
3258       }
3259     }
3260   }
3261 
3262   /*
3263    * Use this to allow tests to override and schedule more frequently.
3264    */
3265 
3266   protected int movedRegionCleanerPeriod() {
3267         return TIMEOUT_REGION_MOVED;
3268   }
3269 
3270   /**
3271    * Creates a Chore thread to clean the moved region cache.
3272    */
3273 
3274   protected final static class MovedRegionsCleaner extends ScheduledChore implements Stoppable {
3275     private HRegionServer regionServer;
3276     Stoppable stoppable;
3277 
3278     private MovedRegionsCleaner(
3279       HRegionServer regionServer, Stoppable stoppable){
3280       super("MovedRegionsCleaner for region " + regionServer, stoppable,
3281           regionServer.movedRegionCleanerPeriod());
3282       this.regionServer = regionServer;
3283       this.stoppable = stoppable;
3284     }
3285 
3286     static MovedRegionsCleaner create(HRegionServer rs){
3287       Stoppable stoppable = new Stoppable() {
3288         private volatile boolean isStopped = false;
3289         @Override public void stop(String why) { isStopped = true;}
3290         @Override public boolean isStopped() {return isStopped;}
3291       };
3292 
3293       return new MovedRegionsCleaner(rs, stoppable);
3294     }
3295 
3296     @Override
3297     protected void chore() {
3298       regionServer.cleanMovedRegions();
3299     }
3300 
3301     @Override
3302     public void stop(String why) {
3303       stoppable.stop(why);
3304     }
3305 
3306     @Override
3307     public boolean isStopped() {
3308       return stoppable.isStopped();
3309     }
3310   }
3311 
3312   private String getMyEphemeralNodePath() {
3313     return ZKUtil.joinZNode(this.zooKeeper.rsZNode, getServerName().toString());
3314   }
3315 
3316   private boolean isHealthCheckerConfigured() {
3317     String healthScriptLocation = this.conf.get(HConstants.HEALTH_SCRIPT_LOC);
3318     return org.apache.commons.lang.StringUtils.isNotBlank(healthScriptLocation);
3319   }
3320 
3321   /**
3322    * @return the underlying {@link CompactSplitThread} for the servers
3323    */
3324   public CompactSplitThread getCompactSplitThread() {
3325     return this.compactSplitThread;
3326   }
3327 
3328   /**
3329    * A helper function to store the last flushed sequence Id with the previous failed RS for a
3330    * recovering region. The Id is used to skip wal edits which are flushed. Since the flushed
3331    * sequence id is only valid for each RS, we associate the Id with corresponding failed RS.
3332    * @throws KeeperException
3333    * @throws IOException
3334    */
3335   private void updateRecoveringRegionLastFlushedSequenceId(Region r) throws KeeperException,
3336       IOException {
3337     if (!r.isRecovering()) {
3338       // return immdiately for non-recovering regions
3339       return;
3340     }
3341 
3342     HRegionInfo regionInfo = r.getRegionInfo();
3343     ZooKeeperWatcher zkw = getZooKeeper();
3344     String previousRSName = this.getLastFailedRSFromZK(regionInfo.getEncodedName());
3345     Map<byte[], Long> maxSeqIdInStores = r.getMaxStoreSeqId();
3346     long minSeqIdForLogReplay = -1;
3347     for (Long storeSeqIdForReplay : maxSeqIdInStores.values()) {
3348       if (minSeqIdForLogReplay == -1 || storeSeqIdForReplay < minSeqIdForLogReplay) {
3349         minSeqIdForLogReplay = storeSeqIdForReplay;
3350       }
3351     }
3352 
3353     try {
3354       long lastRecordedFlushedSequenceId = -1;
3355       String nodePath = ZKUtil.joinZNode(this.zooKeeper.recoveringRegionsZNode,
3356         regionInfo.getEncodedName());
3357       // recovering-region level
3358       byte[] data;
3359       try {
3360         data = ZKUtil.getData(zkw, nodePath);
3361       } catch (InterruptedException e) {
3362         throw new InterruptedIOException();
3363       }
3364       if (data != null) {
3365         lastRecordedFlushedSequenceId = ZKSplitLog.parseLastFlushedSequenceIdFrom(data);
3366       }
3367       if (data == null || lastRecordedFlushedSequenceId < minSeqIdForLogReplay) {
3368         ZKUtil.setData(zkw, nodePath, ZKUtil.positionToByteArray(minSeqIdForLogReplay));
3369       }
3370       if (previousRSName != null) {
3371         // one level deeper for the failed RS
3372         nodePath = ZKUtil.joinZNode(nodePath, previousRSName);
3373         ZKUtil.setData(zkw, nodePath,
3374           ZKUtil.regionSequenceIdsToByteArray(minSeqIdForLogReplay, maxSeqIdInStores));
3375         LOG.debug("Update last flushed sequence id of region " + regionInfo.getEncodedName() +
3376           " for " + previousRSName);
3377       } else {
3378         LOG.warn("Can't find failed region server for recovering region " +
3379             regionInfo.getEncodedName());
3380       }
3381     } catch (NoNodeException ignore) {
3382       LOG.debug("Region " + regionInfo.getEncodedName() +
3383         " must have completed recovery because its recovery znode has been removed", ignore);
3384     }
3385   }
3386 
3387   /**
3388    * Return the last failed RS name under /hbase/recovering-regions/encodedRegionName
3389    * @param encodedRegionName
3390    * @throws KeeperException
3391    */
3392   private String getLastFailedRSFromZK(String encodedRegionName) throws KeeperException {
3393     String result = null;
3394     long maxZxid = 0;
3395     ZooKeeperWatcher zkw = this.getZooKeeper();
3396     String nodePath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, encodedRegionName);
3397     List<String> failedServers = ZKUtil.listChildrenNoWatch(zkw, nodePath);
3398     if (failedServers == null || failedServers.isEmpty()) {
3399       return result;
3400     }
3401     for (String failedServer : failedServers) {
3402       String rsPath = ZKUtil.joinZNode(nodePath, failedServer);
3403       Stat stat = new Stat();
3404       ZKUtil.getDataNoWatch(zkw, rsPath, stat);
3405       if (maxZxid < stat.getCzxid()) {
3406         maxZxid = stat.getCzxid();
3407         result = failedServer;
3408       }
3409     }
3410     return result;
3411   }
3412 
3413   public CoprocessorServiceResponse execRegionServerService(final RpcController controller,
3414       final CoprocessorServiceRequest serviceRequest) throws ServiceException {
3415     try {
3416       ServerRpcController execController = new ServerRpcController();
3417       CoprocessorServiceCall call = serviceRequest.getCall();
3418       String serviceName = call.getServiceName();
3419       String methodName = call.getMethodName();
3420       if (!coprocessorServiceHandlers.containsKey(serviceName)) {
3421         throw new UnknownProtocolException(null,
3422             "No registered coprocessor service found for name " + serviceName);
3423       }
3424       Service service = coprocessorServiceHandlers.get(serviceName);
3425       Descriptors.ServiceDescriptor serviceDesc = service.getDescriptorForType();
3426       Descriptors.MethodDescriptor methodDesc = serviceDesc.findMethodByName(methodName);
3427       if (methodDesc == null) {
3428         throw new UnknownProtocolException(service.getClass(), "Unknown method " + methodName
3429             + " called on service " + serviceName);
3430       }
3431       Message.Builder builderForType = service.getRequestPrototype(methodDesc).newBuilderForType();
3432       ProtobufUtil.mergeFrom(builderForType, call.getRequest());
3433       Message request = builderForType.build();
3434       final Message.Builder responseBuilder =
3435           service.getResponsePrototype(methodDesc).newBuilderForType();
3436       service.callMethod(methodDesc, controller, request, new RpcCallback<Message>() {
3437         @Override
3438         public void run(Message message) {
3439           if (message != null) {
3440             responseBuilder.mergeFrom(message);
3441           }
3442         }
3443       });
3444       Message execResult = responseBuilder.build();
3445       if (execController.getFailedOn() != null) {
3446         throw execController.getFailedOn();
3447       }
3448       ClientProtos.CoprocessorServiceResponse.Builder builder =
3449           ClientProtos.CoprocessorServiceResponse.newBuilder();
3450       builder.setRegion(RequestConverter.buildRegionSpecifier(RegionSpecifierType.REGION_NAME,
3451         HConstants.EMPTY_BYTE_ARRAY));
3452       builder.setValue(builder.getValueBuilder().setName(execResult.getClass().getName())
3453           .setValue(execResult.toByteString()));
3454       return builder.build();
3455     } catch (IOException ie) {
3456       throw new ServiceException(ie);
3457     }
3458   }
3459 
3460   /**
3461    * @return The cache config instance used by the regionserver.
3462    */
3463   public CacheConfig getCacheConfig() {
3464     return this.cacheConfig;
3465   }
3466 
3467   /**
3468    * @return : Returns the ConfigurationManager object for testing purposes.
3469    */
3470   protected ConfigurationManager getConfigurationManager() {
3471     return configurationManager;
3472   }
3473 
3474   /**
3475    * @return Return table descriptors implementation.
3476    */
3477   public TableDescriptors getTableDescriptors() {
3478     return this.tableDescriptors;
3479   }
3480 
3481   /**
3482    * Reload the configuration from disk.
3483    */
3484   public void updateConfiguration() {
3485     LOG.info("Reloading the configuration from disk.");
3486     // Reload the configuration from disk.
3487     conf.reloadConfiguration();
3488     configurationManager.notifyAllObservers(conf);
3489   }
3490 
3491   @Override
3492   public HeapMemoryManager getHeapMemoryManager() {
3493     return hMemManager;
3494   }
3495 
3496   @Override
3497   public double getCompactionPressure() {
3498     double max = 0;
3499     for (Region region : onlineRegions.values()) {
3500       for (Store store : region.getStores()) {
3501         double normCount = store.getCompactionPressure();
3502         if (normCount > max) {
3503           max = normCount;
3504         }
3505       }
3506     }
3507     return max;
3508   }
3509 
3510   @Override
3511   public MetricsRegionServer getMetrics() {
3512     return metricsRegionServer;
3513   }
3514 
3515   @Override
3516   public RegionServerSpaceQuotaManager getRegionServerSpaceQuotaManager() {
3517     return this.rsSpaceQuotaManager;
3518   }
3519 }