View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import java.io.FileNotFoundException;
22  import java.io.IOException;
23  import java.io.InterruptedIOException;
24  import java.net.InetSocketAddress;
25  import java.net.UnknownHostException;
26  import java.util.ArrayList;
27  import java.util.Collections;
28  import java.util.HashMap;
29  import java.util.Iterator;
30  import java.util.List;
31  import java.util.Map;
32  import java.util.NavigableMap;
33  import java.util.Set;
34  import java.util.TreeSet;
35  import java.util.concurrent.ConcurrentHashMap;
36  import java.util.concurrent.atomic.AtomicLong;
37  
38  import org.apache.commons.logging.Log;
39  import org.apache.commons.logging.LogFactory;
40  import org.apache.hadoop.conf.Configuration;
41  import org.apache.hadoop.fs.Path;
42  import org.apache.hadoop.hbase.Cell;
43  import org.apache.hadoop.hbase.CellScannable;
44  import org.apache.hadoop.hbase.CellScanner;
45  import org.apache.hadoop.hbase.CellUtil;
46  import org.apache.hadoop.hbase.DoNotRetryIOException;
47  import org.apache.hadoop.hbase.DroppedSnapshotException;
48  import org.apache.hadoop.hbase.HBaseIOException;
49  import org.apache.hadoop.hbase.HConstants;
50  import org.apache.hadoop.hbase.HRegionInfo;
51  import org.apache.hadoop.hbase.HTableDescriptor;
52  import org.apache.hadoop.hbase.MetaTableAccessor;
53  import org.apache.hadoop.hbase.NotServingRegionException;
54  import org.apache.hadoop.hbase.ServerName;
55  import org.apache.hadoop.hbase.TableName;
56  import org.apache.hadoop.hbase.UnknownScannerException;
57  import org.apache.hadoop.hbase.classification.InterfaceAudience;
58  import org.apache.hadoop.hbase.client.Append;
59  import org.apache.hadoop.hbase.client.ConnectionUtils;
60  import org.apache.hadoop.hbase.client.Delete;
61  import org.apache.hadoop.hbase.client.Durability;
62  import org.apache.hadoop.hbase.client.Get;
63  import org.apache.hadoop.hbase.client.Increment;
64  import org.apache.hadoop.hbase.client.Mutation;
65  import org.apache.hadoop.hbase.client.Put;
66  import org.apache.hadoop.hbase.client.RegionReplicaUtil;
67  import org.apache.hadoop.hbase.client.Result;
68  import org.apache.hadoop.hbase.client.RowMutations;
69  import org.apache.hadoop.hbase.client.Scan;
70  import org.apache.hadoop.hbase.client.VersionInfoUtil;
71  import org.apache.hadoop.hbase.coordination.CloseRegionCoordination;
72  import org.apache.hadoop.hbase.coordination.OpenRegionCoordination;
73  import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
74  import org.apache.hadoop.hbase.exceptions.MergeRegionException;
75  import org.apache.hadoop.hbase.exceptions.OperationConflictException;
76  import org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException;
77  import org.apache.hadoop.hbase.exceptions.ScannerResetException;
78  import org.apache.hadoop.hbase.filter.ByteArrayComparable;
79  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
80  import org.apache.hadoop.hbase.io.hfile.CorruptHFileException;
81  import org.apache.hadoop.hbase.ipc.HBaseRPCErrorHandler;
82  import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
83  import org.apache.hadoop.hbase.ipc.PriorityFunction;
84  import org.apache.hadoop.hbase.ipc.QosPriority;
85  import org.apache.hadoop.hbase.ipc.RpcCallContext;
86  import org.apache.hadoop.hbase.ipc.RpcServer;
87  import org.apache.hadoop.hbase.ipc.RpcServer.BlockingServiceAndInterface;
88  import org.apache.hadoop.hbase.ipc.RpcServerInterface;
89  import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
90  import org.apache.hadoop.hbase.ipc.ServerRpcController;
91  import org.apache.hadoop.hbase.master.MasterRpcServices;
92  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
93  import org.apache.hadoop.hbase.protobuf.RequestConverter;
94  import org.apache.hadoop.hbase.protobuf.ResponseConverter;
95  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
96  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
97  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
98  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactRegionRequest;
99  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactRegionResponse;
100 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.FlushRegionRequest;
101 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.FlushRegionResponse;
102 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
103 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
104 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
105 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse;
106 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoRequest;
107 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoResponse;
108 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileRequest;
109 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileResponse;
110 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsRequest;
111 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsResponse;
112 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
113 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest.RegionOpenInfo;
114 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse;
115 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse.RegionOpeningState;
116 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ReplicateWALEntryRequest;
117 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ReplicateWALEntryResponse;
118 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterRequest;
119 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterResponse;
120 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.SplitRegionRequest;
121 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.SplitRegionResponse;
122 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.StopServerRequest;
123 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.StopServerResponse;
124 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateConfigurationRequest;
125 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateConfigurationResponse;
126 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateFavoredNodesRequest;
127 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateFavoredNodesResponse;
128 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WALEntry;
129 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionRequest;
130 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionResponse;
131 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
132 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.Action;
133 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileRequest;
134 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileRequest.FamilyPath;
135 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileResponse;
136 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
137 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.Condition;
138 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
139 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
140 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest;
141 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetResponse;
142 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiRequest;
143 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiResponse;
144 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateRequest;
145 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateResponse;
146 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
147 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
148 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionAction;
149 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
150 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ResultOrException;
151 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
152 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
153 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo;
154 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier;
155 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType;
156 import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsRequest;
157 import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse;
158 import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse.TableQuotaSnapshot;
159 import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.SpaceViolationPolicy;
160 import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.RequestHeader;
161 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.BulkLoadDescriptor;
162 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
163 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
164 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
165 import org.apache.hadoop.hbase.quotas.ActivePolicyEnforcement;
166 import org.apache.hadoop.hbase.quotas.OperationQuota;
167 import org.apache.hadoop.hbase.quotas.QuotaUtil;
168 import org.apache.hadoop.hbase.quotas.RegionServerRpcQuotaManager;
169 import org.apache.hadoop.hbase.quotas.ActivePolicyEnforcement;
170 import org.apache.hadoop.hbase.quotas.RegionServerSpaceQuotaManager;
171 import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
172 import org.apache.hadoop.hbase.quotas.SpaceViolationPolicyEnforcement;
173 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
174 import org.apache.hadoop.hbase.regionserver.Leases.Lease;
175 import org.apache.hadoop.hbase.regionserver.Leases.LeaseStillHeldException;
176 import org.apache.hadoop.hbase.regionserver.Region.Operation;
177 import org.apache.hadoop.hbase.regionserver.ScannerContext.LimitScope;
178 import org.apache.hadoop.hbase.regionserver.handler.OpenMetaHandler;
179 import org.apache.hadoop.hbase.regionserver.handler.OpenPriorityRegionHandler;
180 import org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler;
181 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
182 import org.apache.hadoop.hbase.security.Superusers;
183 import org.apache.hadoop.hbase.security.User;
184 import org.apache.hadoop.hbase.util.Bytes;
185 import org.apache.hadoop.hbase.util.Counter;
186 import org.apache.hadoop.hbase.util.DNS;
187 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
188 import org.apache.hadoop.hbase.util.Pair;
189 import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
190 import org.apache.hadoop.hbase.util.Strings;
191 import org.apache.hadoop.hbase.wal.WAL;
192 import org.apache.hadoop.hbase.wal.WALKey;
193 import org.apache.hadoop.hbase.wal.WALSplitter;
194 import org.apache.hadoop.hbase.zookeeper.ZKSplitLog;
195 import org.apache.zookeeper.KeeperException;
196 
197 import com.google.common.annotations.VisibleForTesting;
198 import com.google.protobuf.ByteString;
199 import com.google.protobuf.Message;
200 import com.google.protobuf.RpcController;
201 import com.google.protobuf.ServiceException;
202 import com.google.protobuf.TextFormat;
203 
204 /**
205  * Implements the regionserver RPC services.
206  */
207 @InterfaceAudience.Private
208 @SuppressWarnings("deprecation")
209 public class RSRpcServices implements HBaseRPCErrorHandler,
210     AdminService.BlockingInterface, ClientService.BlockingInterface, PriorityFunction {
211   protected static final Log LOG = LogFactory.getLog(RSRpcServices.class);
212 
213   /** RPC scheduler to use for the region server. */
214   public static final String REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS =
215     "hbase.region.server.rpc.scheduler.factory.class";
216 
217   /**
218    * Minimum allowable time limit delta (in milliseconds) that can be enforced during scans. This
219    * configuration exists to prevent the scenario where a time limit is specified to be so
220    * restrictive that the time limit is reached immediately (before any cells are scanned).
221    */
222   private static final String REGION_SERVER_RPC_MINIMUM_SCAN_TIME_LIMIT_DELTA =
223       "hbase.region.server.rpc.minimum.scan.time.limit.delta";
224   /**
225    * Default value of {@link RSRpcServices#REGION_SERVER_RPC_MINIMUM_SCAN_TIME_LIMIT_DELTA}
226    */
227   private static final long DEFAULT_REGION_SERVER_RPC_MINIMUM_SCAN_TIME_LIMIT_DELTA = 10;
228 
229   // Request counter. (Includes requests that are not serviced by regions.)
230   final Counter requestCount = new Counter();
231 
232   // Request counter for rpc get
233   final Counter rpcGetRequestCount = new Counter();
234 
235   // Request counter for rpc scan
236   final Counter rpcScanRequestCount = new Counter();
237 
238   // Request counter for rpc multi
239   final Counter rpcMultiRequestCount = new Counter();
240 
241   // Request counter for rpc mutate
242   final Counter rpcMutateRequestCount = new Counter();
243 
244   // Server to handle client requests.
245   final RpcServerInterface rpcServer;
246   final InetSocketAddress isa;
247 
248   private final HRegionServer regionServer;
249   private final long maxScannerResultSize;
250 
251   // The reference to the priority extraction function
252   private final PriorityFunction priority;
253 
254   private final AtomicLong scannerIdGen = new AtomicLong(0L);
255   private final ConcurrentHashMap<String, RegionScannerHolder> scanners =
256     new ConcurrentHashMap<String, RegionScannerHolder>();
257 
258   /**
259    * The lease timeout period for client scanners (milliseconds).
260    */
261   private final int scannerLeaseTimeoutPeriod;
262 
263   /**
264    * The RPC timeout period (milliseconds)
265    */
266   private final int rpcTimeout;
267 
268   /**
269    * The minimum allowable delta to use for the scan limit
270    */
271   private final long minimumScanTimeLimitDelta;
272 
273   /**
274    * Holder class which holds the RegionScanner and nextCallSeq together.
275    */
276   private static class RegionScannerHolder {
277     private AtomicLong nextCallSeq = new AtomicLong(0);
278     private RegionScanner s;
279     private Region r;
280 
281     public RegionScannerHolder(RegionScanner s, Region r) {
282       this.s = s;
283       this.r = r;
284     }
285 
286     private long getNextCallSeq() {
287       return nextCallSeq.get();
288     }
289 
290     private void incNextCallSeq() {
291       nextCallSeq.incrementAndGet();
292     }
293 
294     private void rollbackNextCallSeq() {
295       nextCallSeq.decrementAndGet();
296     }
297   }
298 
299   /**
300    * Instantiated as a scanner lease. If the lease times out, the scanner is
301    * closed
302    */
303   private class ScannerListener implements LeaseListener {
304     private final String scannerName;
305 
306     ScannerListener(final String n) {
307       this.scannerName = n;
308     }
309 
310     @Override
311     public void leaseExpired() {
312       RegionScannerHolder rsh = scanners.remove(this.scannerName);
313       if (rsh != null) {
314         RegionScanner s = rsh.s;
315         LOG.info("Scanner " + this.scannerName + " lease expired on region "
316           + s.getRegionInfo().getRegionNameAsString());
317         Region region = null;
318         try {
319           region = regionServer.getRegion(s.getRegionInfo().getRegionName());
320           if (region != null && region.getCoprocessorHost() != null) {
321             region.getCoprocessorHost().preScannerClose(s);
322           }
323 
324           s.close();
325           if (region != null && region.getCoprocessorHost() != null) {
326             region.getCoprocessorHost().postScannerClose(s);
327           }
328         } catch (IOException e) {
329           LOG.error("Closing scanner for "
330               + s.getRegionInfo().getRegionNameAsString(), e);
331         } finally {
332           try {
333             s.close();
334             if (region != null && region.getCoprocessorHost() != null) {
335               region.getCoprocessorHost().postScannerClose(s);
336             }
337           } catch (IOException e) {
338             LOG.error("Closing scanner for " + s.getRegionInfo().getRegionNameAsString(), e);
339           }
340         }
341       } else {
342         LOG.warn("Scanner " + this.scannerName + " lease expired, but no related" +
343           " scanner found, hence no chance to close that related scanner!");
344       }
345     }
346   }
347 
348   private static ResultOrException getResultOrException(
349       final ClientProtos.Result r, final int index, final ClientProtos.RegionLoadStats stats) {
350     return getResultOrException(ResponseConverter.buildActionResult(r, stats), index);
351   }
352 
353   private static ResultOrException getResultOrException(final Exception e, final int index) {
354     return getResultOrException(ResponseConverter.buildActionResult(e), index);
355   }
356 
357   private static ResultOrException getResultOrException(
358       final ResultOrException.Builder builder, final int index) {
359     return builder.setIndex(index).build();
360   }
361 
362   /**
363    * Starts the nonce operation for a mutation, if needed.
364    * @param mutation Mutation.
365    * @param nonceGroup Nonce group from the request.
366    * @returns Nonce used (can be NO_NONCE).
367    */
368   private long startNonceOperation(final MutationProto mutation, long nonceGroup)
369       throws IOException, OperationConflictException {
370     if (regionServer.nonceManager == null || !mutation.hasNonce()) return HConstants.NO_NONCE;
371     boolean canProceed = false;
372     try {
373       canProceed = regionServer.nonceManager.startOperation(
374         nonceGroup, mutation.getNonce(), regionServer);
375     } catch (InterruptedException ex) {
376       throw new InterruptedIOException("Nonce start operation interrupted");
377     }
378     if (!canProceed) {
379       // TODO: instead, we could convert append/increment to get w/mvcc
380       String message = "The operation with nonce {" + nonceGroup + ", " + mutation.getNonce()
381         + "} on row [" + Bytes.toString(mutation.getRow().toByteArray())
382         + "] may have already completed";
383       throw new OperationConflictException(message);
384     }
385     return mutation.getNonce();
386   }
387 
388   /**
389    * Ends nonce operation for a mutation, if needed.
390    * @param mutation Mutation.
391    * @param nonceGroup Nonce group from the request. Always 0 in initial implementation.
392    * @param success Whether the operation for this nonce has succeeded.
393    */
394   private void endNonceOperation(final MutationProto mutation,
395       long nonceGroup, boolean success) {
396     if (regionServer.nonceManager != null && mutation.hasNonce()) {
397       regionServer.nonceManager.endOperation(nonceGroup, mutation.getNonce(), success);
398     }
399   }
400 
401   /**
402    * @return True if current call supports cellblocks
403    */
404   private boolean isClientCellBlockSupport() {
405     RpcCallContext context = RpcServer.getCurrentCall();
406     return context != null && context.isClientCellBlockSupport();
407   }
408 
409   private void addResult(final MutateResponse.Builder builder,
410       final Result result, final PayloadCarryingRpcController rpcc) {
411     if (result == null) return;
412     if (isClientCellBlockSupport()) {
413       builder.setResult(ProtobufUtil.toResultNoData(result));
414       rpcc.setCellScanner(result.cellScanner());
415     } else {
416       ClientProtos.Result pbr = ProtobufUtil.toResult(result);
417       builder.setResult(pbr);
418     }
419   }
420 
421   private void addResults(final ScanResponse.Builder builder, final List<Result> results,
422       final RpcController controller, boolean isDefaultRegion) {
423     builder.setStale(!isDefaultRegion);
424     if (results == null || results.isEmpty()) return;
425     if (isClientCellBlockSupport()) {
426       for (Result res : results) {
427         builder.addCellsPerResult(res.size());
428         builder.addPartialFlagPerResult(res.isPartial());
429       }
430       ((PayloadCarryingRpcController)controller).
431         setCellScanner(CellUtil.createCellScanner(results));
432     } else {
433       for (Result res: results) {
434         ClientProtos.Result pbr = ProtobufUtil.toResult(res);
435         builder.addResults(pbr);
436       }
437     }
438   }
439 
440   /**
441    * Mutate a list of rows atomically.
442    *
443    * @param region
444    * @param actions
445    * @param cellScanner if non-null, the mutation data -- the Cell content.
446    * @throws IOException
447    */
448   private ClientProtos.RegionLoadStats mutateRows(final Region region,
449       final List<ClientProtos.Action> actions,
450       final CellScanner cellScanner) throws IOException {
451     if (!region.getRegionInfo().isMetaTable()) {
452       regionServer.cacheFlusher.reclaimMemStoreMemory();
453     }
454     RowMutations rm = null;
455     for (ClientProtos.Action action: actions) {
456       if (action.hasGet()) {
457         throw new DoNotRetryIOException("Atomic put and/or delete only, not a Get=" +
458           action.getGet());
459       }
460       MutationType type = action.getMutation().getMutateType();
461       if (rm == null) {
462         rm = new RowMutations(action.getMutation().getRow().toByteArray());
463       }
464       switch (type) {
465       case PUT:
466         rm.add(ProtobufUtil.toPut(action.getMutation(), cellScanner));
467         break;
468       case DELETE:
469         rm.add(ProtobufUtil.toDelete(action.getMutation(), cellScanner));
470         break;
471       default:
472           throw new DoNotRetryIOException("Atomic put and/or delete only, not " + type.name());
473       }
474     }
475     region.mutateRow(rm);
476     return ((HRegion)region).getRegionStats();
477   }
478 
479   /**
480    * Mutate a list of rows atomically.
481    *
482    * @param region
483    * @param actions
484    * @param cellScanner if non-null, the mutation data -- the Cell content.
485    * @param row
486    * @param family
487    * @param qualifier
488    * @param compareOp
489    * @param comparator @throws IOException
490    */
491   private boolean checkAndRowMutate(final Region region, final List<ClientProtos.Action> actions,
492       final CellScanner cellScanner, byte[] row, byte[] family, byte[] qualifier,
493       CompareOp compareOp, ByteArrayComparable comparator,
494       ActivePolicyEnforcement spaceQuotaEnforcement) throws IOException {
495     if (!region.getRegionInfo().isMetaTable()) {
496       regionServer.cacheFlusher.reclaimMemStoreMemory();
497     }
498     RowMutations rm = null;
499     for (ClientProtos.Action action: actions) {
500       if (action.hasGet()) {
501         throw new DoNotRetryIOException("Atomic put and/or delete only, not a Get=" +
502             action.getGet());
503       }
504       MutationType type = action.getMutation().getMutateType();
505       if (rm == null) {
506         rm = new RowMutations(action.getMutation().getRow().toByteArray());
507       }
508       switch (type) {
509         case PUT:
510           Put put = ProtobufUtil.toPut(action.getMutation(), cellScanner);
511           spaceQuotaEnforcement.getPolicyEnforcement(region).check(put);
512           rm.add(put);
513           break;
514         case DELETE:
515           Delete del = ProtobufUtil.toDelete(action.getMutation(), cellScanner);
516           spaceQuotaEnforcement.getPolicyEnforcement(region).check(del);
517           rm.add(del);
518           break;
519         default:
520           throw new DoNotRetryIOException("Atomic put and/or delete only, not " + type.name());
521       }
522     }
523     return region.checkAndRowMutate(row, family, qualifier, compareOp, comparator, rm, Boolean.TRUE);
524   }
525 
526   /**
527    * Execute an append mutation.
528    *
529    * @param region
530    * @param m
531    * @param cellScanner
532    * @return result to return to client if default operation should be
533    * bypassed as indicated by RegionObserver, null otherwise
534    * @throws IOException
535    */
536   private Result append(final Region region, final OperationQuota quota,
537       final MutationProto m, final CellScanner cellScanner, long nonceGroup,
538       ActivePolicyEnforcement spaceQuota)
539       throws IOException {
540     long before = EnvironmentEdgeManager.currentTime();
541     Append append = ProtobufUtil.toAppend(m, cellScanner);
542     spaceQuota.getPolicyEnforcement(region).check(append);
543     quota.addMutation(append);
544     Result r = null;
545     if (region.getCoprocessorHost() != null) {
546       r = region.getCoprocessorHost().preAppend(append);
547     }
548     if (r == null) {
549       long nonce = startNonceOperation(m, nonceGroup);
550       boolean success = false;
551       try {
552         r = region.append(append, nonceGroup, nonce);
553         success = true;
554       } finally {
555         endNonceOperation(m, nonceGroup, success);
556       }
557       if (region.getCoprocessorHost() != null) {
558         region.getCoprocessorHost().postAppend(append, r);
559       }
560     }
561     if (regionServer.metricsRegionServer != null) {
562       regionServer.metricsRegionServer.updateAppend(
563         EnvironmentEdgeManager.currentTime() - before);
564     }
565     return r;
566   }
567 
568   /**
569    * Execute an increment mutation.
570    *
571    * @param region
572    * @param mutation
573    * @return the Result
574    * @throws IOException
575    */
576   private Result increment(final Region region, final OperationQuota quota,
577       final MutationProto mutation, final CellScanner cells, long nonceGroup,
578       ActivePolicyEnforcement spaceQuota) throws IOException {
579     long before = EnvironmentEdgeManager.currentTime();
580     Increment increment = ProtobufUtil.toIncrement(mutation, cells);
581     spaceQuota.getPolicyEnforcement(region).check(increment);
582     quota.addMutation(increment);
583     Result r = null;
584     if (region.getCoprocessorHost() != null) {
585       r = region.getCoprocessorHost().preIncrement(increment);
586     }
587     if (r == null) {
588       long nonce = startNonceOperation(mutation, nonceGroup);
589       boolean success = false;
590       try {
591         r = region.increment(increment, nonceGroup, nonce);
592         success = true;
593       } finally {
594         endNonceOperation(mutation, nonceGroup, success);
595       }
596       if (region.getCoprocessorHost() != null) {
597         r = region.getCoprocessorHost().postIncrement(increment, r);
598       }
599     }
600     if (regionServer.metricsRegionServer != null) {
601       regionServer.metricsRegionServer.updateIncrement(
602         EnvironmentEdgeManager.currentTime() - before);
603     }
604     return r;
605   }
606 
607   /**
608    * Run through the regionMutation <code>rm</code> and per Mutation, do the work, and then when
609    * done, add an instance of a {@link ResultOrException} that corresponds to each Mutation.
610    * @param region
611    * @param actions
612    * @param cellScanner
613    * @param builder
614    * @param cellsToReturn  Could be null. May be allocated in this method.  This is what this
615    * method returns as a 'result'.
616    * @return Return the <code>cellScanner</code> passed
617    */
618   private List<CellScannable> doNonAtomicRegionMutation(final Region region,
619       final OperationQuota quota, final RegionAction actions, final CellScanner cellScanner,
620       final RegionActionResult.Builder builder, List<CellScannable> cellsToReturn, long nonceGroup,
621       ActivePolicyEnforcement spaceQuotaEnforcement) {
622     // Gather up CONTIGUOUS Puts and Deletes in this mutations List.  Idea is that rather than do
623     // one at a time, we instead pass them in batch.  Be aware that the corresponding
624     // ResultOrException instance that matches each Put or Delete is then added down in the
625     // doBatchOp call.  We should be staying aligned though the Put and Delete are deferred/batched
626     List<ClientProtos.Action> mutations = null;
627     for (ClientProtos.Action action: actions.getActionList()) {
628       ClientProtos.ResultOrException.Builder resultOrExceptionBuilder = null;
629       try {
630         Result r = null;
631         if (action.hasGet()) {
632           Get get = ProtobufUtil.toGet(action.getGet());
633           r = region.get(get);
634         } else if (action.hasServiceCall()) {
635           resultOrExceptionBuilder = ResultOrException.newBuilder();
636           try {
637             Message result = execServiceOnRegion(region, action.getServiceCall());
638             ClientProtos.CoprocessorServiceResult.Builder serviceResultBuilder =
639                 ClientProtos.CoprocessorServiceResult.newBuilder();
640             resultOrExceptionBuilder.setServiceResult(
641                 serviceResultBuilder.setValue(
642                   serviceResultBuilder.getValueBuilder()
643                     .setName(result.getClass().getName())
644                     .setValue(result.toByteString())));
645           } catch (IOException ioe) {
646             rpcServer.getMetrics().exception(ioe);
647             resultOrExceptionBuilder.setException(ResponseConverter.buildException(ioe));
648           }
649         } else if (action.hasMutation()) {
650           MutationType type = action.getMutation().getMutateType();
651           if (type != MutationType.PUT && type != MutationType.DELETE && mutations != null &&
652               !mutations.isEmpty()) {
653             // Flush out any Puts or Deletes already collected.
654             doBatchOp(builder, region, quota, mutations, cellScanner, spaceQuotaEnforcement);
655             mutations.clear();
656           }
657           switch (type) {
658             case APPEND:
659               r = append(region, quota, action.getMutation(), cellScanner, nonceGroup,
660                   spaceQuotaEnforcement);
661               break;
662             case INCREMENT:
663               r = increment(region, quota, action.getMutation(), cellScanner, nonceGroup,
664                   spaceQuotaEnforcement);
665               break;
666             case PUT:
667             case DELETE:
668               // Collect the individual mutations and apply in a batch
669               if (mutations == null) {
670                 mutations = new ArrayList<ClientProtos.Action>(actions.getActionCount());
671               }
672               mutations.add(action);
673               break;
674             default:
675               throw new DoNotRetryIOException("Unsupported mutate type: " + type.name());
676           }
677         } else {
678           throw new HBaseIOException("Unexpected Action type");
679         }
680         if (r != null) {
681           ClientProtos.Result pbResult = null;
682           if (isClientCellBlockSupport()) {
683             pbResult = ProtobufUtil.toResultNoData(r);
684             //  Hard to guess the size here.  Just make a rough guess.
685             if (cellsToReturn == null) cellsToReturn = new ArrayList<CellScannable>();
686             cellsToReturn.add(r);
687           } else {
688             pbResult = ProtobufUtil.toResult(r);
689           }
690           resultOrExceptionBuilder =
691             ClientProtos.ResultOrException.newBuilder().setResult(pbResult);
692         }
693         // Could get to here and there was no result and no exception.  Presumes we added
694         // a Put or Delete to the collecting Mutations List for adding later.  In this
695         // case the corresponding ResultOrException instance for the Put or Delete will be added
696         // down in the doBatchOp method call rather than up here.
697       } catch (IOException ie) {
698         rpcServer.getMetrics().exception(ie);
699         resultOrExceptionBuilder = ResultOrException.newBuilder().
700           setException(ResponseConverter.buildException(ie));
701       }
702       if (resultOrExceptionBuilder != null) {
703         // Propagate index.
704         resultOrExceptionBuilder.setIndex(action.getIndex());
705         builder.addResultOrException(resultOrExceptionBuilder.build());
706       }
707     }
708     // Finish up any outstanding mutations
709     if (mutations != null && !mutations.isEmpty()) {
710       doBatchOp(builder, region, quota, mutations, cellScanner, spaceQuotaEnforcement);
711     }
712     return cellsToReturn;
713   }
714 
715   /**
716    * Execute a list of Put/Delete mutations.
717    *
718    * @param builder
719    * @param region
720    * @param mutations
721    */
722   private void doBatchOp(final RegionActionResult.Builder builder, final Region region,
723       final OperationQuota quota, final List<ClientProtos.Action> mutations,
724       final CellScanner cells, ActivePolicyEnforcement spaceQuotaEnforcement) {
725     Mutation[] mArray = new Mutation[mutations.size()];
726     long before = EnvironmentEdgeManager.currentTime();
727     boolean batchContainsPuts = false, batchContainsDelete = false;
728     try {
729       int i = 0;
730       for (ClientProtos.Action action: mutations) {
731         MutationProto m = action.getMutation();
732         Mutation mutation;
733         if (m.getMutateType() == MutationType.PUT) {
734           mutation = ProtobufUtil.toPut(m, cells);
735           batchContainsPuts = true;
736         } else {
737           mutation = ProtobufUtil.toDelete(m, cells);
738           batchContainsDelete = true;
739         }
740         mArray[i++] = mutation;
741         // Check if a space quota disallows this mutation
742         spaceQuotaEnforcement.getPolicyEnforcement(region).check(mutation);
743         quota.addMutation(mutation);
744       }
745 
746       if (!region.getRegionInfo().isMetaTable()) {
747         regionServer.cacheFlusher.reclaimMemStoreMemory();
748       }
749 
750       OperationStatus codes[] = region.batchMutate(mArray, HConstants.NO_NONCE,
751         HConstants.NO_NONCE);
752       for (i = 0; i < codes.length; i++) {
753         int index = mutations.get(i).getIndex();
754         Exception e = null;
755         switch (codes[i].getOperationStatusCode()) {
756           case BAD_FAMILY:
757             e = new NoSuchColumnFamilyException(codes[i].getExceptionMsg());
758             builder.addResultOrException(getResultOrException(e, index));
759             break;
760 
761           case SANITY_CHECK_FAILURE:
762             e = new FailedSanityCheckException(codes[i].getExceptionMsg());
763             builder.addResultOrException(getResultOrException(e, index));
764             break;
765 
766           default:
767             e = new DoNotRetryIOException(codes[i].getExceptionMsg());
768             builder.addResultOrException(getResultOrException(e, index));
769             break;
770 
771           case SUCCESS:
772             builder.addResultOrException(getResultOrException(
773               ClientProtos.Result.getDefaultInstance(), index,
774                 ((HRegion)region).getRegionStats()));
775             break;
776         }
777       }
778     } catch (IOException ie) {
779       for (int i = 0; i < mutations.size(); i++) {
780         builder.addResultOrException(getResultOrException(ie, mutations.get(i).getIndex()));
781       }
782     }
783     if (regionServer.metricsRegionServer != null) {
784       long after = EnvironmentEdgeManager.currentTime();
785       if (batchContainsPuts) {
786         regionServer.metricsRegionServer.updatePut(after - before);
787       }
788       if (batchContainsDelete) {
789         regionServer.metricsRegionServer.updateDelete(after - before);
790       }
791     }
792   }
793 
794   /**
795    * Execute a list of Put/Delete mutations. The function returns OperationStatus instead of
796    * constructing MultiResponse to save a possible loop if caller doesn't need MultiResponse.
797    * @param region
798    * @param mutations
799    * @param replaySeqId
800    * @return an array of OperationStatus which internally contains the OperationStatusCode and the
801    *         exceptionMessage if any
802    * @throws IOException
803    */
804   private OperationStatus [] doReplayBatchOp(final Region region,
805       final List<WALSplitter.MutationReplay> mutations, long replaySeqId) throws IOException {
806     long before = EnvironmentEdgeManager.currentTime();
807     boolean batchContainsPuts = false, batchContainsDelete = false;
808     try {
809       for (Iterator<WALSplitter.MutationReplay> it = mutations.iterator(); it.hasNext();) {
810         WALSplitter.MutationReplay m = it.next();
811 
812         if (m.type == MutationType.PUT) {
813           batchContainsPuts = true;
814         } else {
815           batchContainsDelete = true;
816         }
817 
818         NavigableMap<byte[], List<Cell>> map = m.mutation.getFamilyCellMap();
819         List<Cell> metaCells = map.get(WALEdit.METAFAMILY);
820         if (metaCells != null && !metaCells.isEmpty()) {
821           for (Cell metaCell : metaCells) {
822             CompactionDescriptor compactionDesc = WALEdit.getCompaction(metaCell);
823             boolean isDefaultReplica = RegionReplicaUtil.isDefaultReplica(region.getRegionInfo());
824             HRegion hRegion = (HRegion)region;
825             if (compactionDesc != null) {
826               // replay the compaction. Remove the files from stores only if we are the primary
827               // region replica (thus own the files)
828               hRegion.replayWALCompactionMarker(compactionDesc, !isDefaultReplica, isDefaultReplica,
829                 replaySeqId);
830               continue;
831             }
832             FlushDescriptor flushDesc = WALEdit.getFlushDescriptor(metaCell);
833             if (flushDesc != null && !isDefaultReplica) {
834               hRegion.replayWALFlushMarker(flushDesc, replaySeqId);
835               continue;
836             }
837             RegionEventDescriptor regionEvent = WALEdit.getRegionEventDescriptor(metaCell);
838             if (regionEvent != null && !isDefaultReplica) {
839               hRegion.replayWALRegionEventMarker(regionEvent);
840               continue;
841             }
842             BulkLoadDescriptor bulkLoadEvent = WALEdit.getBulkLoadDescriptor(metaCell);
843             if (bulkLoadEvent != null) {
844               hRegion.replayWALBulkLoadEventMarker(bulkLoadEvent);
845               continue;
846             }
847           }
848           it.remove();
849         }
850       }
851       requestCount.add(mutations.size());
852       if (!region.getRegionInfo().isMetaTable()) {
853         regionServer.cacheFlusher.reclaimMemStoreMemory();
854       }
855       return region.batchReplay(mutations.toArray(
856         new WALSplitter.MutationReplay[mutations.size()]), replaySeqId);
857     } finally {
858       if (regionServer.metricsRegionServer != null) {
859         long after = EnvironmentEdgeManager.currentTime();
860           if (batchContainsPuts) {
861           regionServer.metricsRegionServer.updatePut(after - before);
862         }
863         if (batchContainsDelete) {
864           regionServer.metricsRegionServer.updateDelete(after - before);
865         }
866       }
867     }
868   }
869 
870   private void closeAllScanners() {
871     // Close any outstanding scanners. Means they'll get an UnknownScanner
872     // exception next time they come in.
873     for (Map.Entry<String, RegionScannerHolder> e : scanners.entrySet()) {
874       try {
875         e.getValue().s.close();
876       } catch (IOException ioe) {
877         LOG.warn("Closing scanner " + e.getKey(), ioe);
878       }
879     }
880   }
881 
882   public RSRpcServices(HRegionServer rs) throws IOException {
883     regionServer = rs;
884 
885     RpcSchedulerFactory rpcSchedulerFactory;
886     try {
887       Class<?> rpcSchedulerFactoryClass = rs.conf.getClass(
888           REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS,
889           SimpleRpcSchedulerFactory.class);
890       rpcSchedulerFactory = ((RpcSchedulerFactory) rpcSchedulerFactoryClass.newInstance());
891     } catch (InstantiationException e) {
892       throw new IllegalArgumentException(e);
893     } catch (IllegalAccessException e) {
894       throw new IllegalArgumentException(e);
895     }
896     // Server to handle client requests.
897     InetSocketAddress initialIsa;
898     InetSocketAddress bindAddress;
899     if(this instanceof MasterRpcServices) {
900       String hostname = getHostname(rs.conf, true);
901       int port = rs.conf.getInt(HConstants.MASTER_PORT, HConstants.DEFAULT_MASTER_PORT);
902       // Creation of a HSA will force a resolve.
903       initialIsa = new InetSocketAddress(hostname, port);
904       bindAddress = new InetSocketAddress(rs.conf.get("hbase.master.ipc.address", hostname), port);
905     } else {
906       String hostname = getHostname(rs.conf, false);
907       int port = rs.conf.getInt(HConstants.REGIONSERVER_PORT,
908         HConstants.DEFAULT_REGIONSERVER_PORT);
909       // Creation of a HSA will force a resolve.
910       initialIsa = new InetSocketAddress(hostname, port);
911       bindAddress = new InetSocketAddress(
912         rs.conf.get("hbase.regionserver.ipc.address", hostname), port);
913     }
914     if (initialIsa.getAddress() == null) {
915       throw new IllegalArgumentException("Failed resolve of " + initialIsa);
916     }
917     priority = new AnnotationReadingPriorityFunction(this);
918     String name = rs.getProcessName() + "/" + initialIsa.toString();
919     // Set how many times to retry talking to another server over HConnection.
920     ConnectionUtils.setServerSideHConnectionRetriesConfig(rs.conf, name, LOG);
921     rpcServer = new RpcServer(rs, name, getServices(),
922       bindAddress, // use final bindAddress for this server.
923       rs.conf,
924       rpcSchedulerFactory.create(rs.conf, this, rs));
925 
926     scannerLeaseTimeoutPeriod = rs.conf.getInt(
927       HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD,
928       HConstants.DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD);
929     maxScannerResultSize = rs.conf.getLong(
930       HConstants.HBASE_SERVER_SCANNER_MAX_RESULT_SIZE_KEY,
931       HConstants.DEFAULT_HBASE_SERVER_SCANNER_MAX_RESULT_SIZE);
932     rpcTimeout = rs.conf.getInt(
933       HConstants.HBASE_RPC_TIMEOUT_KEY,
934       HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
935     minimumScanTimeLimitDelta = rs.conf.getLong(
936       REGION_SERVER_RPC_MINIMUM_SCAN_TIME_LIMIT_DELTA,
937       DEFAULT_REGION_SERVER_RPC_MINIMUM_SCAN_TIME_LIMIT_DELTA);
938 
939     InetSocketAddress address = rpcServer.getListenerAddress();
940     if (address == null) {
941       throw new IOException("Listener channel is closed");
942     }
943     // Set our address, however we need the final port that was given to rpcServer
944     isa = new InetSocketAddress(initialIsa.getHostName(), address.getPort());
945     rpcServer.setErrorHandler(this);
946     rs.setName(name);
947   }
948 
949   public static String getHostname(Configuration conf, boolean isMaster)
950       throws UnknownHostException {
951     String hostname = conf.get(isMaster? HRegionServer.MASTER_HOSTNAME_KEY :
952       HRegionServer.RS_HOSTNAME_KEY);
953     if (hostname == null || hostname.isEmpty()) {
954       String masterOrRS = isMaster ? "master" : "regionserver";
955       return Strings.domainNamePointerToHostName(DNS.getDefaultHost(
956         conf.get("hbase." + masterOrRS + ".dns.interface", "default"),
957         conf.get("hbase." + masterOrRS + ".dns.nameserver", "default")));
958     } else {
959       LOG.info("hostname is configured to be " + hostname);
960       return hostname;
961     }
962   }
963 
964   RegionScanner getScanner(long scannerId) {
965     String scannerIdString = Long.toString(scannerId);
966     RegionScannerHolder scannerHolder = scanners.get(scannerIdString);
967     if (scannerHolder != null) {
968       return scannerHolder.s;
969     }
970     return null;
971   }
972 
973   /**
974    * Get the vtime associated with the scanner.
975    * Currently the vtime is the number of "next" calls.
976    */
977   long getScannerVirtualTime(long scannerId) {
978     String scannerIdString = Long.toString(scannerId);
979     RegionScannerHolder scannerHolder = scanners.get(scannerIdString);
980     if (scannerHolder != null) {
981       return scannerHolder.getNextCallSeq();
982     }
983     return 0L;
984   }
985 
986   long addScanner(RegionScanner s, Region r) throws LeaseStillHeldException {
987     long scannerId = this.scannerIdGen.incrementAndGet();
988     String scannerName = String.valueOf(scannerId);
989 
990     RegionScannerHolder existing =
991       scanners.putIfAbsent(scannerName, new RegionScannerHolder(s, r));
992     assert existing == null : "scannerId must be unique within regionserver's whole lifecycle!";
993 
994     regionServer.leases.createLease(scannerName, this.scannerLeaseTimeoutPeriod,
995         new ScannerListener(scannerName));
996     return scannerId;
997   }
998 
999   /**
1000    * Find the HRegion based on a region specifier
1001    *
1002    * @param regionSpecifier the region specifier
1003    * @return the corresponding region
1004    * @throws IOException if the specifier is not null,
1005    *    but failed to find the region
1006    */
1007   Region getRegion(
1008       final RegionSpecifier regionSpecifier) throws IOException {
1009     ByteString value = regionSpecifier.getValue();
1010     RegionSpecifierType type = regionSpecifier.getType();
1011     switch (type) {
1012       case REGION_NAME:
1013         byte[] regionName = value.toByteArray();
1014         String encodedRegionName = HRegionInfo.encodeRegionName(regionName);
1015         return regionServer.getRegionByEncodedName(regionName, encodedRegionName);
1016       case ENCODED_REGION_NAME:
1017         return regionServer.getRegionByEncodedName(value.toStringUtf8());
1018       default:
1019         throw new DoNotRetryIOException(
1020           "Unsupported region specifier type: " + type);
1021     }
1022   }
1023 
1024   @VisibleForTesting
1025   public PriorityFunction getPriority() {
1026     return priority;
1027   }
1028 
1029   Configuration getConfiguration() {
1030     return regionServer.getConfiguration();
1031   }
1032 
1033   private RegionServerRpcQuotaManager getRpcQuotaManager() {
1034     return regionServer.getRegionServerRpcQuotaManager();
1035   }
1036 
1037   private RegionServerSpaceQuotaManager getSpaceQuotaManager() {
1038     return regionServer.getRegionServerSpaceQuotaManager();
1039   }
1040 
1041   void start() {
1042     rpcServer.start();
1043   }
1044 
1045   void stop() {
1046     closeAllScanners();
1047     rpcServer.stop();
1048   }
1049 
1050   /**
1051    * Called to verify that this server is up and running.
1052    *
1053    * @throws IOException
1054    */
1055   protected void checkOpen() throws IOException {
1056     if (regionServer.isAborted()) {
1057       throw new RegionServerAbortedException("Server " + regionServer.serverName + " aborting");
1058     }
1059     if (regionServer.isStopped()) {
1060       throw new RegionServerStoppedException("Server " + regionServer.serverName + " stopping");
1061     }
1062     if (!regionServer.fsOk) {
1063       throw new RegionServerStoppedException("File system not available");
1064     }
1065     if (!regionServer.isOnline()) {
1066       throw new ServerNotRunningYetException("Server " + regionServer.serverName
1067           + " is not running yet");
1068     }
1069   }
1070 
1071   /**
1072    * @return list of blocking services and their security info classes that this server supports
1073    */
1074   protected List<BlockingServiceAndInterface> getServices() {
1075     List<BlockingServiceAndInterface> bssi = new ArrayList<BlockingServiceAndInterface>(2);
1076     bssi.add(new BlockingServiceAndInterface(
1077       ClientService.newReflectiveBlockingService(this),
1078       ClientService.BlockingInterface.class));
1079     bssi.add(new BlockingServiceAndInterface(
1080       AdminService.newReflectiveBlockingService(this),
1081       AdminService.BlockingInterface.class));
1082     return bssi;
1083   }
1084 
1085   public InetSocketAddress getSocketAddress() {
1086     return isa;
1087   }
1088 
1089   @Override
1090   public int getPriority(RequestHeader header, Message param) {
1091     return priority.getPriority(header, param);
1092   }
1093 
1094   @Override
1095   public long getDeadline(RequestHeader header, Message param) {
1096     return priority.getDeadline(header, param);
1097   }
1098 
1099   /*
1100    * Check if an OOME and, if so, abort immediately to avoid creating more objects.
1101    *
1102    * @param e
1103    *
1104    * @return True if we OOME'd and are aborting.
1105    */
1106   @Override
1107   public boolean checkOOME(final Throwable e) {
1108     boolean stop = false;
1109     try {
1110       if (e instanceof OutOfMemoryError
1111           || (e.getCause() != null && e.getCause() instanceof OutOfMemoryError)
1112           || (e.getMessage() != null && e.getMessage().contains(
1113               "java.lang.OutOfMemoryError"))) {
1114         stop = true;
1115         LOG.fatal("Run out of memory; " + getClass().getSimpleName()
1116           + " will abort itself immediately", e);
1117       }
1118     } finally {
1119       if (stop) {
1120         Runtime.getRuntime().halt(1);
1121       }
1122     }
1123     return stop;
1124   }
1125 
1126   /**
1127    * Close a region on the region server.
1128    *
1129    * @param controller the RPC controller
1130    * @param request the request
1131    * @throws ServiceException
1132    */
1133   @Override
1134   @QosPriority(priority=HConstants.ADMIN_QOS)
1135   public CloseRegionResponse closeRegion(final RpcController controller,
1136       final CloseRegionRequest request) throws ServiceException {
1137     final ServerName sn = (request.hasDestinationServer() ?
1138       ProtobufUtil.toServerName(request.getDestinationServer()) : null);
1139 
1140     try {
1141       checkOpen();
1142       if (request.hasServerStartCode()) {
1143         // check that we are the same server that this RPC is intended for.
1144         long serverStartCode = request.getServerStartCode();
1145         if (regionServer.serverName.getStartcode() !=  serverStartCode) {
1146           throw new ServiceException(new DoNotRetryIOException("This RPC was intended for a " +
1147               "different server with startCode: " + serverStartCode + ", this server is: "
1148               + regionServer.serverName));
1149         }
1150       }
1151       final String encodedRegionName = ProtobufUtil.getRegionEncodedName(request.getRegion());
1152 
1153       // Can be null if we're calling close on a region that's not online
1154       final Region region = regionServer.getFromOnlineRegions(encodedRegionName);
1155       if ((region  != null) && (region .getCoprocessorHost() != null)) {
1156         region.getCoprocessorHost().preClose(false);
1157       }
1158 
1159       requestCount.increment();
1160       LOG.info("Close " + encodedRegionName + ", moving to " + sn);
1161       CloseRegionCoordination.CloseRegionDetails crd = regionServer.getCoordinatedStateManager()
1162         .getCloseRegionCoordination().parseFromProtoRequest(request);
1163 
1164       boolean closed = regionServer.closeRegion(encodedRegionName, false, crd, sn);
1165       CloseRegionResponse.Builder builder = CloseRegionResponse.newBuilder().setClosed(closed);
1166       return builder.build();
1167     } catch (IOException ie) {
1168       throw new ServiceException(ie);
1169     }
1170   }
1171 
1172   /**
1173    * Compact a region on the region server.
1174    *
1175    * @param controller the RPC controller
1176    * @param request the request
1177    * @throws ServiceException
1178    */
1179   @Override
1180   @QosPriority(priority=HConstants.ADMIN_QOS)
1181   public CompactRegionResponse compactRegion(final RpcController controller,
1182       final CompactRegionRequest request) throws ServiceException {
1183     try {
1184       checkOpen();
1185       requestCount.increment();
1186       Region region = getRegion(request.getRegion());
1187       // Quota support is enabled, the requesting user is not system/super user
1188       // and a quota policy is enforced that disables compactions.
1189       if (QuotaUtil.isQuotaEnabled(getConfiguration()) &&
1190           !Superusers.isSuperUser(RpcServer.getRequestUser()) &&
1191           this.regionServer.getRegionServerSpaceQuotaManager().areCompactionsDisabled(
1192               region.getTableDesc().getTableName())) {
1193         throw new DoNotRetryIOException("Compactions on this region are "
1194             + "disabled due to a space quota violation.");
1195       }
1196       region.startRegionOperation(Operation.COMPACT_REGION);
1197       LOG.info("Compacting " + region.getRegionInfo().getRegionNameAsString());
1198       boolean major = false;
1199       byte [] family = null;
1200       Store store = null;
1201       if (request.hasFamily()) {
1202         family = request.getFamily().toByteArray();
1203         store = region.getStore(family);
1204         if (store == null) {
1205           throw new ServiceException(new IOException("column family " + Bytes.toString(family)
1206             + " does not exist in region " + region.getRegionInfo().getRegionNameAsString()));
1207         }
1208       }
1209       if (request.hasMajor()) {
1210         major = request.getMajor();
1211       }
1212       if (major) {
1213         if (family != null) {
1214           store.triggerMajorCompaction();
1215         } else {
1216           region.triggerMajorCompaction();
1217         }
1218       }
1219 
1220       String familyLogMsg = (family != null)?" for column family: " + Bytes.toString(family):"";
1221       if (LOG.isTraceEnabled()) {
1222         LOG.trace("User-triggered compaction requested for region "
1223           + region.getRegionInfo().getRegionNameAsString() + familyLogMsg);
1224       }
1225       String log = "User-triggered " + (major ? "major " : "") + "compaction" + familyLogMsg;
1226       if(family != null) {
1227         regionServer.compactSplitThread.requestCompaction(region, store, log,
1228           Store.PRIORITY_USER, null, RpcServer.getRequestUser());
1229       } else {
1230         regionServer.compactSplitThread.requestCompaction(region, log,
1231           Store.PRIORITY_USER, null, RpcServer.getRequestUser());
1232       }
1233       return CompactRegionResponse.newBuilder().build();
1234     } catch (IOException ie) {
1235       throw new ServiceException(ie);
1236     }
1237   }
1238 
1239   /**
1240    * Flush a region on the region server.
1241    *
1242    * @param controller the RPC controller
1243    * @param request the request
1244    * @throws ServiceException
1245    */
1246   @Override
1247   @QosPriority(priority=HConstants.ADMIN_QOS)
1248   public FlushRegionResponse flushRegion(final RpcController controller,
1249       final FlushRegionRequest request) throws ServiceException {
1250     try {
1251       checkOpen();
1252       requestCount.increment();
1253       Region region = getRegion(request.getRegion());
1254       LOG.info("Flushing " + region.getRegionInfo().getRegionNameAsString());
1255       boolean shouldFlush = true;
1256       if (request.hasIfOlderThanTs()) {
1257         shouldFlush = region.getEarliestFlushTimeForAllStores() < request.getIfOlderThanTs();
1258       }
1259       FlushRegionResponse.Builder builder = FlushRegionResponse.newBuilder();
1260       if (shouldFlush) {
1261         boolean writeFlushWalMarker =  request.hasWriteFlushWalMarker() ?
1262             request.getWriteFlushWalMarker() : false;
1263         // Go behind the curtain so we can manage writing of the flush WAL marker
1264         HRegion.FlushResultImpl flushResult = (HRegion.FlushResultImpl)
1265             ((HRegion)region).flushcache(true, writeFlushWalMarker);
1266         boolean compactionNeeded = flushResult.isCompactionNeeded();
1267         if (compactionNeeded) {
1268           regionServer.compactSplitThread.requestSystemCompaction(region,
1269             "Compaction through user triggered flush");
1270         }
1271         builder.setFlushed(flushResult.isFlushSucceeded());
1272         builder.setWroteFlushWalMarker(flushResult.wroteFlushWalMarker);
1273       }
1274       builder.setLastFlushTime(region.getEarliestFlushTimeForAllStores());
1275       return builder.build();
1276     } catch (DroppedSnapshotException ex) {
1277       // Cache flush can fail in a few places. If it fails in a critical
1278       // section, we get a DroppedSnapshotException and a replay of wal
1279       // is required. Currently the only way to do this is a restart of
1280       // the server.
1281       regionServer.abort("Replay of WAL required. Forcing server shutdown", ex);
1282       throw new ServiceException(ex);
1283     } catch (IOException ie) {
1284       throw new ServiceException(ie);
1285     }
1286   }
1287 
1288   @Override
1289   @QosPriority(priority=HConstants.ADMIN_QOS)
1290   public GetOnlineRegionResponse getOnlineRegion(final RpcController controller,
1291       final GetOnlineRegionRequest request) throws ServiceException {
1292     try {
1293       checkOpen();
1294       requestCount.increment();
1295       Map<String, Region> onlineRegions = regionServer.onlineRegions;
1296       List<HRegionInfo> list = new ArrayList<HRegionInfo>(onlineRegions.size());
1297       for (Region region: onlineRegions.values()) {
1298         list.add(region.getRegionInfo());
1299       }
1300       Collections.sort(list);
1301       return ResponseConverter.buildGetOnlineRegionResponse(list);
1302     } catch (IOException ie) {
1303       throw new ServiceException(ie);
1304     }
1305   }
1306 
1307   @Override
1308   @QosPriority(priority=HConstants.ADMIN_QOS)
1309   public GetRegionInfoResponse getRegionInfo(final RpcController controller,
1310       final GetRegionInfoRequest request) throws ServiceException {
1311     try {
1312       checkOpen();
1313       requestCount.increment();
1314       Region region = getRegion(request.getRegion());
1315       HRegionInfo info = region.getRegionInfo();
1316       GetRegionInfoResponse.Builder builder = GetRegionInfoResponse.newBuilder();
1317       builder.setRegionInfo(HRegionInfo.convert(info));
1318       if (request.hasCompactionState() && request.getCompactionState()) {
1319         builder.setCompactionState(region.getCompactionState());
1320       }
1321       builder.setIsRecovering(region.isRecovering());
1322       return builder.build();
1323     } catch (IOException ie) {
1324       throw new ServiceException(ie);
1325     }
1326   }
1327 
1328   /**
1329    * Get some information of the region server.
1330    *
1331    * @param controller the RPC controller
1332    * @param request the request
1333    * @throws ServiceException
1334    */
1335   @Override
1336   @QosPriority(priority=HConstants.ADMIN_QOS)
1337   public GetServerInfoResponse getServerInfo(final RpcController controller,
1338       final GetServerInfoRequest request) throws ServiceException {
1339     try {
1340       checkOpen();
1341     } catch (IOException ie) {
1342       throw new ServiceException(ie);
1343     }
1344     requestCount.increment();
1345     int infoPort = regionServer.infoServer != null ? regionServer.infoServer.getPort() : -1;
1346     return ResponseConverter.buildGetServerInfoResponse(regionServer.serverName, infoPort);
1347   }
1348 
1349   @Override
1350   @QosPriority(priority=HConstants.ADMIN_QOS)
1351   public GetStoreFileResponse getStoreFile(final RpcController controller,
1352       final GetStoreFileRequest request) throws ServiceException {
1353     try {
1354       checkOpen();
1355       Region region = getRegion(request.getRegion());
1356       requestCount.increment();
1357       Set<byte[]> columnFamilies;
1358       if (request.getFamilyCount() == 0) {
1359         columnFamilies = region.getTableDesc().getFamiliesKeys();
1360       } else {
1361         columnFamilies = new TreeSet<byte[]>(Bytes.BYTES_RAWCOMPARATOR);
1362         for (ByteString cf: request.getFamilyList()) {
1363           columnFamilies.add(cf.toByteArray());
1364         }
1365       }
1366       int nCF = columnFamilies.size();
1367       List<String>  fileList = region.getStoreFileList(
1368         columnFamilies.toArray(new byte[nCF][]));
1369       GetStoreFileResponse.Builder builder = GetStoreFileResponse.newBuilder();
1370       builder.addAllStoreFile(fileList);
1371       return builder.build();
1372     } catch (IOException ie) {
1373       throw new ServiceException(ie);
1374     }
1375   }
1376 
1377   /**
1378    * Merge regions on the region server.
1379    *
1380    * @param controller the RPC controller
1381    * @param request the request
1382    * @return merge regions response
1383    * @throws ServiceException
1384    */
1385   @Override
1386   @QosPriority(priority = HConstants.ADMIN_QOS)
1387   public MergeRegionsResponse mergeRegions(final RpcController controller,
1388       final MergeRegionsRequest request) throws ServiceException {
1389     try {
1390       checkOpen();
1391       requestCount.increment();
1392       Region regionA = getRegion(request.getRegionA());
1393       Region regionB = getRegion(request.getRegionB());
1394       boolean forcible = request.getForcible();
1395       long masterSystemTime = request.hasMasterSystemTime() ? request.getMasterSystemTime() : -1;
1396       regionA.startRegionOperation(Operation.MERGE_REGION);
1397       regionB.startRegionOperation(Operation.MERGE_REGION);
1398       if (regionA.getRegionInfo().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID ||
1399           regionB.getRegionInfo().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1400         throw new ServiceException(new MergeRegionException("Can't merge non-default replicas"));
1401       }
1402       LOG.info("Receiving merging request for  " + regionA + ", " + regionB
1403           + ",forcible=" + forcible);
1404       regionA.flush(true);
1405       regionB.flush(true);
1406       regionServer.compactSplitThread.requestRegionsMerge(regionA, regionB, forcible,
1407           masterSystemTime, RpcServer.getRequestUser());
1408       return MergeRegionsResponse.newBuilder().build();
1409     } catch (DroppedSnapshotException ex) {
1410       regionServer.abort("Replay of WAL required. Forcing server shutdown", ex);
1411       throw new ServiceException(ex);
1412     } catch (IOException ie) {
1413       throw new ServiceException(ie);
1414     }
1415   }
1416 
1417   /**
1418    * Open asynchronously a region or a set of regions on the region server.
1419    *
1420    * The opening is coordinated by ZooKeeper, and this method requires the znode to be created
1421    *  before being called. As a consequence, this method should be called only from the master.
1422    * <p>
1423    * Different manages states for the region are:<ul>
1424    *  <li>region not opened: the region opening will start asynchronously.</li>
1425    *  <li>a close is already in progress: this is considered as an error.</li>
1426    *  <li>an open is already in progress: this new open request will be ignored. This is important
1427    *  because the Master can do multiple requests if it crashes.</li>
1428    *  <li>the region is already opened:  this new open request will be ignored./li>
1429    *  </ul>
1430    * </p>
1431    * <p>
1432    * Bulk assign: If there are more than 1 region to open, it will be considered as a bulk assign.
1433    * For a single region opening, errors are sent through a ServiceException. For bulk assign,
1434    * errors are put in the response as FAILED_OPENING.
1435    * </p>
1436    * @param controller the RPC controller
1437    * @param request the request
1438    * @throws ServiceException
1439    */
1440   @Override
1441   @QosPriority(priority=HConstants.ADMIN_QOS)
1442   public OpenRegionResponse openRegion(final RpcController controller,
1443       final OpenRegionRequest request) throws ServiceException {
1444     requestCount.increment();
1445     if (request.hasServerStartCode()) {
1446       // check that we are the same server that this RPC is intended for.
1447       long serverStartCode = request.getServerStartCode();
1448       if (regionServer.serverName.getStartcode() !=  serverStartCode) {
1449         throw new ServiceException(new DoNotRetryIOException("This RPC was intended for a " +
1450             "different server with startCode: " + serverStartCode + ", this server is: "
1451             + regionServer.serverName));
1452       }
1453     }
1454 
1455     OpenRegionResponse.Builder builder = OpenRegionResponse.newBuilder();
1456     final int regionCount = request.getOpenInfoCount();
1457     final Map<TableName, HTableDescriptor> htds =
1458         new HashMap<TableName, HTableDescriptor>(regionCount);
1459     final boolean isBulkAssign = regionCount > 1;
1460     try {
1461       checkOpen();
1462     } catch (IOException ie) {
1463       TableName tableName = null;
1464       if (regionCount == 1) {
1465         RegionInfo ri = request.getOpenInfo(0).getRegion();
1466         if (ri != null) {
1467           tableName = ProtobufUtil.toTableName(ri.getTableName());
1468         }
1469       }
1470       if (!TableName.META_TABLE_NAME.equals(tableName)) {
1471         throw new ServiceException(ie);
1472       }
1473       // We are assigning meta, wait a little for regionserver to finish initialization.
1474       int timeout = regionServer.conf.getInt(HConstants.HBASE_RPC_TIMEOUT_KEY,
1475         HConstants.DEFAULT_HBASE_RPC_TIMEOUT) >> 2; // Quarter of RPC timeout
1476       long endTime = System.currentTimeMillis() + timeout;
1477       synchronized (regionServer.online) {
1478         try {
1479           while (System.currentTimeMillis() <= endTime
1480               && !regionServer.isStopped() && !regionServer.isOnline()) {
1481             regionServer.online.wait(regionServer.msgInterval);
1482           }
1483           checkOpen();
1484         } catch (InterruptedException t) {
1485           Thread.currentThread().interrupt();
1486           throw new ServiceException(t);
1487         } catch (IOException e) {
1488           throw new ServiceException(e);
1489         }
1490       }
1491     }
1492 
1493     long masterSystemTime = request.hasMasterSystemTime() ? request.getMasterSystemTime() : -1;
1494 
1495     for (RegionOpenInfo regionOpenInfo : request.getOpenInfoList()) {
1496       final HRegionInfo region = HRegionInfo.convert(regionOpenInfo.getRegion());
1497       OpenRegionCoordination coordination = regionServer.getCoordinatedStateManager().
1498         getOpenRegionCoordination();
1499       OpenRegionCoordination.OpenRegionDetails ord =
1500         coordination.parseFromProtoRequest(regionOpenInfo);
1501 
1502       HTableDescriptor htd;
1503       try {
1504         final Region onlineRegion = regionServer.getFromOnlineRegions(region.getEncodedName());
1505         if (onlineRegion != null) {
1506           //Check if the region can actually be opened.
1507           if (onlineRegion.getCoprocessorHost() != null) {
1508             onlineRegion.getCoprocessorHost().preOpen();
1509           }
1510           // See HBASE-5094. Cross check with hbase:meta if still this RS is owning
1511           // the region.
1512           Pair<HRegionInfo, ServerName> p = MetaTableAccessor.getRegion(
1513             regionServer.getConnection(), region.getRegionName());
1514           if (regionServer.serverName.equals(p.getSecond())) {
1515             Boolean closing = regionServer.regionsInTransitionInRS.get(region.getEncodedNameAsBytes());
1516             // Map regionsInTransitionInRSOnly has an entry for a region only if the region
1517             // is in transition on this RS, so here closing can be null. If not null, it can
1518             // be true or false. True means the region is opening on this RS; while false
1519             // means the region is closing. Only return ALREADY_OPENED if not closing (i.e.
1520             // not in transition any more, or still transition to open.
1521             if (!Boolean.FALSE.equals(closing)
1522                 && regionServer.getFromOnlineRegions(region.getEncodedName()) != null) {
1523               LOG.warn("Attempted open of " + region.getEncodedName()
1524                 + " but already online on this server");
1525               builder.addOpeningState(RegionOpeningState.ALREADY_OPENED);
1526               continue;
1527             }
1528           } else {
1529             LOG.warn("The region " + region.getEncodedName() + " is online on this server"
1530               + " but hbase:meta does not have this server - continue opening.");
1531             regionServer.removeFromOnlineRegions(onlineRegion, null);
1532           }
1533         }
1534         LOG.info("Open " + region.getRegionNameAsString());
1535         htd = htds.get(region.getTable());
1536         if (htd == null) {
1537           htd = regionServer.tableDescriptors.get(region.getTable());
1538           htds.put(region.getTable(), htd);
1539         }
1540 
1541         final Boolean previous = regionServer.regionsInTransitionInRS.putIfAbsent(
1542           region.getEncodedNameAsBytes(), Boolean.TRUE);
1543 
1544         if (Boolean.FALSE.equals(previous)) {
1545           // There is a close in progress. We need to mark this open as failed in ZK.
1546 
1547           coordination.tryTransitionFromOfflineToFailedOpen(regionServer, region, ord);
1548 
1549           throw new RegionAlreadyInTransitionException("Received OPEN for the region:"
1550             + region.getRegionNameAsString() + " , which we are already trying to CLOSE ");
1551         }
1552 
1553         if (Boolean.TRUE.equals(previous)) {
1554           // An open is in progress. This is supported, but let's log this.
1555           LOG.info("Receiving OPEN for the region:" +
1556             region.getRegionNameAsString() + " , which we are already trying to OPEN"
1557               + " - ignoring this new request for this region.");
1558         }
1559 
1560         // We are opening this region. If it moves back and forth for whatever reason, we don't
1561         // want to keep returning the stale moved record while we are opening/if we close again.
1562         regionServer.removeFromMovedRegions(region.getEncodedName());
1563 
1564         if (previous == null) {
1565           // check if the region to be opened is marked in recovering state in ZK
1566           if (ZKSplitLog.isRegionMarkedRecoveringInZK(regionServer.getZooKeeper(),
1567               region.getEncodedName())) {
1568             // Check if current region open is for distributedLogReplay. This check is to support
1569             // rolling restart/upgrade where we want to Master/RS see same configuration
1570             if (!regionOpenInfo.hasOpenForDistributedLogReplay()
1571                   || regionOpenInfo.getOpenForDistributedLogReplay()) {
1572               regionServer.recoveringRegions.put(region.getEncodedName(), null);
1573             } else {
1574               // Remove stale recovery region from ZK when we open region not for recovering which
1575               // could happen when turn distributedLogReplay off from on.
1576               List<String> tmpRegions = new ArrayList<String>();
1577               tmpRegions.add(region.getEncodedName());
1578               ZKSplitLog.deleteRecoveringRegionZNodes(regionServer.getZooKeeper(),
1579                 tmpRegions);
1580             }
1581           }
1582           if (htd == null) {
1583             throw new IOException("Missing table descriptor for " + region.getEncodedName());
1584           }
1585           // If there is no action in progress, we can submit a specific handler.
1586           // Need to pass the expected version in the constructor.
1587           if (region.isMetaRegion()) {
1588             regionServer.service.submit(new OpenMetaHandler(
1589               regionServer, regionServer, region, htd, masterSystemTime, coordination, ord));
1590           } else {
1591             regionServer.updateRegionFavoredNodesMapping(region.getEncodedName(),
1592               regionOpenInfo.getFavoredNodesList());
1593             if (htd.getPriority() >= HConstants.ADMIN_QOS || region.getTable().isSystemTable()) {
1594               regionServer.service.submit(new OpenPriorityRegionHandler(
1595                 regionServer, regionServer, region, htd, masterSystemTime, coordination, ord));
1596             } else {
1597               regionServer.service.submit(new OpenRegionHandler(
1598                 regionServer, regionServer, region, htd, masterSystemTime, coordination, ord));
1599             }
1600           }
1601         }
1602 
1603         builder.addOpeningState(RegionOpeningState.OPENED);
1604 
1605       } catch (KeeperException zooKeeperEx) {
1606         LOG.error("Can't retrieve recovering state from zookeeper", zooKeeperEx);
1607         throw new ServiceException(zooKeeperEx);
1608       } catch (IOException ie) {
1609         LOG.warn("Failed opening region " + region.getRegionNameAsString(), ie);
1610         if (isBulkAssign) {
1611           builder.addOpeningState(RegionOpeningState.FAILED_OPENING);
1612         } else {
1613           throw new ServiceException(ie);
1614         }
1615       }
1616     }
1617     return builder.build();
1618   }
1619 
1620   /**
1621    *  Wamrmup a region on this server.
1622    *
1623    * This method should only be called by Master. It synchrnously opens the region and
1624    * closes the region bringing the most important pages in cache.
1625    * <p>
1626    *
1627    * @param controller the RPC controller
1628    * @param request the request
1629    * @throws ServiceException
1630    */
1631   @Override
1632   public WarmupRegionResponse warmupRegion(final RpcController controller,
1633       final WarmupRegionRequest request) throws ServiceException {
1634 
1635     RegionInfo regionInfo = request.getRegionInfo();
1636     final HRegionInfo region = HRegionInfo.convert(regionInfo);
1637     HTableDescriptor htd;
1638     WarmupRegionResponse response = WarmupRegionResponse.getDefaultInstance();
1639 
1640     try {
1641       checkOpen();
1642       String encodedName = region.getEncodedName();
1643       byte[] encodedNameBytes = region.getEncodedNameAsBytes();
1644       final Region onlineRegion = regionServer.getFromOnlineRegions(encodedName);
1645 
1646       if (onlineRegion != null) {
1647         LOG.info("Region already online. Skipping warming up " + region);
1648         return response;
1649       }
1650 
1651       if (LOG.isDebugEnabled()) {
1652         LOG.debug("Warming up Region " + region.getRegionNameAsString());
1653       }
1654 
1655       htd = regionServer.tableDescriptors.get(region.getTable());
1656 
1657       if (regionServer.getRegionsInTransitionInRS().containsKey(encodedNameBytes)) {
1658         LOG.info("Region is in transition. Skipping warmup " + region);
1659         return response;
1660       }
1661 
1662       HRegion.warmupHRegion(region, htd, regionServer.getWAL(region),
1663           regionServer.getConfiguration(), regionServer, null);
1664 
1665     } catch (IOException ie) {
1666       LOG.error("Failed warming up region " + region.getRegionNameAsString(), ie);
1667       throw new ServiceException(ie);
1668     }
1669 
1670     return response;
1671   }
1672 
1673   /**
1674    * Replay the given changes when distributedLogReplay WAL edits from a failed RS. The guarantee is
1675    * that the given mutations will be durable on the receiving RS if this method returns without any
1676    * exception.
1677    * @param controller the RPC controller
1678    * @param request the request
1679    * @throws ServiceException
1680    */
1681   @Override
1682   @QosPriority(priority = HConstants.REPLAY_QOS)
1683   public ReplicateWALEntryResponse replay(final RpcController controller,
1684       final ReplicateWALEntryRequest request) throws ServiceException {
1685     long before = EnvironmentEdgeManager.currentTime();
1686     CellScanner cells = ((PayloadCarryingRpcController) controller).cellScanner();
1687     try {
1688       checkOpen();
1689       List<WALEntry> entries = request.getEntryList();
1690       if (entries == null || entries.isEmpty()) {
1691         // empty input
1692         return ReplicateWALEntryResponse.newBuilder().build();
1693       }
1694       ByteString regionName = entries.get(0).getKey().getEncodedRegionName();
1695       Region region = regionServer.getRegionByEncodedName(regionName.toStringUtf8());
1696       RegionCoprocessorHost coprocessorHost =
1697           ServerRegionReplicaUtil.isDefaultReplica(region.getRegionInfo())
1698             ? region.getCoprocessorHost()
1699             : null; // do not invoke coprocessors if this is a secondary region replica
1700       List<Pair<WALKey, WALEdit>> walEntries = new ArrayList<Pair<WALKey, WALEdit>>();
1701 
1702       // Skip adding the edits to WAL if this is a secondary region replica
1703       boolean isPrimary = RegionReplicaUtil.isDefaultReplica(region.getRegionInfo());
1704       Durability durability = isPrimary ? Durability.USE_DEFAULT : Durability.SKIP_WAL;
1705 
1706       for (WALEntry entry : entries) {
1707         if (!regionName.equals(entry.getKey().getEncodedRegionName())) {
1708           throw new NotServingRegionException("Replay request contains entries from multiple " +
1709               "regions. First region:" + regionName.toStringUtf8() + " , other region:"
1710               + entry.getKey().getEncodedRegionName());
1711         }
1712         if (regionServer.nonceManager != null && isPrimary) {
1713           long nonceGroup = entry.getKey().hasNonceGroup()
1714             ? entry.getKey().getNonceGroup() : HConstants.NO_NONCE;
1715           long nonce = entry.getKey().hasNonce() ? entry.getKey().getNonce() : HConstants.NO_NONCE;
1716           regionServer.nonceManager.reportOperationFromWal(nonceGroup, nonce, entry.getKey().getWriteTime());
1717         }
1718         Pair<WALKey, WALEdit> walEntry = (coprocessorHost == null) ? null :
1719           new Pair<WALKey, WALEdit>();
1720         List<WALSplitter.MutationReplay> edits = WALSplitter.getMutationsFromWALEntry(entry,
1721           cells, walEntry, durability);
1722         if (coprocessorHost != null) {
1723           // Start coprocessor replay here. The coprocessor is for each WALEdit instead of a
1724           // KeyValue.
1725           if (coprocessorHost.preWALRestore(region.getRegionInfo(), walEntry.getFirst(),
1726             walEntry.getSecond())) {
1727             // if bypass this log entry, ignore it ...
1728             continue;
1729           }
1730           walEntries.add(walEntry);
1731         }
1732         if(edits!=null && !edits.isEmpty()) {
1733           long replaySeqId = (entry.getKey().hasOrigSequenceNumber()) ?
1734             entry.getKey().getOrigSequenceNumber() : entry.getKey().getLogSequenceNumber();
1735           OperationStatus[] result = doReplayBatchOp(region, edits, replaySeqId);
1736           // check if it's a partial success
1737           for (int i = 0; result != null && i < result.length; i++) {
1738             if (result[i] != OperationStatus.SUCCESS) {
1739               throw new IOException(result[i].getExceptionMsg());
1740             }
1741           }
1742         }
1743       }
1744 
1745       //sync wal at the end because ASYNC_WAL is used above
1746       WAL wal = getWAL(region);
1747       if (wal != null) {
1748         wal.sync();
1749       }
1750 
1751       if (coprocessorHost != null) {
1752         for (Pair<WALKey, WALEdit> entry : walEntries) {
1753           coprocessorHost.postWALRestore(region.getRegionInfo(), entry.getFirst(),
1754             entry.getSecond());
1755         }
1756       }
1757       return ReplicateWALEntryResponse.newBuilder().build();
1758     } catch (IOException ie) {
1759       throw new ServiceException(ie);
1760     } finally {
1761       if (regionServer.metricsRegionServer != null) {
1762         regionServer.metricsRegionServer.updateReplay(
1763           EnvironmentEdgeManager.currentTime() - before);
1764       }
1765     }
1766   }
1767 
1768   WAL getWAL(Region region) {
1769     return ((HRegion)region).getWAL();
1770   }
1771 
1772   /**
1773    * Replicate WAL entries on the region server.
1774    *
1775    * @param controller the RPC controller
1776    * @param request the request
1777    * @throws ServiceException
1778    */
1779   @Override
1780   @QosPriority(priority=HConstants.REPLICATION_QOS)
1781   public ReplicateWALEntryResponse replicateWALEntry(final RpcController controller,
1782       final ReplicateWALEntryRequest request) throws ServiceException {
1783     try {
1784       checkOpen();
1785       if (regionServer.replicationSinkHandler != null) {
1786         requestCount.increment();
1787         List<WALEntry> entries = request.getEntryList();
1788         CellScanner cellScanner = ((PayloadCarryingRpcController)controller).cellScanner();
1789         regionServer.getRegionServerCoprocessorHost().preReplicateLogEntries(entries, cellScanner);
1790         regionServer.replicationSinkHandler.replicateLogEntries(entries, cellScanner,
1791           request.getReplicationClusterId(), request.getSourceBaseNamespaceDirPath(),
1792           request.getSourceHFileArchiveDirPath());
1793         regionServer.getRegionServerCoprocessorHost().postReplicateLogEntries(entries, cellScanner);
1794         return ReplicateWALEntryResponse.newBuilder().build();
1795       } else {
1796         throw new ServiceException("Replication services are not initialized yet");
1797       }
1798     } catch (IOException ie) {
1799       throw new ServiceException(ie);
1800     }
1801   }
1802 
1803   /**
1804    * Roll the WAL writer of the region server.
1805    * @param controller the RPC controller
1806    * @param request the request
1807    * @throws ServiceException
1808    */
1809   @Override
1810   public RollWALWriterResponse rollWALWriter(final RpcController controller,
1811       final RollWALWriterRequest request) throws ServiceException {
1812     try {
1813       checkOpen();
1814       requestCount.increment();
1815       regionServer.getRegionServerCoprocessorHost().preRollWALWriterRequest();
1816       regionServer.walRoller.requestRollAll();
1817       regionServer.getRegionServerCoprocessorHost().postRollWALWriterRequest();
1818       RollWALWriterResponse.Builder builder = RollWALWriterResponse.newBuilder();
1819       return builder.build();
1820     } catch (IOException ie) {
1821       throw new ServiceException(ie);
1822     }
1823   }
1824 
1825   /**
1826    * Split a region on the region server.
1827    *
1828    * @param controller the RPC controller
1829    * @param request the request
1830    * @throws ServiceException
1831    */
1832   @Override
1833   @QosPriority(priority=HConstants.ADMIN_QOS)
1834   public SplitRegionResponse splitRegion(final RpcController controller,
1835       final SplitRegionRequest request) throws ServiceException {
1836     try {
1837       checkOpen();
1838       requestCount.increment();
1839       Region region = getRegion(request.getRegion());
1840       region.startRegionOperation(Operation.SPLIT_REGION);
1841       if (region.getRegionInfo().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1842         throw new IOException("Can't split replicas directly. "
1843             + "Replicas are auto-split when their primary is split.");
1844       }
1845       LOG.info("Splitting " + region.getRegionInfo().getRegionNameAsString());
1846       region.flush(true);
1847       byte[] splitPoint = null;
1848       if (request.hasSplitPoint()) {
1849         splitPoint = request.getSplitPoint().toByteArray();
1850       }
1851       ((HRegion)region).forceSplit(splitPoint);
1852       regionServer.compactSplitThread.requestSplit(region, ((HRegion)region).checkSplit(),
1853         RpcServer.getRequestUser());
1854       return SplitRegionResponse.newBuilder().build();
1855     } catch (DroppedSnapshotException ex) {
1856       regionServer.abort("Replay of WAL required. Forcing server shutdown", ex);
1857       throw new ServiceException(ex);
1858     } catch (IOException ie) {
1859       throw new ServiceException(ie);
1860     }
1861   }
1862 
1863   /**
1864    * Stop the region server.
1865    *
1866    * @param controller the RPC controller
1867    * @param request the request
1868    * @throws ServiceException
1869    */
1870   @Override
1871   @QosPriority(priority=HConstants.ADMIN_QOS)
1872   public StopServerResponse stopServer(final RpcController controller,
1873       final StopServerRequest request) throws ServiceException {
1874     requestCount.increment();
1875     String reason = request.getReason();
1876     regionServer.stop(reason);
1877     return StopServerResponse.newBuilder().build();
1878   }
1879 
1880   @Override
1881   public UpdateFavoredNodesResponse updateFavoredNodes(RpcController controller,
1882       UpdateFavoredNodesRequest request) throws ServiceException {
1883     List<UpdateFavoredNodesRequest.RegionUpdateInfo> openInfoList = request.getUpdateInfoList();
1884     UpdateFavoredNodesResponse.Builder respBuilder = UpdateFavoredNodesResponse.newBuilder();
1885     for (UpdateFavoredNodesRequest.RegionUpdateInfo regionUpdateInfo : openInfoList) {
1886       HRegionInfo hri = HRegionInfo.convert(regionUpdateInfo.getRegion());
1887       regionServer.updateRegionFavoredNodesMapping(hri.getEncodedName(),
1888         regionUpdateInfo.getFavoredNodesList());
1889     }
1890     respBuilder.setResponse(openInfoList.size());
1891     return respBuilder.build();
1892   }
1893 
1894   /**
1895    * Atomically bulk load several HFiles into an open region
1896    * @return true if successful, false is failed but recoverably (no action)
1897    * @throws IOException if failed unrecoverably
1898    */
1899   @Override
1900   public BulkLoadHFileResponse bulkLoadHFile(final RpcController controller,
1901       final BulkLoadHFileRequest request) throws ServiceException {
1902     try {
1903       checkOpen();
1904       requestCount.increment();
1905       Region region = getRegion(request.getRegion());
1906       List<Pair<byte[], String>> familyPaths = new ArrayList<Pair<byte[], String>>();
1907       for (FamilyPath familyPath: request.getFamilyPathList()) {
1908         familyPaths.add(new Pair<byte[], String>(familyPath.getFamily().toByteArray(),
1909           familyPath.getPath()));
1910       }
1911       boolean bypass = false;
1912       if (region.getCoprocessorHost() != null) {
1913         bypass = region.getCoprocessorHost().preBulkLoadHFile(familyPaths);
1914       }
1915       boolean loaded = false;
1916 
1917       // Check to see if this bulk load would exceed the space quota for this table
1918       if (QuotaUtil.isQuotaEnabled(getConfiguration())) {
1919         ActivePolicyEnforcement activeSpaceQuotas = getSpaceQuotaManager().getActiveEnforcements();
1920         SpaceViolationPolicyEnforcement enforcement = activeSpaceQuotas.getPolicyEnforcement(
1921             region);
1922         if (enforcement != null) {
1923           // Bulk loads must still be atomic. We must enact all or none.
1924           List<String> filePaths = new ArrayList<>(request.getFamilyPathCount());
1925           for (FamilyPath familyPath : request.getFamilyPathList()) {
1926             filePaths.add(familyPath.getPath());
1927           }
1928           // Check if the batch of files exceeds the current quota
1929           enforcement.checkBulkLoad(regionServer.getFileSystem(), filePaths);
1930         }
1931       }
1932 
1933       Map<byte[], List<Path>> map = null;
1934       if (!bypass) {
1935         map = region.bulkLoadHFiles(familyPaths, request.getAssignSeqNum(), null,
1936             request.getCopyFile());
1937         if (map != null) {
1938           loaded = true;
1939         }
1940       }
1941       if (region.getCoprocessorHost() != null) {
1942         loaded = region.getCoprocessorHost().postBulkLoadHFile(familyPaths, map, loaded);
1943       }
1944       BulkLoadHFileResponse.Builder builder = BulkLoadHFileResponse.newBuilder();
1945       if (map != null) {
1946         loaded = true;
1947       }
1948       builder.setLoaded(loaded);
1949       return builder.build();
1950     } catch (IOException ie) {
1951       throw new ServiceException(ie);
1952     }
1953   }
1954 
1955   @Override
1956   public CoprocessorServiceResponse execService(final RpcController controller,
1957       final CoprocessorServiceRequest request) throws ServiceException {
1958     try {
1959       checkOpen();
1960       requestCount.increment();
1961       Region region = getRegion(request.getRegion());
1962       Message result = execServiceOnRegion(region, request.getCall());
1963       CoprocessorServiceResponse.Builder builder =
1964         CoprocessorServiceResponse.newBuilder();
1965       builder.setRegion(RequestConverter.buildRegionSpecifier(
1966         RegionSpecifierType.REGION_NAME, region.getRegionInfo().getRegionName()));
1967       builder.setValue(
1968         builder.getValueBuilder().setName(result.getClass().getName())
1969           .setValue(result.toByteString()));
1970       return builder.build();
1971     } catch (IOException ie) {
1972       throw new ServiceException(ie);
1973     }
1974   }
1975 
1976   private Message execServiceOnRegion(Region region,
1977       final ClientProtos.CoprocessorServiceCall serviceCall) throws IOException {
1978     // ignore the passed in controller (from the serialized call)
1979     ServerRpcController execController = new ServerRpcController();
1980     Message result = region.execService(execController, serviceCall);
1981     if (execController.getFailedOn() != null) {
1982       throw execController.getFailedOn();
1983     }
1984     return result;
1985   }
1986 
1987   /**
1988    * Get data from a table.
1989    *
1990    * @param controller the RPC controller
1991    * @param request the get request
1992    * @throws ServiceException
1993    */
1994   @Override
1995   public GetResponse get(final RpcController controller,
1996       final GetRequest request) throws ServiceException {
1997     long before = EnvironmentEdgeManager.currentTime();
1998     OperationQuota quota = null;
1999     try {
2000       checkOpen();
2001       requestCount.increment();
2002       rpcGetRequestCount.increment();
2003       Region region = getRegion(request.getRegion());
2004 
2005       GetResponse.Builder builder = GetResponse.newBuilder();
2006       ClientProtos.Get get = request.getGet();
2007       Boolean existence = null;
2008       Result r = null;
2009       quota = getRpcQuotaManager().checkQuota(region, OperationQuota.OperationType.GET);
2010 
2011       if (get.hasClosestRowBefore() && get.getClosestRowBefore()) {
2012         if (get.getColumnCount() != 1) {
2013           throw new DoNotRetryIOException(
2014             "get ClosestRowBefore supports one and only one family now, not "
2015               + get.getColumnCount() + " families");
2016         }
2017         byte[] row = get.getRow().toByteArray();
2018         byte[] family = get.getColumn(0).getFamily().toByteArray();
2019         r = region.getClosestRowBefore(row, family);
2020       } else {
2021         Get clientGet = ProtobufUtil.toGet(get);
2022         if (get.getExistenceOnly() && region.getCoprocessorHost() != null) {
2023           existence = region.getCoprocessorHost().preExists(clientGet);
2024         }
2025         if (existence == null) {
2026           r = region.get(clientGet);
2027           if (get.getExistenceOnly()) {
2028             boolean exists = r.getExists();
2029             if (region.getCoprocessorHost() != null) {
2030               exists = region.getCoprocessorHost().postExists(clientGet, exists);
2031             }
2032             existence = exists;
2033           }
2034         }
2035       }
2036       if (existence != null){
2037         ClientProtos.Result pbr =
2038             ProtobufUtil.toResult(existence, region.getRegionInfo().getReplicaId() != 0);
2039         builder.setResult(pbr);
2040       } else  if (r != null) {
2041         ClientProtos.Result pbr = ProtobufUtil.toResult(r);
2042         builder.setResult(pbr);
2043       }
2044       if (r != null) {
2045         quota.addGetResult(r);
2046       }
2047       return builder.build();
2048     } catch (IOException ie) {
2049       throw new ServiceException(ie);
2050     } finally {
2051       if (regionServer.metricsRegionServer != null) {
2052         regionServer.metricsRegionServer.updateGet(
2053           EnvironmentEdgeManager.currentTime() - before);
2054       }
2055       if (quota != null) {
2056         quota.close();
2057       }
2058     }
2059   }
2060 
2061   /**
2062    * Execute multiple actions on a table: get, mutate, and/or execCoprocessor
2063    *
2064    * @param rpcc the RPC controller
2065    * @param request the multi request
2066    * @throws ServiceException
2067    */
2068   @Override
2069   public MultiResponse multi(final RpcController rpcc, final MultiRequest request)
2070   throws ServiceException {
2071     try {
2072       checkOpen();
2073     } catch (IOException ie) {
2074       throw new ServiceException(ie);
2075     }
2076 
2077     // rpc controller is how we bring in data via the back door;  it is unprotobuf'ed data.
2078     // It is also the conduit via which we pass back data.
2079     PayloadCarryingRpcController controller = (PayloadCarryingRpcController)rpcc;
2080     CellScanner cellScanner = controller != null ? controller.cellScanner(): null;
2081     if (controller != null) controller.setCellScanner(null);
2082 
2083     long nonceGroup = request.hasNonceGroup() ? request.getNonceGroup() : HConstants.NO_NONCE;
2084 
2085     // this will contain all the cells that we need to return. It's created later, if needed.
2086     List<CellScannable> cellsToReturn = null;
2087     MultiResponse.Builder responseBuilder = MultiResponse.newBuilder();
2088     RegionActionResult.Builder regionActionResultBuilder = RegionActionResult.newBuilder();
2089     Boolean processed = null;
2090 
2091     RpcCallContext context = RpcServer.getCurrentCall();
2092     this.rpcMultiRequestCount.increment();
2093     Map<RegionSpecifier, ClientProtos.RegionLoadStats> regionStats = new HashMap<>(request
2094       .getRegionActionCount());
2095     ActivePolicyEnforcement spaceQuotaEnforcement = getSpaceQuotaManager().getActiveEnforcements();
2096     for (RegionAction regionAction : request.getRegionActionList()) {
2097       this.requestCount.add(regionAction.getActionCount());
2098       OperationQuota quota;
2099       Region region;
2100       regionActionResultBuilder.clear();
2101       try {
2102         region = getRegion(regionAction.getRegion());
2103         quota = getRpcQuotaManager().checkQuota(region, regionAction.getActionList());
2104       } catch (IOException e) {
2105         rpcServer.getMetrics().exception(e);
2106         regionActionResultBuilder.setException(ResponseConverter.buildException(e));
2107         responseBuilder.addRegionActionResult(regionActionResultBuilder.build());
2108         if (cellScanner != null) {
2109           skipCellsForMutations(regionAction.getActionList(), cellScanner);
2110         }
2111         continue;  // For this region it's a failure.
2112       }
2113 
2114       if (regionAction.hasAtomic() && regionAction.getAtomic()) {
2115         // How does this call happen?  It may need some work to play well w/ the surroundings.
2116         // Need to return an item per Action along w/ Action index.  TODO.
2117         try {
2118           if (request.hasCondition()) {
2119             Condition condition = request.getCondition();
2120             byte[] row = condition.getRow().toByteArray();
2121             byte[] family = condition.getFamily().toByteArray();
2122             byte[] qualifier = condition.getQualifier().toByteArray();
2123             CompareOp compareOp = CompareOp.valueOf(condition.getCompareType().name());
2124             ByteArrayComparable comparator =
2125                 ProtobufUtil.toComparator(condition.getComparator());
2126             processed = checkAndRowMutate(region, regionAction.getActionList(),
2127                   cellScanner, row, family, qualifier, compareOp,
2128                   comparator, spaceQuotaEnforcement);
2129           } else {
2130             ClientProtos.RegionLoadStats stats = mutateRows(region, regionAction.getActionList(),
2131                 cellScanner);
2132             // add the stats to the request
2133             if(stats != null) {
2134               responseBuilder.addRegionActionResult(RegionActionResult.newBuilder()
2135                   .addResultOrException(ResultOrException.newBuilder().setLoadStats(stats)));
2136             }
2137             processed = Boolean.TRUE;
2138           }
2139         } catch (IOException e) {
2140           rpcServer.getMetrics().exception(e);
2141           // As it's atomic, we may expect it's a global failure.
2142           regionActionResultBuilder.setException(ResponseConverter.buildException(e));
2143         }
2144       } else {
2145         // doNonAtomicRegionMutation manages the exception internally
2146         cellsToReturn = doNonAtomicRegionMutation(region, quota, regionAction, cellScanner,
2147             regionActionResultBuilder, cellsToReturn, nonceGroup, spaceQuotaEnforcement);
2148       }
2149       responseBuilder.addRegionActionResult(regionActionResultBuilder.build());
2150       quota.close();
2151     }
2152     // Load the controller with the Cells to return.
2153     if (cellsToReturn != null && !cellsToReturn.isEmpty() && controller != null) {
2154       controller.setCellScanner(CellUtil.createCellScanner(cellsToReturn));
2155     }
2156     if (processed != null) responseBuilder.setProcessed(processed);
2157     return responseBuilder.build();
2158   }
2159 
2160   private void skipCellsForMutations(List<Action> actions, CellScanner cellScanner) {
2161     for (Action action : actions) {
2162       skipCellsForMutation(action, cellScanner);
2163     }
2164   }
2165 
2166   private void skipCellsForMutation(Action action, CellScanner cellScanner) {
2167     try {
2168       if (action.hasMutation()) {
2169         MutationProto m = action.getMutation();
2170         if (m.hasAssociatedCellCount()) {
2171           for (int i = 0; i < m.getAssociatedCellCount(); i++) {
2172             cellScanner.advance();
2173           }
2174         }
2175       }
2176     } catch (IOException e) {
2177       // No need to handle these Individual Muatation level issue. Any way this entire RegionAction
2178       // marked as failed as we could not see the Region here. At client side the top level
2179       // RegionAction exception will be considered first.
2180       LOG.error("Error while skipping Cells in CellScanner for invalid Region Mutations", e);
2181     }
2182   }
2183 
2184   /**
2185    * Mutate data in a table.
2186    *
2187    * @param rpcc the RPC controller
2188    * @param request the mutate request
2189    * @throws ServiceException
2190    */
2191   @Override
2192   public MutateResponse mutate(final RpcController rpcc,
2193       final MutateRequest request) throws ServiceException {
2194     // rpc controller is how we bring in data via the back door;  it is unprotobuf'ed data.
2195     // It is also the conduit via which we pass back data.
2196     PayloadCarryingRpcController controller = (PayloadCarryingRpcController)rpcc;
2197     CellScanner cellScanner = controller != null? controller.cellScanner(): null;
2198     OperationQuota quota = null;
2199     ActivePolicyEnforcement spaceQuotaEnforcement = null;
2200     // Clear scanner so we are not holding on to reference across call.
2201     if (controller != null) controller.setCellScanner(null);
2202     try {
2203       checkOpen();
2204       requestCount.increment();
2205       rpcMutateRequestCount.increment();
2206       Region region = getRegion(request.getRegion());
2207       MutateResponse.Builder builder = MutateResponse.newBuilder();
2208       MutationProto mutation = request.getMutation();
2209       if (!region.getRegionInfo().isMetaTable()) {
2210         regionServer.cacheFlusher.reclaimMemStoreMemory();
2211       }
2212       long nonceGroup = request.hasNonceGroup() ? request.getNonceGroup() : HConstants.NO_NONCE;
2213       Result r = null;
2214       Boolean processed = null;
2215       MutationType type = mutation.getMutateType();
2216       quota = getRpcQuotaManager().checkQuota(region, OperationQuota.OperationType.MUTATE);
2217       spaceQuotaEnforcement = getSpaceQuotaManager().getActiveEnforcements();
2218 
2219       switch (type) {
2220       case APPEND:
2221         // TODO: this doesn't actually check anything.
2222         r = append(region, quota, mutation, cellScanner, nonceGroup, spaceQuotaEnforcement);
2223         break;
2224       case INCREMENT:
2225         // TODO: this doesn't actually check anything.
2226         r = increment(region, quota, mutation, cellScanner, nonceGroup, spaceQuotaEnforcement);
2227         break;
2228       case PUT:
2229         Put put = ProtobufUtil.toPut(mutation, cellScanner);
2230         // Throws an exception when violated
2231         spaceQuotaEnforcement.getPolicyEnforcement(region).check(put);
2232         quota.addMutation(put);
2233         if (request.hasCondition()) {
2234           Condition condition = request.getCondition();
2235           byte[] row = condition.getRow().toByteArray();
2236           byte[] family = condition.getFamily().toByteArray();
2237           byte[] qualifier = condition.getQualifier().toByteArray();
2238           CompareOp compareOp = CompareOp.valueOf(condition.getCompareType().name());
2239           ByteArrayComparable comparator =
2240             ProtobufUtil.toComparator(condition.getComparator());
2241           if (region.getCoprocessorHost() != null) {
2242             processed = region.getCoprocessorHost().preCheckAndPut(
2243               row, family, qualifier, compareOp, comparator, put);
2244           }
2245           if (processed == null) {
2246             boolean result = region.checkAndMutate(row, family,
2247               qualifier, compareOp, comparator, put, true);
2248             if (region.getCoprocessorHost() != null) {
2249               result = region.getCoprocessorHost().postCheckAndPut(row, family,
2250                 qualifier, compareOp, comparator, put, result);
2251             }
2252             processed = result;
2253           }
2254         } else {
2255           region.put(put);
2256           processed = Boolean.TRUE;
2257         }
2258         break;
2259       case DELETE:
2260         Delete delete = ProtobufUtil.toDelete(mutation, cellScanner);
2261         spaceQuotaEnforcement.getPolicyEnforcement(region).check(delete);
2262         quota.addMutation(delete);
2263         if (request.hasCondition()) {
2264           Condition condition = request.getCondition();
2265           byte[] row = condition.getRow().toByteArray();
2266           byte[] family = condition.getFamily().toByteArray();
2267           byte[] qualifier = condition.getQualifier().toByteArray();
2268           CompareOp compareOp = CompareOp.valueOf(condition.getCompareType().name());
2269           ByteArrayComparable comparator =
2270             ProtobufUtil.toComparator(condition.getComparator());
2271           if (region.getCoprocessorHost() != null) {
2272             processed = region.getCoprocessorHost().preCheckAndDelete(
2273               row, family, qualifier, compareOp, comparator, delete);
2274           }
2275           if (processed == null) {
2276             boolean result = region.checkAndMutate(row, family,
2277               qualifier, compareOp, comparator, delete, true);
2278             if (region.getCoprocessorHost() != null) {
2279               result = region.getCoprocessorHost().postCheckAndDelete(row, family,
2280                 qualifier, compareOp, comparator, delete, result);
2281             }
2282             processed = result;
2283           }
2284         } else {
2285           region.delete(delete);
2286           processed = Boolean.TRUE;
2287         }
2288         break;
2289       default:
2290           throw new DoNotRetryIOException(
2291             "Unsupported mutate type: " + type.name());
2292       }
2293       if (processed != null) builder.setProcessed(processed.booleanValue());
2294       addResult(builder, r, controller);
2295       return builder.build();
2296     } catch (IOException ie) {
2297       regionServer.checkFileSystem();
2298       throw new ServiceException(ie);
2299     } finally {
2300       if (quota != null) {
2301         quota.close();
2302       }
2303     }
2304   }
2305 
2306   /**
2307    * Scan data in a table.
2308    *
2309    * @param controller the RPC controller
2310    * @param request the scan request
2311    * @throws ServiceException
2312    */
2313   @Override
2314   public ScanResponse scan(final RpcController controller, final ScanRequest request)
2315   throws ServiceException {
2316     OperationQuota quota = null;
2317     Leases.Lease lease = null;
2318     String scannerName = null;
2319     try {
2320       if (!request.hasScannerId() && !request.hasScan()) {
2321         throw new DoNotRetryIOException(
2322           "Missing required input: scannerId or scan");
2323       }
2324       long scannerId = -1;
2325       if (request.hasScannerId()) {
2326         scannerId = request.getScannerId();
2327         scannerName = String.valueOf(scannerId);
2328       }
2329       try {
2330         checkOpen();
2331       } catch (IOException e) {
2332         // If checkOpen failed, server not running or filesystem gone,
2333         // cancel this lease; filesystem is gone or we're closing or something.
2334         if (scannerName != null) {
2335           LOG.debug("Server shutting down and client tried to access missing scanner "
2336             + scannerName);
2337           if (regionServer.leases != null) {
2338             try {
2339               regionServer.leases.cancelLease(scannerName);
2340             } catch (LeaseException le) {
2341               // No problem, ignore
2342             }
2343           }
2344         }
2345         throw e;
2346       }
2347       requestCount.increment();
2348       rpcScanRequestCount.increment();
2349 
2350       int ttl = 0;
2351       Region region = null;
2352       RegionScanner scanner = null;
2353       RegionScannerHolder rsh = null;
2354       boolean moreResults = true;
2355       boolean closeScanner = false;
2356       boolean isSmallScan = false;
2357       ScanResponse.Builder builder = ScanResponse.newBuilder();
2358       if (request.hasCloseScanner()) {
2359         closeScanner = request.getCloseScanner();
2360       }
2361       int rows = closeScanner ? 0 : 1;
2362       if (request.hasNumberOfRows()) {
2363         rows = request.getNumberOfRows();
2364       }
2365       if (request.hasScannerId()) {
2366         rsh = scanners.get(scannerName);
2367         if (rsh == null) {
2368           LOG.info("Client tried to access missing scanner " + scannerName);
2369           throw new UnknownScannerException(
2370             "Name: " + scannerName + ", already closed?");
2371         }
2372         scanner = rsh.s;
2373         HRegionInfo hri = scanner.getRegionInfo();
2374         region = regionServer.getRegion(hri.getRegionName());
2375         if (region != rsh.r) { // Yes, should be the same instance
2376           throw new NotServingRegionException("Region was re-opened after the scanner"
2377             + scannerName + " was created: " + hri.getRegionNameAsString());
2378         }
2379       } else {
2380         region = getRegion(request.getRegion());
2381         ClientProtos.Scan protoScan = request.getScan();
2382         boolean isLoadingCfsOnDemandSet = protoScan.hasLoadColumnFamiliesOnDemand();
2383         Scan scan = ProtobufUtil.toScan(protoScan);
2384         // if the request doesn't set this, get the default region setting.
2385         if (!isLoadingCfsOnDemandSet) {
2386           scan.setLoadColumnFamiliesOnDemand(region.isLoadingCfsOnDemandDefault());
2387         }
2388 
2389         isSmallScan = scan.isSmall();
2390         if (!scan.hasFamilies()) {
2391           // Adding all families to scanner
2392           for (byte[] family: region.getTableDesc().getFamiliesKeys()) {
2393             scan.addFamily(family);
2394           }
2395         }
2396 
2397         if (region.getCoprocessorHost() != null) {
2398           scanner = region.getCoprocessorHost().preScannerOpen(scan);
2399         }
2400         if (scanner == null) {
2401           scanner = region.getScanner(scan);
2402         }
2403         if (region.getCoprocessorHost() != null) {
2404           scanner = region.getCoprocessorHost().postScannerOpen(scan, scanner);
2405         }
2406         scannerId = addScanner(scanner, region);
2407         scannerName = String.valueOf(scannerId);
2408         ttl = this.scannerLeaseTimeoutPeriod;
2409       }
2410       if (request.hasRenew() && request.getRenew()) {
2411         rsh = scanners.get(scannerName);
2412         lease = regionServer.leases.removeLease(scannerName);
2413         if (lease != null && rsh != null) {
2414           regionServer.leases.addLease(lease);
2415           // Increment the nextCallSeq value which is the next expected from client.
2416           rsh.incNextCallSeq();
2417         }
2418         return builder.build();
2419       }
2420 
2421       quota = getRpcQuotaManager().checkQuota(region, OperationQuota.OperationType.SCAN);
2422       long maxQuotaResultSize = Math.min(maxScannerResultSize, quota.getReadAvailable());
2423       if (rows > 0) {
2424         // if nextCallSeq does not match throw Exception straight away. This needs to be
2425         // performed even before checking of Lease.
2426         // See HBASE-5974
2427         if (request.hasNextCallSeq()) {
2428           if (rsh == null) {
2429             rsh = scanners.get(scannerName);
2430           }
2431           if (rsh != null) {
2432             if (request.getNextCallSeq() != rsh.getNextCallSeq()) {
2433               throw new OutOfOrderScannerNextException(
2434                 "Expected nextCallSeq: " + rsh.getNextCallSeq()
2435                 + " But the nextCallSeq got from client: " + request.getNextCallSeq() +
2436                 "; request=" + TextFormat.shortDebugString(request));
2437             }
2438             // Increment the nextCallSeq value which is the next expected from client.
2439             rsh.incNextCallSeq();
2440           }
2441         }
2442         try {
2443           // Remove lease while its being processed in server; protects against case
2444           // where processing of request takes > lease expiration time.
2445           lease = regionServer.leases.removeLease(scannerName);
2446           List<Result> results = new ArrayList<Result>();
2447           long totalCellSize = 0;
2448           long currentScanResultSize = 0;
2449 
2450           boolean done = false;
2451           // Call coprocessor. Get region info from scanner.
2452           if (region != null && region.getCoprocessorHost() != null) {
2453             Boolean bypass = region.getCoprocessorHost().preScannerNext(
2454               scanner, results, rows);
2455             if (!results.isEmpty()) {
2456               for (Result r : results) {
2457                 for (Cell cell : r.rawCells()) {
2458                   totalCellSize += CellUtil.estimatedSerializedSizeOf(cell);
2459                   currentScanResultSize += CellUtil.estimatedHeapSizeOfWithoutTags(cell);
2460                 }
2461               }
2462             }
2463             if (bypass != null && bypass.booleanValue()) {
2464               done = true;
2465             }
2466           }
2467 
2468           if (!done) {
2469             long maxResultSize = Math.min(scanner.getMaxResultSize(), maxQuotaResultSize);
2470             if (maxResultSize <= 0) {
2471               maxResultSize = maxQuotaResultSize;
2472             }
2473             // This is cells inside a row. Default size is 10 so if many versions or many cfs,
2474             // then we'll resize. Resizings show in profiler. Set it higher than 10. For now
2475             // arbitrary 32. TODO: keep record of general size of results being returned.
2476             List<Cell> values = new ArrayList<Cell>(32);
2477             region.startRegionOperation(Operation.SCAN);
2478             try {
2479               int i = 0;
2480               long before = EnvironmentEdgeManager.currentTime();
2481               synchronized(scanner) {
2482                 boolean stale = (region.getRegionInfo().getReplicaId() != 0);
2483                 boolean clientHandlesPartials =
2484                     request.hasClientHandlesPartials() && request.getClientHandlesPartials();
2485                 boolean clientHandlesHeartbeats =
2486                     request.hasClientHandlesHeartbeats() && request.getClientHandlesHeartbeats();
2487 
2488                 // On the server side we must ensure that the correct ordering of partial results is
2489                 // returned to the client to allow them to properly reconstruct the partial results.
2490                 // If the coprocessor host is adding to the result list, we cannot guarantee the
2491                 // correct ordering of partial results and so we prevent partial results from being
2492                 // formed.
2493                 boolean serverGuaranteesOrderOfPartials = currentScanResultSize == 0;
2494                 boolean allowPartialResults =
2495                     clientHandlesPartials && serverGuaranteesOrderOfPartials && !isSmallScan;
2496                 boolean moreRows = false;
2497 
2498                 // Heartbeat messages occur when the processing of the ScanRequest is exceeds a
2499                 // certain time threshold on the server. When the time threshold is exceeded, the
2500                 // server stops the scan and sends back whatever Results it has accumulated within
2501                 // that time period (may be empty). Since heartbeat messages have the potential to
2502                 // create partial Results (in the event that the timeout occurs in the middle of a
2503                 // row), we must only generate heartbeat messages when the client can handle both
2504                 // heartbeats AND partials
2505                 boolean allowHeartbeatMessages = clientHandlesHeartbeats && allowPartialResults;
2506 
2507                 // Default value of timeLimit is negative to indicate no timeLimit should be
2508                 // enforced.
2509                 long timeLimit = -1;
2510 
2511                 // Set the time limit to be half of the more restrictive timeout value (one of the
2512                 // timeout values must be positive). In the event that both values are positive, the
2513                 // more restrictive of the two is used to calculate the limit.
2514                 if (allowHeartbeatMessages && (scannerLeaseTimeoutPeriod > 0 || rpcTimeout > 0)) {
2515                   long timeLimitDelta;
2516                   if (scannerLeaseTimeoutPeriod > 0 && rpcTimeout > 0) {
2517                     timeLimitDelta = Math.min(scannerLeaseTimeoutPeriod, rpcTimeout);
2518                   } else {
2519                     timeLimitDelta =
2520                         scannerLeaseTimeoutPeriod > 0 ? scannerLeaseTimeoutPeriod : rpcTimeout;
2521                   }
2522                   // Use half of whichever timeout value was more restrictive... But don't allow
2523                   // the time limit to be less than the allowable minimum (could cause an
2524                   // immediatate timeout before scanning any data).
2525                   timeLimitDelta = Math.max(timeLimitDelta / 2, minimumScanTimeLimitDelta);
2526                   timeLimit = System.currentTimeMillis() + timeLimitDelta;
2527                 }
2528 
2529                 final LimitScope sizeScope =
2530                     allowPartialResults ? LimitScope.BETWEEN_CELLS : LimitScope.BETWEEN_ROWS;
2531                 final LimitScope timeScope =
2532                     allowHeartbeatMessages ? LimitScope.BETWEEN_CELLS : LimitScope.BETWEEN_ROWS;
2533 
2534                 // Configure with limits for this RPC. Set keep progress true since size progress
2535                 // towards size limit should be kept between calls to nextRaw
2536                 ScannerContext.Builder contextBuilder = ScannerContext.newBuilder(true);
2537                 contextBuilder.setSizeLimit(sizeScope, maxResultSize);
2538                 contextBuilder.setBatchLimit(scanner.getBatch());
2539                 contextBuilder.setTimeLimit(timeScope, timeLimit);
2540                 ScannerContext scannerContext = contextBuilder.build();
2541 
2542                 boolean limitReached = false;
2543                 while (i < rows) {
2544                   // Reset the batch progress to 0 before every call to RegionScanner#nextRaw. The
2545                   // batch limit is a limit on the number of cells per Result. Thus, if progress is
2546                   // being tracked (i.e. scannerContext.keepProgress() is true) then we need to
2547                   // reset the batch progress between nextRaw invocations since we don't want the
2548                   // batch progress from previous calls to affect future calls
2549                   scannerContext.setBatchProgress(0);
2550 
2551                   // Collect values to be returned here
2552                   moreRows = scanner.nextRaw(values, scannerContext);
2553 
2554                   if (!values.isEmpty()) {
2555                     for (Cell cell : values) {
2556                       totalCellSize += CellUtil.estimatedSerializedSizeOf(cell);
2557                     }
2558                     final boolean partial = scannerContext.partialResultFormed();
2559                     results.add(Result.create(values, null, stale, partial));
2560                     i++;
2561                   }
2562 
2563                   boolean sizeLimitReached = scannerContext.checkSizeLimit(LimitScope.BETWEEN_ROWS);
2564                   boolean timeLimitReached = scannerContext.checkTimeLimit(LimitScope.BETWEEN_ROWS);
2565                   boolean rowLimitReached = i >= rows;
2566                   limitReached = sizeLimitReached || timeLimitReached || rowLimitReached;
2567 
2568                   if (limitReached || !moreRows) {
2569                     if (LOG.isTraceEnabled()) {
2570                       LOG.trace("Done scanning. limitReached: " + limitReached + " moreRows: "
2571                           + moreRows + " scannerContext: " + scannerContext);
2572                     }
2573                     // We only want to mark a ScanResponse as a heartbeat message in the event that
2574                     // there are more values to be read server side. If there aren't more values,
2575                     // marking it as a heartbeat is wasteful because the client will need to issue
2576                     // another ScanRequest only to realize that they already have all the values
2577                     if (moreRows) {
2578                       // Heartbeat messages occur when the time limit has been reached.
2579                       builder.setHeartbeatMessage(timeLimitReached);
2580                     }
2581                     break;
2582                   }
2583                   values.clear();
2584                 }
2585 
2586                 if (limitReached || moreRows) {
2587                   // We stopped prematurely
2588                   builder.setMoreResultsInRegion(true);
2589                 } else {
2590                   // We didn't get a single batch
2591                   builder.setMoreResultsInRegion(false);
2592                 }
2593               }
2594               region.updateReadRequestsCount(i);
2595               long end = EnvironmentEdgeManager.currentTime();
2596               region.getMetrics().updateScanTime(end - before);
2597               if (regionServer.metricsRegionServer != null) {
2598                 regionServer.metricsRegionServer.updateScanSize(totalCellSize);
2599                 regionServer.metricsRegionServer.updateScanTime(end - before);
2600               }
2601             } finally {
2602               region.closeRegionOperation();
2603             }
2604 
2605             // coprocessor postNext hook
2606             if (region != null && region.getCoprocessorHost() != null) {
2607               region.getCoprocessorHost().postScannerNext(scanner, results, rows, true);
2608             }
2609           }
2610 
2611           quota.addScanResult(results);
2612 
2613           // If the scanner's filter - if any - is done with the scan
2614           // and wants to tell the client to stop the scan. This is done by passing
2615           // a null result, and setting moreResults to false.
2616           if (scanner.isFilterDone() && results.isEmpty()) {
2617             moreResults = false;
2618             results = null;
2619           } else {
2620             addResults(builder, results, controller, RegionReplicaUtil.isDefaultReplica(region.getRegionInfo()));
2621           }
2622         } catch (IOException e) {
2623           // The scanner state might be left in a dirty state, so we will tell the Client to
2624           // fail this RPC and close the scanner while opening up another one from the start of
2625           // row that the client has last seen.
2626           closeScanner(region, scanner, scannerName);
2627 
2628           // If it is a DoNotRetryIOException already, throw as it is. Unfortunately, DNRIOE is
2629           // used in two different semantics.
2630           // (1) The first is to close the client scanner and bubble up the exception all the way
2631           // to the application. This is preferred when the exception is really un-recoverable
2632           // (like CorruptHFileException, etc). Plain DoNotRetryIOException also falls into this
2633           // bucket usually.
2634           // (2) Second semantics is to close the current region scanner only, but continue the
2635           // client scanner by overriding the exception. This is usually UnknownScannerException,
2636           // OutOfOrderScannerNextException, etc where the region scanner has to be closed, but the
2637           // application-level ClientScanner has to continue without bubbling up the exception to
2638           // the client. See ClientScanner code to see how it deals with these special exceptions.
2639           if (e instanceof DoNotRetryIOException) {
2640             throw e;
2641           }
2642 
2643           // If it is a CorruptHFileException or a FileNotFoundException, throw the
2644           // DoNotRetryIOException. This can avoid the retry in ClientScanner.
2645           if (e instanceof CorruptHFileException || e instanceof FileNotFoundException) {
2646             throw new DoNotRetryIOException(e);
2647           }
2648 
2649           // We closed the scanner already. Instead of throwing the IOException, and client
2650           // retrying with the same scannerId only to get USE on the next RPC, we directly throw
2651           // a special exception to save an RPC.
2652           RpcCallContext context = RpcServer.getCurrentCall();
2653           if (context != null && VersionInfoUtil.hasMinimumVersion(context.getClientVersionInfo(), 1, 4)) {
2654             // 1.4.0+ clients know how to handle
2655             throw new ScannerResetException("Scanner is closed on the server-side", e);
2656           } else {
2657             // older clients do not know about SRE. Just throw USE, which they will handle
2658             throw new UnknownScannerException("Throwing UnknownScannerException to reset the client"
2659                 + " scanner state for clients older than 1.3.", e);
2660           }
2661         } finally {
2662           // We're done. On way out re-add the above removed lease.
2663           // Adding resets expiration time on lease.
2664           if (scanners.containsKey(scannerName)) {
2665             if (lease != null) regionServer.leases.addLease(lease);
2666             ttl = this.scannerLeaseTimeoutPeriod;
2667           }
2668         }
2669       }
2670 
2671       if (!moreResults || closeScanner) {
2672         ttl = 0;
2673         moreResults = false;
2674         if (closeScanner(region, scanner, scannerName)) {
2675           return builder.build(); // bypass
2676         }
2677       }
2678 
2679       if (ttl > 0) {
2680         builder.setTtl(ttl);
2681       }
2682       builder.setScannerId(scannerId);
2683       builder.setMoreResults(moreResults);
2684       return builder.build();
2685     } catch (IOException ie) {
2686       if (scannerName != null && ie instanceof NotServingRegionException) {
2687         RegionScannerHolder rsh = scanners.remove(scannerName);
2688         if (rsh != null) {
2689           try {
2690             RegionScanner scanner = rsh.s;
2691             LOG.warn(scannerName + " encountered " + ie.getMessage() + ", closing ...");
2692             scanner.close();
2693             regionServer.leases.cancelLease(scannerName);
2694           } catch (IOException e) {
2695            LOG.warn("Getting exception closing " + scannerName, e);
2696           }
2697         }
2698       }
2699       throw new ServiceException(ie);
2700     } finally {
2701       if (quota != null) {
2702         quota.close();
2703       }
2704     }
2705   }
2706 
2707   private boolean closeScanner(Region region, RegionScanner scanner, String scannerName) 
2708     throws IOException {
2709     if (region != null && region.getCoprocessorHost() != null) {
2710       if (region.getCoprocessorHost().preScannerClose(scanner)) {
2711         return true; // bypass
2712       }
2713     }
2714     RegionScannerHolder rsh = scanners.remove(scannerName);
2715     if (rsh != null) {
2716       scanner = rsh.s;
2717       scanner.close();
2718       try {
2719         regionServer.leases.cancelLease(scannerName);
2720       } catch (LeaseException le) {
2721         // No problem, ignore
2722         if (LOG.isTraceEnabled()) {
2723           LOG.trace("Un-able to cancel lease of scanner. It could already be closed.");
2724         }
2725       }
2726       if (region != null && region.getCoprocessorHost() != null) {
2727         region.getCoprocessorHost().postScannerClose(scanner);
2728       }
2729     }
2730     return false;
2731   }
2732 
2733   @Override
2734   public CoprocessorServiceResponse execRegionServerService(RpcController controller,
2735       CoprocessorServiceRequest request) throws ServiceException {
2736     return regionServer.execRegionServerService(controller, request);
2737   }
2738 
2739   @Override
2740   public UpdateConfigurationResponse updateConfiguration(
2741       RpcController controller, UpdateConfigurationRequest request)
2742       throws ServiceException {
2743     try {
2744       this.regionServer.updateConfiguration();
2745     } catch (Exception e) {
2746       throw new ServiceException(e);
2747     }
2748     return UpdateConfigurationResponse.getDefaultInstance();
2749   }
2750 
2751   @Override
2752   public GetSpaceQuotaSnapshotsResponse getSpaceQuotaSnapshots(
2753       RpcController controller, GetSpaceQuotaSnapshotsRequest request) throws ServiceException {
2754     try {
2755       final RegionServerSpaceQuotaManager manager =
2756           regionServer.getRegionServerSpaceQuotaManager();
2757       final GetSpaceQuotaSnapshotsResponse.Builder builder =
2758           GetSpaceQuotaSnapshotsResponse.newBuilder();
2759       if (manager != null) {
2760         final Map<TableName,SpaceQuotaSnapshot> snapshots = manager.copyQuotaSnapshots();
2761         for (Map.Entry<TableName,SpaceQuotaSnapshot> snapshot : snapshots.entrySet()) {
2762           builder.addSnapshots(TableQuotaSnapshot.newBuilder()
2763               .setTableName(ProtobufUtil.toProtoTableName(snapshot.getKey()))
2764               .setSnapshot(SpaceQuotaSnapshot.toProtoSnapshot(snapshot.getValue()))
2765               .build());
2766         }
2767       }
2768       return builder.build();
2769     } catch (Exception e) {
2770       throw new ServiceException(e);
2771     }
2772   }
2773 }