1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.client;
21
22 import java.io.IOException;
23 import java.io.InterruptedIOException;
24 import java.lang.reflect.UndeclaredThrowableException;
25 import java.net.SocketTimeoutException;
26 import java.util.ArrayList;
27 import java.util.List;
28 import java.util.concurrent.atomic.AtomicBoolean;
29
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.hadoop.hbase.classification.InterfaceAudience;
33 import org.apache.hadoop.hbase.DoNotRetryIOException;
34 import org.apache.hadoop.hbase.exceptions.PreemptiveFastFailException;
35 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
36 import org.apache.hadoop.hbase.util.ExceptionUtil;
37 import org.apache.hadoop.ipc.RemoteException;
38
39 import com.google.protobuf.ServiceException;
40
41
42
43
44
45
46
47
48
49
50 @InterfaceAudience.Private
51 public class RpcRetryingCaller<T> {
52 public static final Log LOG = LogFactory.getLog(RpcRetryingCaller.class);
53
54
55
56 private long globalStartTime;
57
58
59
60 private final static int MIN_RPC_TIMEOUT = 1;
61
62 private final int startLogErrorsCnt;
63
64 private final long pause;
65 private final int retries;
66 private final int rpcTimeout;
67 private final AtomicBoolean cancelled = new AtomicBoolean(false);
68 private final RetryingCallerInterceptor interceptor;
69 private final RetryingCallerInterceptorContext context;
70
71 public RpcRetryingCaller(long pause, int retries, int startLogErrorsCnt) {
72 this(pause, retries, RetryingCallerInterceptorFactory.NO_OP_INTERCEPTOR, startLogErrorsCnt, 0);
73 }
74
75 public RpcRetryingCaller(long pause, int retries,
76 RetryingCallerInterceptor interceptor, int startLogErrorsCnt, int rpcTimeout) {
77 this.pause = pause;
78 this.retries = retries;
79 this.interceptor = interceptor;
80 context = interceptor.createEmptyContext();
81 this.startLogErrorsCnt = startLogErrorsCnt;
82 this.rpcTimeout = rpcTimeout;
83 }
84
85 private int getRemainingTime(int callTimeout) {
86 if (callTimeout <= 0) {
87 return 0;
88 } else {
89 if (callTimeout == Integer.MAX_VALUE) return Integer.MAX_VALUE;
90 int remainingTime = (int) (callTimeout -
91 (EnvironmentEdgeManager.currentTime() - this.globalStartTime));
92 if (remainingTime < MIN_RPC_TIMEOUT) {
93
94
95
96 remainingTime = MIN_RPC_TIMEOUT;
97 }
98 return remainingTime;
99 }
100 }
101
102 private int getTimeout(int callTimeout){
103 int timeout = getRemainingTime(callTimeout);
104 if (timeout <= 0 || rpcTimeout > 0 && rpcTimeout < timeout){
105 timeout = rpcTimeout;
106 }
107 return timeout;
108 }
109
110 public void cancel(){
111 synchronized (cancelled){
112 cancelled.set(true);
113 cancelled.notifyAll();
114 }
115 }
116
117
118
119
120
121
122
123
124
125 public T callWithRetries(RetryingCallable<T> callable, int callTimeout)
126 throws IOException, RuntimeException {
127 List<RetriesExhaustedException.ThrowableWithExtraContext> exceptions =
128 new ArrayList<RetriesExhaustedException.ThrowableWithExtraContext>();
129 this.globalStartTime = EnvironmentEdgeManager.currentTime();
130 context.clear();
131 for (int tries = 0;; tries++) {
132 long expectedSleep;
133 try {
134
135 callable.prepare(tries != 0);
136 interceptor.intercept(context.prepare(callable, tries));
137 return callable.call(getTimeout(callTimeout));
138 } catch (PreemptiveFastFailException e) {
139 throw e;
140 } catch (Throwable t) {
141 ExceptionUtil.rethrowIfInterrupt(t);
142
143
144 interceptor.handleFailure(context, t);
145 t = translateException(t);
146 if (tries > startLogErrorsCnt) {
147 LOG.info("Call exception, tries=" + tries + ", retries=" + retries + ", started=" +
148 (EnvironmentEdgeManager.currentTime() - this.globalStartTime) + " ms ago, "
149 + "cancelled=" + cancelled.get() + ", msg="
150 + t.getMessage() + " " + callable.getExceptionMessageAdditionalDetail());
151 }
152
153 callable.throwable(t, retries != 1);
154 RetriesExhaustedException.ThrowableWithExtraContext qt =
155 new RetriesExhaustedException.ThrowableWithExtraContext(t,
156 EnvironmentEdgeManager.currentTime(), toString());
157 exceptions.add(qt);
158 if (tries >= retries - 1) {
159 throw new RetriesExhaustedException(tries, exceptions);
160 }
161
162
163
164 expectedSleep = callable.sleep(pause, tries);
165
166
167 long duration = singleCallDuration(expectedSleep);
168 if (duration > callTimeout) {
169 String msg = "callTimeout=" + callTimeout + ", callDuration=" + duration +
170 ": " + t.getMessage() + " " + callable.getExceptionMessageAdditionalDetail();
171 throw (SocketTimeoutException)(new SocketTimeoutException(msg).initCause(t));
172 }
173 } finally {
174 interceptor.updateFailureInfo(context);
175 }
176 try {
177 if (expectedSleep > 0) {
178 synchronized (cancelled) {
179 if (cancelled.get()) return null;
180 cancelled.wait(expectedSleep);
181 }
182 }
183 if (cancelled.get()) return null;
184 } catch (InterruptedException e) {
185 throw new InterruptedIOException("Interrupted after " + tries + " tries on " + retries);
186 }
187 }
188 }
189
190
191
192
193 private long singleCallDuration(final long expectedSleep) {
194 return (EnvironmentEdgeManager.currentTime() - this.globalStartTime) + expectedSleep;
195 }
196
197
198
199
200
201
202
203
204
205
206 public T callWithoutRetries(RetryingCallable<T> callable, int callTimeout)
207 throws IOException, RuntimeException {
208
209 this.globalStartTime = EnvironmentEdgeManager.currentTime();
210 try {
211 callable.prepare(false);
212 return callable.call(callTimeout);
213 } catch (Throwable t) {
214 Throwable t2 = translateException(t);
215 ExceptionUtil.rethrowIfInterrupt(t2);
216
217 if (t2 instanceof IOException) {
218 throw (IOException)t2;
219 } else {
220 throw new RuntimeException(t2);
221 }
222 }
223 }
224
225
226
227
228
229
230
231 static Throwable translateException(Throwable t) throws DoNotRetryIOException {
232 if (t instanceof UndeclaredThrowableException) {
233 if (t.getCause() != null) {
234 t = t.getCause();
235 }
236 }
237 if (t instanceof RemoteException) {
238 t = ((RemoteException)t).unwrapRemoteException();
239 }
240 if (t instanceof LinkageError) {
241 throw new DoNotRetryIOException(t);
242 }
243 if (t instanceof ServiceException) {
244 ServiceException se = (ServiceException)t;
245 Throwable cause = se.getCause();
246 if (cause != null) {
247 if (cause instanceof DoNotRetryIOException) {
248 throw (DoNotRetryIOException)cause;
249 } else if (cause instanceof NeedUnmanagedConnectionException) {
250 throw new DoNotRetryIOException(cause);
251 }
252 }
253
254 t = cause;
255
256 translateException(t);
257 } else if (t instanceof DoNotRetryIOException) {
258 throw (DoNotRetryIOException)t;
259 } else if (t instanceof NeedUnmanagedConnectionException) {
260 throw new DoNotRetryIOException(t);
261 }
262 return t;
263 }
264
265 @Override
266 public String toString() {
267 return "RpcRetryingCaller{" + "globalStartTime=" + globalStartTime +
268 ", pause=" + pause + ", retries=" + retries + '}';
269 }
270 }