View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.client;
21  
22  import java.io.IOException;
23  import java.io.InterruptedIOException;
24  import java.lang.reflect.UndeclaredThrowableException;
25  import java.net.SocketTimeoutException;
26  import java.util.ArrayList;
27  import java.util.List;
28  import java.util.concurrent.atomic.AtomicBoolean;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.hbase.classification.InterfaceAudience;
33  import org.apache.hadoop.hbase.DoNotRetryIOException;
34  import org.apache.hadoop.hbase.exceptions.PreemptiveFastFailException;
35  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
36  import org.apache.hadoop.hbase.util.ExceptionUtil;
37  import org.apache.hadoop.ipc.RemoteException;
38  
39  import com.google.protobuf.ServiceException;
40  
41  /**
42   * Runs an rpc'ing {@link RetryingCallable}. Sets into rpc client
43   * threadlocal outstanding timeouts as so we don't persist too much.
44   * Dynamic rather than static so can set the generic appropriately.
45   *
46   * This object has a state. It should not be used by in parallel by different threads.
47   * Reusing it is possible however, even between multiple threads. However, the user will
48   *  have to manage the synchronization on its side: there is no synchronization inside the class.
49   */
50  @InterfaceAudience.Private
51  public class RpcRetryingCaller<T> {
52    public static final Log LOG = LogFactory.getLog(RpcRetryingCaller.class);
53    /**
54     * When we started making calls.
55     */
56    private long globalStartTime;
57    /**
58     * Start and end times for a single call.
59     */
60    private final static int MIN_RPC_TIMEOUT = 1;
61    /** How many retries are allowed before we start to log */
62    private final int startLogErrorsCnt;
63  
64    private final long pause;
65    private final int retries;
66    private final int rpcTimeout;// timeout for each rpc request
67    private final AtomicBoolean cancelled = new AtomicBoolean(false);
68    private final RetryingCallerInterceptor interceptor;
69    private final RetryingCallerInterceptorContext context;
70  
71    public RpcRetryingCaller(long pause, int retries, int startLogErrorsCnt) {
72      this(pause, retries, RetryingCallerInterceptorFactory.NO_OP_INTERCEPTOR, startLogErrorsCnt, 0);
73    }
74  
75    public RpcRetryingCaller(long pause, int retries,
76        RetryingCallerInterceptor interceptor, int startLogErrorsCnt, int rpcTimeout) {
77      this.pause = pause;
78      this.retries = retries;
79      this.interceptor = interceptor;
80      context = interceptor.createEmptyContext();
81      this.startLogErrorsCnt = startLogErrorsCnt;
82      this.rpcTimeout = rpcTimeout;
83    }
84  
85    private int getRemainingTime(int callTimeout) {
86      if (callTimeout <= 0) {
87        return 0;
88      } else {
89        if (callTimeout == Integer.MAX_VALUE) return Integer.MAX_VALUE;
90        int remainingTime = (int) (callTimeout -
91            (EnvironmentEdgeManager.currentTime() - this.globalStartTime));
92        if (remainingTime < MIN_RPC_TIMEOUT) {
93          // If there is no time left, we're trying anyway. It's too late.
94          // 0 means no timeout, and it's not the intent here. So we secure both cases by
95          // resetting to the minimum.
96          remainingTime = MIN_RPC_TIMEOUT;
97        }
98        return remainingTime;
99      }
100   }
101 
102   private int getTimeout(int callTimeout){
103     int timeout = getRemainingTime(callTimeout);
104     if (timeout <= 0 || rpcTimeout > 0 && rpcTimeout < timeout){
105       timeout = rpcTimeout;
106     }
107     return timeout;
108   }
109 
110   public void cancel(){
111     synchronized (cancelled){
112       cancelled.set(true);
113       cancelled.notifyAll();
114     }
115   }
116 
117   /**
118    * Retries if invocation fails.
119    * @param callTimeout Timeout for this call
120    * @param callable The {@link RetryingCallable} to run.
121    * @return an object of type T
122    * @throws IOException if a remote or network exception occurs
123    * @throws RuntimeException other unspecified error
124    */
125   public T callWithRetries(RetryingCallable<T> callable, int callTimeout)
126   throws IOException, RuntimeException {
127     List<RetriesExhaustedException.ThrowableWithExtraContext> exceptions =
128       new ArrayList<RetriesExhaustedException.ThrowableWithExtraContext>();
129     this.globalStartTime = EnvironmentEdgeManager.currentTime();
130     context.clear();
131     for (int tries = 0;; tries++) {
132       long expectedSleep;
133       try {
134         // bad cache entries are cleared in the call to RetryingCallable#throwable() in catch block
135         callable.prepare(tries != 0); // if called with false, check table status on ZK
136         interceptor.intercept(context.prepare(callable, tries));
137         return callable.call(getTimeout(callTimeout));
138       } catch (PreemptiveFastFailException e) {
139         throw e;
140       } catch (Throwable t) {
141         ExceptionUtil.rethrowIfInterrupt(t);
142 
143         // translateException throws exception when should not retry: i.e. when request is bad.
144         interceptor.handleFailure(context, t);
145         t = translateException(t);
146         if (tries > startLogErrorsCnt) {
147           LOG.info("Call exception, tries=" + tries + ", retries=" + retries + ", started=" +
148               (EnvironmentEdgeManager.currentTime() - this.globalStartTime) + " ms ago, "
149               + "cancelled=" + cancelled.get() + ", msg="
150               + t.getMessage() + " " + callable.getExceptionMessageAdditionalDetail());
151         }
152 
153         callable.throwable(t, retries != 1);
154         RetriesExhaustedException.ThrowableWithExtraContext qt =
155             new RetriesExhaustedException.ThrowableWithExtraContext(t,
156                 EnvironmentEdgeManager.currentTime(), toString());
157         exceptions.add(qt);
158         if (tries >= retries - 1) {
159           throw new RetriesExhaustedException(tries, exceptions);
160         }
161         // If the server is dead, we need to wait a little before retrying, to give
162         //  a chance to the regions to be
163         // get right pause time, start by RETRY_BACKOFF[0] * pause
164         expectedSleep = callable.sleep(pause, tries);
165 
166         // If, after the planned sleep, there won't be enough time left, we stop now.
167         long duration = singleCallDuration(expectedSleep);
168         if (duration > callTimeout) {
169           String msg = "callTimeout=" + callTimeout + ", callDuration=" + duration +
170               ": " + t.getMessage() + " " + callable.getExceptionMessageAdditionalDetail();
171           throw (SocketTimeoutException)(new SocketTimeoutException(msg).initCause(t));
172         }
173       } finally {
174         interceptor.updateFailureInfo(context);
175       }
176       try {
177         if (expectedSleep > 0) {
178           synchronized (cancelled) {
179             if (cancelled.get()) return null;
180             cancelled.wait(expectedSleep);
181           }
182         }
183         if (cancelled.get()) return null;
184       } catch (InterruptedException e) {
185         throw new InterruptedIOException("Interrupted after " + tries + " tries  on " + retries);
186       }
187     }
188   }
189 
190   /**
191    * @return Calculate how long a single call took
192    */
193   private long singleCallDuration(final long expectedSleep) {
194     return (EnvironmentEdgeManager.currentTime() - this.globalStartTime) + expectedSleep;
195   }
196 
197   /**
198    * Call the server once only.
199    * {@link RetryingCallable} has a strange shape so we can do retrys.  Use this invocation if you
200    * want to do a single call only (A call to {@link RetryingCallable#call(int)} will not likely
201    * succeed).
202    * @return an object of type T
203    * @throws IOException if a remote or network exception occurs
204    * @throws RuntimeException other unspecified error
205    */
206   public T callWithoutRetries(RetryingCallable<T> callable, int callTimeout)
207   throws IOException, RuntimeException {
208     // The code of this method should be shared with withRetries.
209     this.globalStartTime = EnvironmentEdgeManager.currentTime();
210     try {
211       callable.prepare(false);
212       return callable.call(callTimeout);
213     } catch (Throwable t) {
214       Throwable t2 = translateException(t);
215       ExceptionUtil.rethrowIfInterrupt(t2);
216       // It would be nice to clear the location cache here.
217       if (t2 instanceof IOException) {
218         throw (IOException)t2;
219       } else {
220         throw new RuntimeException(t2);
221       }
222     }
223   }
224 
225   /**
226    * Get the good or the remote exception if any, throws the DoNotRetryIOException.
227    * @param t the throwable to analyze
228    * @return the translated exception, if it's not a DoNotRetryIOException
229    * @throws DoNotRetryIOException - if we find it, we throw it instead of translating.
230    */
231   static Throwable translateException(Throwable t) throws DoNotRetryIOException {
232     if (t instanceof UndeclaredThrowableException) {
233       if (t.getCause() != null) {
234         t = t.getCause();
235       }
236     }
237     if (t instanceof RemoteException) {
238       t = ((RemoteException)t).unwrapRemoteException();
239     }
240     if (t instanceof LinkageError) {
241       throw new DoNotRetryIOException(t);
242     }
243     if (t instanceof ServiceException) {
244       ServiceException se = (ServiceException)t;
245       Throwable cause = se.getCause();
246       if (cause != null) {
247         if (cause instanceof DoNotRetryIOException) {
248           throw (DoNotRetryIOException)cause;
249         } else if (cause instanceof NeedUnmanagedConnectionException) {
250           throw new DoNotRetryIOException(cause);
251         }
252       }
253       // Don't let ServiceException out; its rpc specific.
254       t = cause;
255       // t could be a RemoteException so go aaround again.
256       translateException(t);
257     } else if (t instanceof DoNotRetryIOException) {
258       throw (DoNotRetryIOException)t;
259     } else if (t instanceof NeedUnmanagedConnectionException) {
260       throw new DoNotRetryIOException(t);
261     }
262     return t;
263   }
264 
265   @Override
266   public String toString() {
267     return "RpcRetryingCaller{" + "globalStartTime=" + globalStartTime +
268         ", pause=" + pause + ", retries=" + retries + '}';
269   }
270 }