View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.io;
19  
20  import java.io.IOException;
21  import java.io.InputStream;
22  import java.lang.reflect.InvocationTargetException;
23  import java.lang.reflect.Method;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.fs.FSDataInputStream;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.fs.HFileSystem;
31  import org.apache.hadoop.hbase.io.FileLink;
32  
33  import com.google.common.annotations.VisibleForTesting;
34  
35  /**
36   * Wrapper for input stream(s) that takes care of the interaction of FS and HBase checksums,
37   * as well as closing streams. Initialization is not thread-safe, but normal operation is;
38   * see method comments.
39   */
40  public class FSDataInputStreamWrapper {
41    private static final Log LOG = LogFactory.getLog(FSDataInputStreamWrapper.class);
42  
43    private final HFileSystem hfs;
44    private final Path path;
45    private final FileLink link;
46    private final boolean doCloseStreams;
47  
48    /** Two stream handles, one with and one without FS-level checksum.
49     * HDFS checksum setting is on FS level, not single read level, so you have to keep two
50     * FS objects and two handles open to interleave different reads freely, which is very sad.
51     * This is what we do:
52     * 1) First, we need to read the trailer of HFile to determine checksum parameters.
53     *  We always use FS checksum to do that, so ctor opens {@link #stream}.
54     * 2.1) After that, if HBase checksum is not used, we'd just always use {@link #stream};
55     * 2.2) If HBase checksum can be used, we'll open {@link #streamNoFsChecksum},
56     *  and close {@link #stream}. User MUST call prepareForBlockReader for that to happen;
57     *  if they don't, (2.1) will be the default.
58     * 3) The users can call {@link #shouldUseHBaseChecksum()}, and pass its result to
59     *  {@link #getStream(boolean)} to get stream (if Java had out/pointer params we could
60     *  return both in one call). This stream is guaranteed to be set.
61     * 4) The first time HBase checksum fails, one would call {@link #fallbackToFsChecksum(int)}.
62     * That will take lock, and open {@link #stream}. While this is going on, others will
63     * continue to use the old stream; if they also want to fall back, they'll also call
64     * {@link #fallbackToFsChecksum(int)}, and block until {@link #stream} is set.
65     * 5) After some number of checksumOk() calls, we will go back to using HBase checksum.
66     * We will have 2 handles; however we presume checksums fail so rarely that we don't care.
67     */
68    private volatile FSDataInputStream stream = null;
69    private volatile FSDataInputStream streamNoFsChecksum = null;
70    private Object streamNoFsChecksumFirstCreateLock = new Object();
71  
72    // The configuration states that we should validate hbase checksums
73    private boolean useHBaseChecksumConfigured;
74  
75    // Record the current state of this reader with respect to
76    // validating checkums in HBase. This is originally set the same
77    // value as useHBaseChecksumConfigured, but can change state as and when
78    // we encounter checksum verification failures.
79    private volatile boolean useHBaseChecksum;
80  
81    // In the case of a checksum failure, do these many succeeding
82    // reads without hbase checksum verification.
83    private volatile int hbaseChecksumOffCount = -1;
84  
85    private Boolean instanceOfCanUnbuffer = null;
86    // Using reflection to get org.apache.hadoop.fs.CanUnbuffer#unbuffer method to avoid compilation
87    // errors against Hadoop pre 2.6.4 and 2.7.1 versions.
88    private Method unbuffer = null;
89  
90    public FSDataInputStreamWrapper(FileSystem fs, Path path) throws IOException {
91      this(fs, null, path);
92    }
93  
94    public FSDataInputStreamWrapper(FileSystem fs, FileLink link) throws IOException {
95      this(fs, link, null);
96    }
97  
98    private FSDataInputStreamWrapper(FileSystem fs, FileLink link, Path path) throws IOException {
99      assert (path == null) != (link == null);
100     this.path = path;
101     this.link = link;
102     this.doCloseStreams = true;
103     // If the fs is not an instance of HFileSystem, then create an instance of HFileSystem
104     // that wraps over the specified fs. In this case, we will not be able to avoid
105     // checksumming inside the filesystem.
106     this.hfs = (fs instanceof HFileSystem) ? (HFileSystem)fs : new HFileSystem(fs);
107 
108     // Initially we are going to read the tail block. Open the reader w/FS checksum.
109     this.useHBaseChecksumConfigured = this.useHBaseChecksum = false;
110     this.stream = (link != null) ? link.open(hfs) : hfs.open(path);
111   }
112 
113   /**
114    * Prepares the streams for block reader. NOT THREAD SAFE. Must be called once, after any
115    * reads finish and before any other reads start (what happens in reality is we read the
116    * tail, then call this based on what's in the tail, then read blocks).
117    * @param forceNoHBaseChecksum Force not using HBase checksum.
118    */
119   public void prepareForBlockReader(boolean forceNoHBaseChecksum) throws IOException {
120     if (hfs == null) return;
121     assert this.stream != null && !this.useHBaseChecksumConfigured;
122     boolean useHBaseChecksum =
123         !forceNoHBaseChecksum && hfs.useHBaseChecksum() && (hfs.getNoChecksumFs() != hfs);
124 
125     if (useHBaseChecksum) {
126       FileSystem fsNc = hfs.getNoChecksumFs();
127       this.streamNoFsChecksum = (link != null) ? link.open(fsNc) : fsNc.open(path);
128       this.useHBaseChecksumConfigured = this.useHBaseChecksum = useHBaseChecksum;
129       // Close the checksum stream; we will reopen it if we get an HBase checksum failure.
130       this.stream.close();
131       this.stream = null;
132     }
133   }
134 
135   /** For use in tests. */
136   @VisibleForTesting
137   public FSDataInputStreamWrapper(FSDataInputStream fsdis) {
138     this(fsdis, fsdis);
139   }
140 
141   /** For use in tests. */
142   @VisibleForTesting
143   public FSDataInputStreamWrapper(FSDataInputStream fsdis, FSDataInputStream noChecksum) {
144     doCloseStreams = false;
145     stream = fsdis;
146     streamNoFsChecksum = noChecksum;
147     path = null;
148     link = null;
149     hfs = null;
150     useHBaseChecksumConfigured = useHBaseChecksum = false;
151   }
152 
153   /**
154    * @return Whether we are presently using HBase checksum.
155    */
156   public boolean shouldUseHBaseChecksum() {
157     return this.useHBaseChecksum;
158   }
159 
160   /**
161    * Get the stream to use. Thread-safe.
162    * @param useHBaseChecksum must be the value that shouldUseHBaseChecksum has returned
163    *  at some point in the past, otherwise the result is undefined.
164    */
165   public FSDataInputStream getStream(boolean useHBaseChecksum) {
166     return useHBaseChecksum ? this.streamNoFsChecksum : this.stream;
167   }
168 
169   /**
170    * Read from non-checksum stream failed, fall back to FS checksum. Thread-safe.
171    * @param offCount For how many checksumOk calls to turn off the HBase checksum.
172    */
173   public FSDataInputStream fallbackToFsChecksum(int offCount) throws IOException {
174     // checksumOffCount is speculative, but let's try to reset it less.
175     boolean partOfConvoy = false;
176     if (this.stream == null) {
177       synchronized (streamNoFsChecksumFirstCreateLock) {
178         partOfConvoy = (this.stream != null);
179         if (!partOfConvoy) {
180           this.stream = (link != null) ? link.open(hfs) : hfs.open(path);
181         }
182       }
183     }
184     if (!partOfConvoy) {
185       this.useHBaseChecksum = false;
186       this.hbaseChecksumOffCount = offCount;
187     }
188     return this.stream;
189   }
190 
191   /** Report that checksum was ok, so we may ponder going back to HBase checksum. */
192   public void checksumOk() {
193     if (this.useHBaseChecksumConfigured && !this.useHBaseChecksum
194         && (this.hbaseChecksumOffCount-- < 0)) {
195       // The stream we need is already open (because we were using HBase checksum in the past).
196       assert this.streamNoFsChecksum != null;
197       this.useHBaseChecksum = true;
198     }
199   }
200 
201   /** Close stream(s) if necessary. */
202   public void close() throws IOException {
203     if (!doCloseStreams) return;
204     try {
205       if (stream != streamNoFsChecksum && streamNoFsChecksum != null) {
206         streamNoFsChecksum.close();
207         streamNoFsChecksum = null;
208       }
209     } finally {
210       if (stream != null) {
211         stream.close();
212         stream = null;
213       }
214     }
215   }
216 
217   public HFileSystem getHfs() {
218     return this.hfs;
219   }
220 
221   /**
222    * This will free sockets and file descriptors held by the stream only when the stream implements
223    * org.apache.hadoop.fs.CanUnbuffer. NOT THREAD SAFE. Must be called only when all the clients
224    * using this stream to read the blocks have finished reading. If by chance the stream is
225    * unbuffered and there are clients still holding this stream for read then on next client read
226    * request a new socket will be opened by Datanode without client knowing about it and will serve
227    * its read request. Note: If this socket is idle for some time then the DataNode will close the
228    * socket and the socket will move into CLOSE_WAIT state and on the next client request on this
229    * stream, the current socket will be closed and a new socket will be opened to serve the
230    * requests.
231    */
232   @SuppressWarnings({ "rawtypes" })
233   public void unbuffer() {
234     FSDataInputStream stream = this.getStream(this.shouldUseHBaseChecksum());
235     if (stream != null) {
236       InputStream wrappedStream = stream.getWrappedStream();
237       // CanUnbuffer interface was added as part of HDFS-7694 and the fix is available in Hadoop
238       // 2.6.4+ and 2.7.1+ versions only so check whether the stream object implements the
239       // CanUnbuffer interface or not and based on that call the unbuffer api.
240       final Class<? extends InputStream> streamClass = wrappedStream.getClass();
241       if (this.instanceOfCanUnbuffer == null) {
242         // To ensure we compute whether the stream is instance of CanUnbuffer only once.
243         this.instanceOfCanUnbuffer = false;
244         Class<?>[] streamInterfaces = streamClass.getInterfaces();
245         for (Class c : streamInterfaces) {
246           if (c.getCanonicalName().toString().equals("org.apache.hadoop.fs.CanUnbuffer")) {
247             try {
248               this.unbuffer = streamClass.getDeclaredMethod("unbuffer");
249             } catch (NoSuchMethodException | SecurityException e) {
250               LOG.warn("Failed to find 'unbuffer' method in class " + streamClass
251                   + " . So there may be a TCP socket connection "
252                   + "left open in CLOSE_WAIT state.",
253                 e);
254               return;
255             }
256             this.instanceOfCanUnbuffer = true;
257             break;
258           }
259         }
260       }
261       if (this.instanceOfCanUnbuffer) {
262         try {
263           this.unbuffer.invoke(wrappedStream);
264         } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
265           LOG.warn("Failed to invoke 'unbuffer' method in class " + streamClass
266               + " . So there may be a TCP socket connection left open in CLOSE_WAIT state.",
267             e);
268         }
269       } else {
270         LOG.warn("Failed to find 'unbuffer' method in class " + streamClass
271             + " . So there may be a TCP socket connection "
272             + "left open in CLOSE_WAIT state. For more details check "
273             + "https://issues.apache.org/jira/browse/HBASE-9393");
274       }
275     }
276   }
277 }