1 /**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18 package org.apache.hadoop.hbase.io;
19
20 import java.io.IOException;
21 import java.io.InputStream;
22 import java.lang.reflect.InvocationTargetException;
23 import java.lang.reflect.Method;
24
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.fs.FSDataInputStream;
28 import org.apache.hadoop.fs.FileSystem;
29 import org.apache.hadoop.fs.Path;
30 import org.apache.hadoop.hbase.fs.HFileSystem;
31 import org.apache.hadoop.hbase.io.FileLink;
32
33 import com.google.common.annotations.VisibleForTesting;
34
35 /**
36 * Wrapper for input stream(s) that takes care of the interaction of FS and HBase checksums,
37 * as well as closing streams. Initialization is not thread-safe, but normal operation is;
38 * see method comments.
39 */
40 public class FSDataInputStreamWrapper {
41 private static final Log LOG = LogFactory.getLog(FSDataInputStreamWrapper.class);
42
43 private final HFileSystem hfs;
44 private final Path path;
45 private final FileLink link;
46 private final boolean doCloseStreams;
47
48 /** Two stream handles, one with and one without FS-level checksum.
49 * HDFS checksum setting is on FS level, not single read level, so you have to keep two
50 * FS objects and two handles open to interleave different reads freely, which is very sad.
51 * This is what we do:
52 * 1) First, we need to read the trailer of HFile to determine checksum parameters.
53 * We always use FS checksum to do that, so ctor opens {@link #stream}.
54 * 2.1) After that, if HBase checksum is not used, we'd just always use {@link #stream};
55 * 2.2) If HBase checksum can be used, we'll open {@link #streamNoFsChecksum},
56 * and close {@link #stream}. User MUST call prepareForBlockReader for that to happen;
57 * if they don't, (2.1) will be the default.
58 * 3) The users can call {@link #shouldUseHBaseChecksum()}, and pass its result to
59 * {@link #getStream(boolean)} to get stream (if Java had out/pointer params we could
60 * return both in one call). This stream is guaranteed to be set.
61 * 4) The first time HBase checksum fails, one would call {@link #fallbackToFsChecksum(int)}.
62 * That will take lock, and open {@link #stream}. While this is going on, others will
63 * continue to use the old stream; if they also want to fall back, they'll also call
64 * {@link #fallbackToFsChecksum(int)}, and block until {@link #stream} is set.
65 * 5) After some number of checksumOk() calls, we will go back to using HBase checksum.
66 * We will have 2 handles; however we presume checksums fail so rarely that we don't care.
67 */
68 private volatile FSDataInputStream stream = null;
69 private volatile FSDataInputStream streamNoFsChecksum = null;
70 private Object streamNoFsChecksumFirstCreateLock = new Object();
71
72 // The configuration states that we should validate hbase checksums
73 private boolean useHBaseChecksumConfigured;
74
75 // Record the current state of this reader with respect to
76 // validating checkums in HBase. This is originally set the same
77 // value as useHBaseChecksumConfigured, but can change state as and when
78 // we encounter checksum verification failures.
79 private volatile boolean useHBaseChecksum;
80
81 // In the case of a checksum failure, do these many succeeding
82 // reads without hbase checksum verification.
83 private volatile int hbaseChecksumOffCount = -1;
84
85 private Boolean instanceOfCanUnbuffer = null;
86 // Using reflection to get org.apache.hadoop.fs.CanUnbuffer#unbuffer method to avoid compilation
87 // errors against Hadoop pre 2.6.4 and 2.7.1 versions.
88 private Method unbuffer = null;
89
90 public FSDataInputStreamWrapper(FileSystem fs, Path path) throws IOException {
91 this(fs, null, path);
92 }
93
94 public FSDataInputStreamWrapper(FileSystem fs, FileLink link) throws IOException {
95 this(fs, link, null);
96 }
97
98 private FSDataInputStreamWrapper(FileSystem fs, FileLink link, Path path) throws IOException {
99 assert (path == null) != (link == null);
100 this.path = path;
101 this.link = link;
102 this.doCloseStreams = true;
103 // If the fs is not an instance of HFileSystem, then create an instance of HFileSystem
104 // that wraps over the specified fs. In this case, we will not be able to avoid
105 // checksumming inside the filesystem.
106 this.hfs = (fs instanceof HFileSystem) ? (HFileSystem)fs : new HFileSystem(fs);
107
108 // Initially we are going to read the tail block. Open the reader w/FS checksum.
109 this.useHBaseChecksumConfigured = this.useHBaseChecksum = false;
110 this.stream = (link != null) ? link.open(hfs) : hfs.open(path);
111 }
112
113 /**
114 * Prepares the streams for block reader. NOT THREAD SAFE. Must be called once, after any
115 * reads finish and before any other reads start (what happens in reality is we read the
116 * tail, then call this based on what's in the tail, then read blocks).
117 * @param forceNoHBaseChecksum Force not using HBase checksum.
118 */
119 public void prepareForBlockReader(boolean forceNoHBaseChecksum) throws IOException {
120 if (hfs == null) return;
121 assert this.stream != null && !this.useHBaseChecksumConfigured;
122 boolean useHBaseChecksum =
123 !forceNoHBaseChecksum && hfs.useHBaseChecksum() && (hfs.getNoChecksumFs() != hfs);
124
125 if (useHBaseChecksum) {
126 FileSystem fsNc = hfs.getNoChecksumFs();
127 this.streamNoFsChecksum = (link != null) ? link.open(fsNc) : fsNc.open(path);
128 this.useHBaseChecksumConfigured = this.useHBaseChecksum = useHBaseChecksum;
129 // Close the checksum stream; we will reopen it if we get an HBase checksum failure.
130 this.stream.close();
131 this.stream = null;
132 }
133 }
134
135 /** For use in tests. */
136 @VisibleForTesting
137 public FSDataInputStreamWrapper(FSDataInputStream fsdis) {
138 this(fsdis, fsdis);
139 }
140
141 /** For use in tests. */
142 @VisibleForTesting
143 public FSDataInputStreamWrapper(FSDataInputStream fsdis, FSDataInputStream noChecksum) {
144 doCloseStreams = false;
145 stream = fsdis;
146 streamNoFsChecksum = noChecksum;
147 path = null;
148 link = null;
149 hfs = null;
150 useHBaseChecksumConfigured = useHBaseChecksum = false;
151 }
152
153 /**
154 * @return Whether we are presently using HBase checksum.
155 */
156 public boolean shouldUseHBaseChecksum() {
157 return this.useHBaseChecksum;
158 }
159
160 /**
161 * Get the stream to use. Thread-safe.
162 * @param useHBaseChecksum must be the value that shouldUseHBaseChecksum has returned
163 * at some point in the past, otherwise the result is undefined.
164 */
165 public FSDataInputStream getStream(boolean useHBaseChecksum) {
166 return useHBaseChecksum ? this.streamNoFsChecksum : this.stream;
167 }
168
169 /**
170 * Read from non-checksum stream failed, fall back to FS checksum. Thread-safe.
171 * @param offCount For how many checksumOk calls to turn off the HBase checksum.
172 */
173 public FSDataInputStream fallbackToFsChecksum(int offCount) throws IOException {
174 // checksumOffCount is speculative, but let's try to reset it less.
175 boolean partOfConvoy = false;
176 if (this.stream == null) {
177 synchronized (streamNoFsChecksumFirstCreateLock) {
178 partOfConvoy = (this.stream != null);
179 if (!partOfConvoy) {
180 this.stream = (link != null) ? link.open(hfs) : hfs.open(path);
181 }
182 }
183 }
184 if (!partOfConvoy) {
185 this.useHBaseChecksum = false;
186 this.hbaseChecksumOffCount = offCount;
187 }
188 return this.stream;
189 }
190
191 /** Report that checksum was ok, so we may ponder going back to HBase checksum. */
192 public void checksumOk() {
193 if (this.useHBaseChecksumConfigured && !this.useHBaseChecksum
194 && (this.hbaseChecksumOffCount-- < 0)) {
195 // The stream we need is already open (because we were using HBase checksum in the past).
196 assert this.streamNoFsChecksum != null;
197 this.useHBaseChecksum = true;
198 }
199 }
200
201 /** Close stream(s) if necessary. */
202 public void close() throws IOException {
203 if (!doCloseStreams) return;
204 try {
205 if (stream != streamNoFsChecksum && streamNoFsChecksum != null) {
206 streamNoFsChecksum.close();
207 streamNoFsChecksum = null;
208 }
209 } finally {
210 if (stream != null) {
211 stream.close();
212 stream = null;
213 }
214 }
215 }
216
217 public HFileSystem getHfs() {
218 return this.hfs;
219 }
220
221 /**
222 * This will free sockets and file descriptors held by the stream only when the stream implements
223 * org.apache.hadoop.fs.CanUnbuffer. NOT THREAD SAFE. Must be called only when all the clients
224 * using this stream to read the blocks have finished reading. If by chance the stream is
225 * unbuffered and there are clients still holding this stream for read then on next client read
226 * request a new socket will be opened by Datanode without client knowing about it and will serve
227 * its read request. Note: If this socket is idle for some time then the DataNode will close the
228 * socket and the socket will move into CLOSE_WAIT state and on the next client request on this
229 * stream, the current socket will be closed and a new socket will be opened to serve the
230 * requests.
231 */
232 @SuppressWarnings({ "rawtypes" })
233 public void unbuffer() {
234 FSDataInputStream stream = this.getStream(this.shouldUseHBaseChecksum());
235 if (stream != null) {
236 InputStream wrappedStream = stream.getWrappedStream();
237 // CanUnbuffer interface was added as part of HDFS-7694 and the fix is available in Hadoop
238 // 2.6.4+ and 2.7.1+ versions only so check whether the stream object implements the
239 // CanUnbuffer interface or not and based on that call the unbuffer api.
240 final Class<? extends InputStream> streamClass = wrappedStream.getClass();
241 if (this.instanceOfCanUnbuffer == null) {
242 // To ensure we compute whether the stream is instance of CanUnbuffer only once.
243 this.instanceOfCanUnbuffer = false;
244 Class<?>[] streamInterfaces = streamClass.getInterfaces();
245 for (Class c : streamInterfaces) {
246 if (c.getCanonicalName().toString().equals("org.apache.hadoop.fs.CanUnbuffer")) {
247 try {
248 this.unbuffer = streamClass.getDeclaredMethod("unbuffer");
249 } catch (NoSuchMethodException | SecurityException e) {
250 LOG.warn("Failed to find 'unbuffer' method in class " + streamClass
251 + " . So there may be a TCP socket connection "
252 + "left open in CLOSE_WAIT state.",
253 e);
254 return;
255 }
256 this.instanceOfCanUnbuffer = true;
257 break;
258 }
259 }
260 }
261 if (this.instanceOfCanUnbuffer) {
262 try {
263 this.unbuffer.invoke(wrappedStream);
264 } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
265 LOG.warn("Failed to invoke 'unbuffer' method in class " + streamClass
266 + " . So there may be a TCP socket connection left open in CLOSE_WAIT state.",
267 e);
268 }
269 } else {
270 LOG.warn("Failed to find 'unbuffer' method in class " + streamClass
271 + " . So there may be a TCP socket connection "
272 + "left open in CLOSE_WAIT state. For more details check "
273 + "https://issues.apache.org/jira/browse/HBASE-9393");
274 }
275 }
276 }
277 }