View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with this
4    * work for additional information regarding copyright ownership. The ASF
5    * licenses this file to you under the Apache License, Version 2.0 (the
6    * "License"); you may not use this file except in compliance with the License.
7    * You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14   * License for the specific language governing permissions and limitations
15   * under the License.
16   */
17  package org.apache.hadoop.hbase.io.encoding;
18  
19  import java.nio.ByteBuffer;
20  
21  import org.apache.hadoop.hbase.Cell;
22  import org.apache.hadoop.hbase.CellUtil;
23  import org.apache.hadoop.hbase.HConstants;
24  import org.apache.hadoop.hbase.KeyValue;
25  import org.apache.hadoop.hbase.KeyValue.KVComparator;
26  import org.apache.hadoop.hbase.NoTagsKeyValue;
27  import org.apache.hadoop.hbase.classification.InterfaceAudience;
28  import org.apache.hadoop.hbase.io.encoding.AbstractDataBlockEncoder.AbstractEncodedSeeker;
29  import org.apache.hadoop.hbase.util.ByteBufferUtils;
30  import org.apache.hadoop.hbase.util.Bytes;
31  import org.apache.hadoop.hbase.util.SimpleMutableByteRange;
32  import org.apache.hadoop.io.WritableUtils;
33  
34  @InterfaceAudience.Private
35  public class RowIndexSeekerV1 extends AbstractEncodedSeeker {
36  
37    private ByteBuffer currentBuffer;
38    private SeekerState current = new SeekerState(); // always valid
39    private SeekerState previous = new SeekerState(); // may not be valid
40  
41    private int rowNumber;
42    private ByteBuffer rowOffsets = null;
43  
44    public RowIndexSeekerV1(KVComparator comparator,
45        HFileBlockDecodingContext decodingCtx) {
46      super(comparator, decodingCtx);
47    }
48  
49    @Override
50    public void setCurrentBuffer(ByteBuffer buffer) {
51      int onDiskSize = Bytes.toIntUnsafe(buffer.array(), buffer.arrayOffset()
52          + buffer.limit() - Bytes.SIZEOF_INT);
53      // int onDiskSize = buffer.getInt(buffer.limit() - Bytes.SIZEOF_INT);
54  
55      // Data part
56      ByteBuffer dup = buffer.duplicate();
57      dup.position(buffer.position());
58      dup.limit(buffer.position() + onDiskSize);
59      currentBuffer = dup.slice();
60      current.currentBuffer = currentBuffer;
61      ByteBufferUtils.skip(buffer, onDiskSize);
62  
63      // Row offset
64      rowNumber = buffer.getInt();
65      // equals Bytes.SIZEOF_INT * rowNumber
66      int totalRowOffsetsLength = rowNumber << 2;
67      ByteBuffer rowDup = buffer.duplicate();
68      rowDup.position(buffer.position());
69      rowDup.limit(buffer.position() + totalRowOffsetsLength);
70      rowOffsets = rowDup.slice();
71  
72      decodeFirst();
73    }
74  
75    @Override
76    public ByteBuffer getKeyDeepCopy() {
77      ByteBuffer keyBuffer = ByteBuffer.allocate(current.keyLength);
78      keyBuffer.put(current.keyBuffer.getBytes(), current.keyBuffer.getOffset(),
79          current.keyLength);
80      keyBuffer.rewind();
81      return keyBuffer;
82    }
83  
84    @Override
85    public ByteBuffer getValueShallowCopy() {
86      ByteBuffer dup = currentBuffer.duplicate();
87      dup.position(current.valueOffset);
88      dup.limit(current.valueOffset + current.valueLength);
89      return dup.slice();
90    }
91  
92    ByteBuffer getKeyValueBuffer() {
93      ByteBuffer kvBuffer = createKVBuffer();
94      kvBuffer.putInt(current.keyLength);
95      kvBuffer.putInt(current.valueLength);
96      kvBuffer.put(current.keyBuffer.getBytes(), current.keyBuffer.getOffset(),
97          current.keyLength);
98      ByteBufferUtils.copyFromBufferToBuffer(kvBuffer, currentBuffer,
99          current.valueOffset, current.valueLength);
100     if (current.tagsLength > 0) {
101       // Put short as unsigned
102       kvBuffer.put((byte) (current.tagsLength >> 8 & 0xff));
103       kvBuffer.put((byte) (current.tagsLength & 0xff));
104       if (current.tagsOffset != -1) {
105         ByteBufferUtils.copyFromBufferToBuffer(kvBuffer, currentBuffer,
106             current.tagsOffset, current.tagsLength);
107       }
108     }
109     if (includesMvcc()) {
110       ByteBufferUtils.writeVLong(kvBuffer, current.getSequenceId());
111     }
112     kvBuffer.rewind();
113     return kvBuffer;
114   }
115 
116   protected ByteBuffer createKVBuffer() {
117     int kvBufSize = (int) KeyValue.getKeyValueDataStructureSize(
118         current.keyLength, current.valueLength, current.tagsLength);
119     if (includesMvcc()) {
120       kvBufSize += WritableUtils.getVIntSize(current.getSequenceId());
121     }
122     ByteBuffer kvBuffer = ByteBuffer.allocate(kvBufSize);
123     return kvBuffer;
124   }
125 
126   @Override
127   public Cell getKeyValue() {
128     return current.toCell();
129   }
130 
131   @Override
132   public void rewind() {
133     currentBuffer.rewind();
134     decodeFirst();
135   }
136 
137   @Override
138   public boolean next() {
139     if (!currentBuffer.hasRemaining()) {
140       return false;
141     }
142     decodeNext();
143     previous.invalidate();
144     return true;
145   }
146 
147   @Override
148   public int seekToKeyInBlock(byte[] key, int offset, int length,
149       boolean seekBefore) {
150     return seekToKeyInBlock(new KeyValue.KeyOnlyKeyValue(key, offset, length),
151         seekBefore);
152   }
153 
154   private int binarySearch(Cell seekCell, boolean seekBefore) {
155     int low = 0;
156     int high = rowNumber - 1;
157     int mid = (low + high) >>> 1;
158     int comp = 0;
159     SimpleMutableByteRange row = new SimpleMutableByteRange();
160     while (low <= high) {
161       mid = (low + high) >>> 1;
162       getRow(mid, row);
163       comp = comparator.compareRows(row.getBytes(), row.getOffset(),
164           row.getLength(), seekCell.getRowArray(), seekCell.getRowOffset(),
165           seekCell.getRowLength());
166       if (comp < 0) {
167         low = mid + 1;
168       } else if (comp > 0) {
169         high = mid - 1;
170       } else {
171         // key found
172         if (seekBefore) {
173           return mid - 1;
174         } else {
175           return mid;
176         }
177       }
178     }
179     // key not found.
180     if (comp > 0) {
181       return mid - 1;
182     } else {
183       return mid;
184     }
185   }
186 
187   private void getRow(int index, SimpleMutableByteRange row) {
188     int offset = Bytes.toIntUnsafe(rowOffsets.array(), rowOffsets.arrayOffset()
189         + (index << 2)); // index * Bytes.SIZEOF_INT
190     int position = currentBuffer.arrayOffset() + offset + Bytes.SIZEOF_LONG;
191     short rowLen = Bytes.toShortUnsafe(currentBuffer.array(), position);
192     row.set(currentBuffer.array(), position + Bytes.SIZEOF_SHORT, rowLen);
193   }
194 
195   @Override
196   public int seekToKeyInBlock(Cell seekCell, boolean seekBefore) {
197     previous.invalidate();
198     int index = binarySearch(seekCell, seekBefore);
199     if (index < 0) {
200       return HConstants.INDEX_KEY_MAGIC; // using optimized index key
201     } else {
202       int offset = Bytes.toIntUnsafe(rowOffsets.array(),
203           rowOffsets.arrayOffset() + (index << 2));
204       if (offset != 0) {
205         decodeAtPosition(offset);
206       }
207     }
208     do {
209       int comp;
210       comp = comparator.compareOnlyKeyPortion(seekCell, current.currentKey);
211       if (comp == 0) { // exact match
212         if (seekBefore) {
213           if (!previous.isValid()) {
214             // The caller (seekBefore) has to ensure that we are not at the
215             // first key in the block.
216             throw new IllegalStateException("Cannot seekBefore if "
217                 + "positioned at the first key in the block: key="
218                 + Bytes.toStringBinary(seekCell.getRowArray()));
219           }
220           moveToPrevious();
221           return 1;
222         }
223         return 0;
224       }
225 
226       if (comp < 0) { // already too large, check previous
227         if (previous.isValid()) {
228           moveToPrevious();
229         } else {
230           return HConstants.INDEX_KEY_MAGIC; // using optimized index key
231         }
232         return 1;
233       }
234 
235       // move to next, if more data is available
236       if (currentBuffer.hasRemaining()) {
237         previous.copyFromNext(current);
238         decodeNext();
239       } else {
240         break;
241       }
242     } while (true);
243 
244     // we hit the end of the block, not an exact match
245     return 1;
246   }
247 
248   private void moveToPrevious() {
249     if (!previous.isValid()) {
250       throw new IllegalStateException(
251           "Can move back only once and not in first key in the block.");
252     }
253 
254     SeekerState tmp = previous;
255     previous = current;
256     current = tmp;
257 
258     // move after last key value
259     currentBuffer.position(current.nextKvOffset);
260     previous.invalidate();
261   }
262 
263   @Override
264   public int compareKey(KVComparator comparator, byte[] key, int offset,
265       int length) {
266     return comparator.compareFlatKey(key, offset, length,
267         current.keyBuffer.getBytes(), current.keyBuffer.getOffset(),
268         current.keyBuffer.getLength());
269   }
270 
271   @Override
272   public int compareKey(KVComparator comparator, Cell key) {
273     return comparator.compareOnlyKeyPortion(key, new KeyValue.KeyOnlyKeyValue(
274         current.keyBuffer.getBytes(), current.keyBuffer.getOffset(),
275         current.keyBuffer.getLength()));
276   }
277 
278   protected void decodeFirst() {
279     decodeNext();
280     previous.invalidate();
281   }
282 
283   protected void decodeAtPosition(int position) {
284     currentBuffer.position(position);
285     decodeNext();
286     previous.invalidate();
287   }
288 
289   protected void decodeNext() {
290     current.startOffset = currentBuffer.position();
291     int p = currentBuffer.position() + currentBuffer.arrayOffset();
292     long ll = Bytes.toLong(currentBuffer.array(), p);
293     // Read top half as an int of key length and bottom int as value length
294     current.keyLength = (int) (ll >> Integer.SIZE);
295     current.valueLength = (int) (Bytes.MASK_FOR_LOWER_INT_IN_LONG ^ ll);
296     ByteBufferUtils.skip(currentBuffer, Bytes.SIZEOF_LONG);
297     // key part
298     current.keyBuffer.set(currentBuffer.array(), currentBuffer.arrayOffset()
299         + currentBuffer.position(), current.keyLength);
300     ByteBufferUtils.skip(currentBuffer, current.keyLength);
301     // value part
302     current.valueOffset = currentBuffer.position();
303     ByteBufferUtils.skip(currentBuffer, current.valueLength);
304     if (includesTags()) {
305       decodeTags();
306     }
307     if (includesMvcc()) {
308       current.memstoreTS = ByteBufferUtils.readVLong(currentBuffer);
309     } else {
310       current.memstoreTS = 0;
311     }
312     current.nextKvOffset = currentBuffer.position();
313     current.setKey(current.keyBuffer.getBytes(), current.keyBuffer.getOffset(),
314         current.keyBuffer.getLength());
315   }
316 
317   protected void decodeTags() {
318     current.tagsLength = currentBuffer.getShort();
319     current.tagsOffset = currentBuffer.position();
320     ByteBufferUtils.skip(currentBuffer, current.tagsLength);
321   }
322 
323   protected class SeekerState {
324     /**
325      * The size of a (key length, value length) tuple that prefixes each entry
326      * in a data block.
327      */
328     public final static int KEY_VALUE_LEN_SIZE = 2 * Bytes.SIZEOF_INT;
329 
330     protected ByteBuffer currentBuffer;
331     protected int startOffset = -1;
332     protected int valueOffset = -1;
333     protected int keyLength;
334     protected int valueLength;
335     protected int tagsLength = 0;
336     protected int tagsOffset = -1;
337 
338     protected SimpleMutableByteRange keyBuffer = new SimpleMutableByteRange();
339     protected long memstoreTS;
340     protected int nextKvOffset;
341     protected KeyValue.KeyOnlyKeyValue currentKey = new KeyValue.KeyOnlyKeyValue();
342 
343     protected boolean isValid() {
344       return valueOffset != -1;
345     }
346 
347     protected void invalidate() {
348       valueOffset = -1;
349       currentKey = new KeyValue.KeyOnlyKeyValue();
350       currentBuffer = null;
351     }
352 
353     protected void setKey(byte[] key, int offset, int length) {
354       currentKey.setKey(key, offset, length);
355     }
356 
357     protected long getSequenceId() {
358       return memstoreTS;
359     }
360 
361     /**
362      * Copy the state from the next one into this instance (the previous state
363      * placeholder). Used to save the previous state when we are advancing the
364      * seeker to the next key/value.
365      */
366     protected void copyFromNext(SeekerState nextState) {
367       keyBuffer.set(nextState.keyBuffer.getBytes(),
368           nextState.keyBuffer.getOffset(), nextState.keyBuffer.getLength());
369       currentKey.setKey(nextState.keyBuffer.getBytes(),
370           nextState.keyBuffer.getOffset(), nextState.keyBuffer.getLength());
371 
372       startOffset = nextState.startOffset;
373       valueOffset = nextState.valueOffset;
374       keyLength = nextState.keyLength;
375       valueLength = nextState.valueLength;
376       nextKvOffset = nextState.nextKvOffset;
377       memstoreTS = nextState.memstoreTS;
378       currentBuffer = nextState.currentBuffer;
379       tagsOffset = nextState.tagsOffset;
380       tagsLength = nextState.tagsLength;
381     }
382 
383     @Override
384     public String toString() {
385       return CellUtil.getCellKeyAsString(toCell());
386     }
387 
388     protected int getCellBufSize() {
389       int kvBufSize = KEY_VALUE_LEN_SIZE + keyLength + valueLength;
390       if (includesTags()) {
391         kvBufSize += Bytes.SIZEOF_SHORT + tagsLength;
392       }
393       return kvBufSize;
394     }
395 
396     protected Cell formNoTagsKeyValue() {
397       NoTagsKeyValue ret = new NoTagsKeyValue(currentBuffer.array(),
398           currentBuffer.arrayOffset() + startOffset, getCellBufSize());
399       if (includesMvcc()) {
400         ret.setSequenceId(memstoreTS);
401       }
402       return ret;
403     }
404 
405     public Cell toCell() {
406       if (tagsOffset > 0) {
407         KeyValue ret = new KeyValue(currentBuffer.array(),
408             currentBuffer.arrayOffset() + startOffset, getCellBufSize());
409         if (includesMvcc()) {
410           ret.setSequenceId(memstoreTS);
411         }
412         return ret;
413       } else {
414         return formNoTagsKeyValue();
415       }
416     }
417   }
418 
419 }