001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.encoding;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.Collection;
026import java.util.List;
027import java.util.Map;
028import org.apache.hadoop.hbase.ArrayBackedTag;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseTestingUtility;
031import org.apache.hadoop.hbase.HColumnDescriptor;
032import org.apache.hadoop.hbase.HConstants;
033import org.apache.hadoop.hbase.KeyValue;
034import org.apache.hadoop.hbase.Tag;
035import org.apache.hadoop.hbase.client.Durability;
036import org.apache.hadoop.hbase.client.Get;
037import org.apache.hadoop.hbase.client.Put;
038import org.apache.hadoop.hbase.client.Result;
039import org.apache.hadoop.hbase.io.hfile.CacheConfig;
040import org.apache.hadoop.hbase.io.hfile.HFile;
041import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
042import org.apache.hadoop.hbase.regionserver.BloomType;
043import org.apache.hadoop.hbase.regionserver.HRegion;
044import org.apache.hadoop.hbase.regionserver.Region;
045import org.apache.hadoop.hbase.testclassification.IOTests;
046import org.apache.hadoop.hbase.testclassification.MediumTests;
047import org.apache.hadoop.hbase.util.Bytes;
048import org.apache.hadoop.hbase.util.LoadTestKVGenerator;
049import org.apache.hadoop.hbase.util.Strings;
050import org.junit.ClassRule;
051import org.junit.Test;
052import org.junit.experimental.categories.Category;
053import org.junit.runner.RunWith;
054import org.junit.runners.Parameterized;
055import org.junit.runners.Parameterized.Parameters;
056
057/**
058 * Tests encoded seekers by loading and reading values.
059 */
060@Category({IOTests.class, MediumTests.class})
061@RunWith(Parameterized.class)
062public class TestEncodedSeekers {
063
064  @ClassRule
065  public static final HBaseClassTestRule CLASS_RULE =
066      HBaseClassTestRule.forClass(TestEncodedSeekers.class);
067
068  private static final String TABLE_NAME = "encodedSeekersTable";
069  private static final String CF_NAME = "encodedSeekersCF";
070  private static final byte[] CF_BYTES = Bytes.toBytes(CF_NAME);
071  private static final int MAX_VERSIONS = 5;
072
073  private static final int BLOCK_SIZE = 64 * 1024;
074  private static final int MIN_VALUE_SIZE = 30;
075  private static final int MAX_VALUE_SIZE = 60;
076  private static final int NUM_ROWS = 1003;
077  private static final int NUM_COLS_PER_ROW = 20;
078  private static final int NUM_HFILES = 4;
079  private static final int NUM_ROWS_PER_FLUSH = NUM_ROWS / NUM_HFILES;
080
081  private final HBaseTestingUtility testUtil = HBaseTestingUtility.createLocalHTU();
082  private final DataBlockEncoding encoding;
083  private final boolean includeTags;
084  private final boolean compressTags;
085
086  /** Enable when debugging */
087  private static final boolean VERBOSE = false;
088
089  @Parameters
090  public static Collection<Object[]> parameters() {
091    List<Object[]> paramList = new ArrayList<>();
092    for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
093      for (boolean includeTags : new boolean[] { false, true }) {
094        for (boolean compressTags : new boolean[] { false, true }) {
095          paramList.add(new Object[] { encoding, includeTags, compressTags });
096        }
097      }
098    }
099    return paramList;
100  }
101
102  public TestEncodedSeekers(DataBlockEncoding encoding, boolean includeTags, boolean compressTags) {
103    this.encoding = encoding;
104    this.includeTags = includeTags;
105    this.compressTags = compressTags;
106  }
107
108  @Test
109  public void testEncodedSeeker() throws IOException {
110    System.err.println("Testing encoded seekers for encoding : " + encoding + ", includeTags : "
111        + includeTags + ", compressTags : " + compressTags);
112    if(includeTags) {
113      testUtil.getConfiguration().setInt(HFile.FORMAT_VERSION_KEY, 3);
114    }
115    LruBlockCache cache =
116      (LruBlockCache)new CacheConfig(testUtil.getConfiguration()).getBlockCache();
117    cache.clearCache();
118    // Need to disable default row bloom filter for this test to pass.
119    HColumnDescriptor hcd = (new HColumnDescriptor(CF_NAME)).setMaxVersions(MAX_VERSIONS).
120        setDataBlockEncoding(encoding).
121        setBlocksize(BLOCK_SIZE).
122        setBloomFilterType(BloomType.NONE).
123        setCompressTags(compressTags);
124    HRegion region = testUtil.createTestRegion(TABLE_NAME, hcd);
125
126    //write the data, but leave some in the memstore
127    doPuts(region);
128
129    //verify correctness when memstore contains data
130    doGets(region);
131
132    //verify correctness again after compacting
133    region.compact(false);
134    doGets(region);
135
136    Map<DataBlockEncoding, Integer> encodingCounts = cache.getEncodingCountsForTest();
137
138    // Ensure that compactions don't pollute the cache with unencoded blocks
139    // in case of in-cache-only encoding.
140    System.err.println("encodingCounts=" + encodingCounts);
141    assertEquals(1, encodingCounts.size());
142    DataBlockEncoding encodingInCache = encodingCounts.keySet().iterator().next();
143    assertEquals(encoding, encodingInCache);
144    assertTrue(encodingCounts.get(encodingInCache) > 0);
145  }
146
147
148  private void doPuts(HRegion region) throws IOException{
149    LoadTestKVGenerator dataGenerator = new LoadTestKVGenerator(MIN_VALUE_SIZE, MAX_VALUE_SIZE);
150     for (int i = 0; i < NUM_ROWS; ++i) {
151      byte[] key = LoadTestKVGenerator.md5PrefixedKey(i).getBytes();
152      for (int j = 0; j < NUM_COLS_PER_ROW; ++j) {
153        Put put = new Put(key);
154        put.setDurability(Durability.ASYNC_WAL);
155        byte[] col = Bytes.toBytes(String.valueOf(j));
156        byte[] value = dataGenerator.generateRandomSizeValue(key, col);
157        if (includeTags) {
158          Tag[] tag = new Tag[1];
159          tag[0] = new ArrayBackedTag((byte) 1, "Visibility");
160          KeyValue kv = new KeyValue(key, CF_BYTES, col, HConstants.LATEST_TIMESTAMP, value, tag);
161          put.add(kv);
162        } else {
163          put.addColumn(CF_BYTES, col, value);
164        }
165        if(VERBOSE){
166          KeyValue kvPut = new KeyValue(key, CF_BYTES, col, value);
167          System.err.println(Strings.padFront(i+"", ' ', 4)+" "+kvPut);
168        }
169        region.put(put);
170      }
171      if (i % NUM_ROWS_PER_FLUSH == 0) {
172        region.flush(true);
173      }
174    }
175  }
176
177
178  private void doGets(Region region) throws IOException{
179    for (int i = 0; i < NUM_ROWS; ++i) {
180      final byte[] rowKey = LoadTestKVGenerator.md5PrefixedKey(i).getBytes();
181      for (int j = 0; j < NUM_COLS_PER_ROW; ++j) {
182        final String qualStr = String.valueOf(j);
183        if (VERBOSE) {
184          System.err.println("Reading row " + i + ", column " + j + " " + Bytes.toString(rowKey)+"/"
185              +qualStr);
186        }
187        final byte[] qualBytes = Bytes.toBytes(qualStr);
188        Get get = new Get(rowKey);
189        get.addColumn(CF_BYTES, qualBytes);
190        Result result = region.get(get);
191        assertEquals(1, result.size());
192        byte[] value = result.getValue(CF_BYTES, qualBytes);
193        assertTrue(LoadTestKVGenerator.verify(value, rowKey, qualBytes));
194      }
195    }
196  }
197
198}