001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.io.hfile;
019
020import static org.junit.Assert.assertEquals;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.Collection;
025import java.util.List;
026import java.util.Set;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.hbase.Cell;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseTestingUtility;
031import org.apache.hadoop.hbase.HColumnDescriptor;
032import org.apache.hadoop.hbase.HRegionInfo;
033import org.apache.hadoop.hbase.HTableDescriptor;
034import org.apache.hadoop.hbase.TableName;
035import org.apache.hadoop.hbase.client.Put;
036import org.apache.hadoop.hbase.client.Scan;
037import org.apache.hadoop.hbase.regionserver.HRegion;
038import org.apache.hadoop.hbase.regionserver.HStore;
039import org.apache.hadoop.hbase.regionserver.InternalScanner;
040import org.apache.hadoop.hbase.testclassification.IOTests;
041import org.apache.hadoop.hbase.testclassification.MediumTests;
042import org.apache.hadoop.hbase.util.Bytes;
043import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
044import org.apache.hadoop.hbase.util.Threads;
045import org.junit.ClassRule;
046import org.junit.Test;
047import org.junit.experimental.categories.Category;
048import org.junit.runner.RunWith;
049import org.junit.runners.Parameterized;
050import org.junit.runners.Parameterized.Parameters;
051import org.slf4j.Logger;
052import org.slf4j.LoggerFactory;
053
054/**
055 * Test the optimization that does not scan files where all timestamps are
056 * expired.
057 */
058@RunWith(Parameterized.class)
059@Category({IOTests.class, MediumTests.class})
060public class TestScannerSelectionUsingTTL {
061
062  @ClassRule
063  public static final HBaseClassTestRule CLASS_RULE =
064      HBaseClassTestRule.forClass(TestScannerSelectionUsingTTL.class);
065
066  private static final Logger LOG =
067      LoggerFactory.getLogger(TestScannerSelectionUsingTTL.class);
068
069  private static final HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
070  private static TableName TABLE = TableName.valueOf("myTable");
071  private static String FAMILY = "myCF";
072  private static byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY);
073
074  private static final int TTL_SECONDS = 10;
075  private static final int TTL_MS = TTL_SECONDS * 1000;
076
077  private static final int NUM_EXPIRED_FILES = 2;
078  private static final int NUM_ROWS = 8;
079  private static final int NUM_COLS_PER_ROW = 5;
080
081  public final int numFreshFiles, totalNumFiles;
082
083  /** Whether we are specifying the exact files to compact */
084  private final boolean explicitCompaction;
085
086  @Parameters
087  public static Collection<Object[]> parameters() {
088    List<Object[]> params = new ArrayList<>();
089    for (int numFreshFiles = 1; numFreshFiles <= 3; ++numFreshFiles) {
090      for (boolean explicitCompaction : new boolean[] { false, true }) {
091        params.add(new Object[] { numFreshFiles, explicitCompaction });
092      }
093    }
094    return params;
095  }
096
097  public TestScannerSelectionUsingTTL(int numFreshFiles,
098      boolean explicitCompaction) {
099    this.numFreshFiles = numFreshFiles;
100    this.totalNumFiles = numFreshFiles + NUM_EXPIRED_FILES;
101    this.explicitCompaction = explicitCompaction;
102  }
103
104  @Test
105  public void testScannerSelection() throws IOException {
106    Configuration conf = TEST_UTIL.getConfiguration();
107    conf.setBoolean("hbase.store.delete.expired.storefile", false);
108    HColumnDescriptor hcd =
109      new HColumnDescriptor(FAMILY_BYTES)
110          .setMaxVersions(Integer.MAX_VALUE)
111          .setTimeToLive(TTL_SECONDS);
112    HTableDescriptor htd = new HTableDescriptor(TABLE);
113    htd.addFamily(hcd);
114    HRegionInfo info = new HRegionInfo(TABLE);
115    HRegion region = HBaseTestingUtility.createRegionAndWAL(info,
116      TEST_UTIL.getDataTestDir(info.getEncodedName()), conf, htd);
117
118    long ts = EnvironmentEdgeManager.currentTime();
119    long version = 0; //make sure each new set of Put's have a new ts
120    for (int iFile = 0; iFile < totalNumFiles; ++iFile) {
121      if (iFile == NUM_EXPIRED_FILES) {
122        Threads.sleepWithoutInterrupt(TTL_MS);
123        version += TTL_MS;
124      }
125
126      for (int iRow = 0; iRow < NUM_ROWS; ++iRow) {
127        Put put = new Put(Bytes.toBytes("row" + iRow));
128        for (int iCol = 0; iCol < NUM_COLS_PER_ROW; ++iCol) {
129          put.addColumn(FAMILY_BYTES, Bytes.toBytes("col" + iCol), ts + version,
130                  Bytes.toBytes("value" + iFile + "_" + iRow + "_" + iCol));
131        }
132        region.put(put);
133      }
134      region.flush(true);
135      version++;
136    }
137
138    Scan scan = new Scan();
139    scan.setMaxVersions(Integer.MAX_VALUE);
140    CacheConfig cacheConf = new CacheConfig(conf);
141    LruBlockCache cache = (LruBlockCache) cacheConf.getBlockCache();
142    cache.clearCache();
143    InternalScanner scanner = region.getScanner(scan);
144    List<Cell> results = new ArrayList<>();
145    final int expectedKVsPerRow = numFreshFiles * NUM_COLS_PER_ROW;
146    int numReturnedRows = 0;
147    LOG.info("Scanning the entire table");
148    while (scanner.next(results) || results.size() > 0) {
149      assertEquals(expectedKVsPerRow, results.size());
150      ++numReturnedRows;
151      results.clear();
152    }
153    assertEquals(NUM_ROWS, numReturnedRows);
154    Set<String> accessedFiles = cache.getCachedFileNamesForTest();
155    LOG.debug("Files accessed during scan: " + accessedFiles);
156
157    // Exercise both compaction codepaths.
158    if (explicitCompaction) {
159      HStore store = region.getStore(FAMILY_BYTES);
160      store.compactRecentForTestingAssumingDefaultPolicy(totalNumFiles);
161    } else {
162      region.compact(false);
163    }
164
165    HBaseTestingUtility.closeRegionAndWAL(region);
166  }
167}