View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  import java.io.IOException;
21  import java.util.ArrayList;
22  import java.util.Arrays;
23  import java.util.List;
24  
25  import junit.framework.TestCase;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.FileSystem;
31  import org.apache.hadoop.fs.Path;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.HColumnDescriptor;
34  import org.apache.hadoop.hbase.HConstants;
35  import org.apache.hadoop.hbase.HRegionInfo;
36  import org.apache.hadoop.hbase.HTableDescriptor;
37  import org.apache.hadoop.hbase.testclassification.SmallTests;
38  import org.apache.hadoop.hbase.TableName;
39  import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
40  import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
41  import org.junit.Assert;
42  import org.junit.Test;
43  import org.apache.hadoop.hbase.wal.DefaultWALProvider;
44  import org.apache.hadoop.hbase.wal.WALFactory;
45  import org.apache.hadoop.hbase.util.Bytes;
46  import org.apache.hadoop.hbase.util.FSUtils;
47  import org.junit.After;
48  import org.junit.Ignore;
49  import org.junit.experimental.categories.Category;
50  
51  import com.google.common.collect.Lists;
52  
53  @Category(SmallTests.class)
54  @Ignore
55  public class TestDefaultCompactSelection extends TestCompactionPolicy {
56    private final static Log LOG = LogFactory.getLog(TestDefaultCompactSelection.class);
57  
58    @Test
59    public void testCompactionRatio() throws IOException {
60      /**
61       * NOTE: these tests are specific to describe the implementation of the
62       * current compaction algorithm.  Developed to ensure that refactoring
63       * doesn't implicitly alter this.
64       */
65      long tooBig = maxSize + 1;
66  
67      // default case. preserve user ratio on size
68      compactEquals(sfCreate(100,50,23,12,12), 23, 12, 12);
69      // less than compact threshold = don't compact
70      compactEquals(sfCreate(100,50,25,12,12) /* empty */);
71      // greater than compact size = skip those
72      compactEquals(sfCreate(tooBig, tooBig, 700, 700, 700), 700, 700, 700);
73      // big size + threshold
74      compactEquals(sfCreate(tooBig, tooBig, 700,700) /* empty */);
75      // small files = don't care about ratio
76      compactEquals(sfCreate(7,1,1), 7,1,1);
77  
78      // don't exceed max file compact threshold
79      // note:  file selection starts with largest to smallest.
80      compactEquals(sfCreate(7, 6, 5, 4, 3, 2, 1), 5, 4, 3, 2, 1);
81  
82      compactEquals(sfCreate(50, 10, 10 ,10, 10), 10, 10, 10, 10);
83  
84      compactEquals(sfCreate(10, 10, 10, 10, 50), 10, 10, 10, 10);
85  
86      compactEquals(sfCreate(251, 253, 251, maxSize -1), 251, 253, 251);
87  
88      compactEquals(sfCreate(maxSize -1,maxSize -1,maxSize -1) /* empty */);
89  
90      // Always try and compact something to get below blocking storefile count
91      this.conf.setLong("hbase.hstore.compaction.min.size", 1);
92      store.storeEngine.getCompactionPolicy().setConf(conf);
93      compactEquals(sfCreate(512,256,128,64,32,16,8,4,2,1), 4,2,1);
94      this.conf.setLong("hbase.hstore.compaction.min.size", minSize);
95      store.storeEngine.getCompactionPolicy().setConf(conf);
96  
97      /* MAJOR COMPACTION */
98      // if a major compaction has been forced, then compact everything
99      compactEquals(sfCreate(50,25,12,12), true, 50, 25, 12, 12);
100     // also choose files < threshold on major compaction
101     compactEquals(sfCreate(12,12), true, 12, 12);
102     // even if one of those files is too big
103     compactEquals(sfCreate(tooBig, 12,12), true, tooBig, 12, 12);
104     // don't exceed max file compact threshold, even with major compaction
105     store.forceMajor = true;
106     compactEquals(sfCreate(7, 6, 5, 4, 3, 2, 1), 5, 4, 3, 2, 1);
107     store.forceMajor = false;
108     // if we exceed maxCompactSize, downgrade to minor
109     // if not, it creates a 'snowball effect' when files >> maxCompactSize:
110     // the last file in compaction is the aggregate of all previous compactions
111     compactEquals(sfCreate(100,50,23,12,12), true, 23, 12, 12);
112     conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 1);
113     conf.setFloat("hbase.hregion.majorcompaction.jitter", 0);
114     store.storeEngine.getCompactionPolicy().setConf(conf);
115     try {
116       // trigger an aged major compaction
117       compactEquals(sfCreate(50,25,12,12), 50, 25, 12, 12);
118       // major sure exceeding maxCompactSize also downgrades aged minors
119       compactEquals(sfCreate(100,50,23,12,12), 23, 12, 12);
120     } finally {
121       conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 1000*60*60*24);
122       conf.setFloat("hbase.hregion.majorcompaction.jitter", 0.20F);
123     }
124 
125     /* REFERENCES == file is from a region that was split */
126     // treat storefiles that have references like a major compaction
127     compactEquals(sfCreate(true, 100,50,25,12,12), 100, 50, 25, 12, 12);
128     // reference files shouldn't obey max threshold
129     compactEquals(sfCreate(true, tooBig, 12,12), tooBig, 12, 12);
130     // reference files should obey max file compact to avoid OOM
131     compactEquals(sfCreate(true, 7, 6, 5, 4, 3, 2, 1), 7, 6, 5, 4, 3);
132 
133     // empty case
134     compactEquals(new ArrayList<StoreFile>() /* empty */);
135     // empty case (because all files are too big)
136     compactEquals(sfCreate(tooBig, tooBig) /* empty */);
137   }
138 
139   @Test
140   public void testOffPeakCompactionRatio() throws IOException {
141     /*
142      * NOTE: these tests are specific to describe the implementation of the
143      * current compaction algorithm.  Developed to ensure that refactoring
144      * doesn't implicitly alter this.
145      */
146     // set an off-peak compaction threshold
147     this.conf.setFloat("hbase.hstore.compaction.ratio.offpeak", 5.0F);
148     store.storeEngine.getCompactionPolicy().setConf(this.conf);
149     // Test with and without the flag.
150     compactEquals(sfCreate(999, 50, 12, 12, 1), false, true, 50, 12, 12, 1);
151     compactEquals(sfCreate(999, 50, 12, 12, 1), 12, 12, 1);
152   }
153 
154   @Test
155   public void testStuckStoreCompaction() throws IOException {
156     // Select the smallest compaction if the store is stuck.
157     compactEquals(sfCreate(99,99,99,99,99,99, 30,30,30,30), 30, 30, 30);
158     // If not stuck, standard policy applies.
159     compactEquals(sfCreate(99,99,99,99,99, 30,30,30,30), 99, 30, 30, 30, 30);
160 
161     // Add sufficiently small files to compaction, though
162     compactEquals(sfCreate(99,99,99,99,99,99, 30,30,30,15), 30, 30, 30, 15);
163     // Prefer earlier compaction to latter if the benefit is not significant
164     compactEquals(sfCreate(99,99,99,99, 30,26,26,29,25,25), 30, 26, 26);
165     // Prefer later compaction if the benefit is significant.
166     compactEquals(sfCreate(99,99,99,99, 27,27,27,20,20,20), 20, 20, 20);
167   }
168 
169   @Test
170   public void testCompactionEmptyHFile() throws IOException {
171     // Set TTL
172     ScanInfo oldScanInfo = store.getScanInfo();
173     ScanInfo newScanInfo = new ScanInfo(oldScanInfo.getFamily(),
174         oldScanInfo.getMinVersions(), oldScanInfo.getMaxVersions(), 600,
175         oldScanInfo.getKeepDeletedCells(), oldScanInfo.getTimeToPurgeDeletes(),
176         oldScanInfo.getComparator());
177     store.setScanInfo(newScanInfo);
178     // Do not compact empty store file
179     List<StoreFile> candidates = sfCreate(0);
180     for (StoreFile file : candidates) {
181       if (file instanceof MockStoreFile) {
182         MockStoreFile mockFile = (MockStoreFile) file;
183         mockFile.setTimeRangeTracker(new TimeRangeTracker(-1, -1));
184         mockFile.setEntries(0);
185       }
186     }
187     // Test Default compactions
188     CompactionRequest result = ((RatioBasedCompactionPolicy) store.storeEngine
189         .getCompactionPolicy()).selectCompaction(candidates,
190         new ArrayList<StoreFile>(), false, false, false);
191     Assert.assertTrue(result.getFiles().size() == 0);
192     store.setScanInfo(oldScanInfo);
193   }
194 }