1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver.compactions;
19
20 import static org.junit.Assert.assertEquals;
21
22 import java.io.IOException;
23 import java.util.List;
24 import java.util.concurrent.ThreadLocalRandom;
25
26 import org.apache.hadoop.conf.Configuration;
27 import org.apache.hadoop.hbase.DoNotRetryIOException;
28 import org.apache.hadoop.hbase.HBaseTestingUtility;
29 import org.apache.hadoop.hbase.HColumnDescriptor;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.HTableDescriptor;
32 import org.apache.hadoop.hbase.MiniHBaseCluster;
33 import org.apache.hadoop.hbase.TableName;
34 import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
35 import org.apache.hadoop.hbase.client.HBaseAdmin;
36 import org.apache.hadoop.hbase.client.Put;
37 import org.apache.hadoop.hbase.client.Table;
38 import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
39 import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
40 import org.apache.hadoop.hbase.regionserver.HRegionServer;
41 import org.apache.hadoop.hbase.regionserver.HStore;
42 import org.apache.hadoop.hbase.regionserver.Region;
43 import org.apache.hadoop.hbase.regionserver.Store;
44 import org.apache.hadoop.hbase.testclassification.MediumTests;
45 import org.apache.hadoop.hbase.util.Bytes;
46 import org.apache.hadoop.hbase.util.EnvironmentEdge;
47 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
48 import org.apache.hadoop.hbase.util.JVMClusterUtil;
49 import org.apache.hadoop.hbase.util.TimeOffsetEnvironmentEdge;
50 import org.junit.AfterClass;
51 import org.junit.BeforeClass;
52 import org.junit.Rule;
53 import org.junit.Test;
54 import org.junit.experimental.categories.Category;
55 import org.junit.rules.ExpectedException;
56
57 @Category({ MediumTests.class })
58 public class TestFIFOCompactionPolicy {
59
60 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
61
62 private final byte[] family = Bytes.toBytes("f");
63
64 private final byte[] qualifier = Bytes.toBytes("q");
65
66 @Rule
67 public ExpectedException error = ExpectedException.none();
68
69 private Store getStoreWithName(TableName tableName) {
70 MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster();
71 List<JVMClusterUtil.RegionServerThread> rsts = cluster.getRegionServerThreads();
72 for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) {
73 HRegionServer hrs = rsts.get(i).getRegionServer();
74 for (Region region : hrs.getOnlineRegions(tableName)) {
75 return region.getStores().iterator().next();
76 }
77 }
78 return null;
79 }
80
81 private Store prepareData(TableName tableName) throws IOException {
82 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
83 HTableDescriptor desc = new HTableDescriptor(tableName);
84 desc.setConfiguration(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
85 FIFOCompactionPolicy.class.getName());
86 desc.setConfiguration(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
87 DisabledRegionSplitPolicy.class.getName());
88 HColumnDescriptor colDesc = new HColumnDescriptor(family);
89 colDesc.setTimeToLive(1);
90 desc.addFamily(colDesc);
91
92 admin.createTable(desc);
93 Table table = TEST_UTIL.getConnection().getTable(tableName);
94 TimeOffsetEnvironmentEdge edge =
95 (TimeOffsetEnvironmentEdge) EnvironmentEdgeManager.getDelegate();
96 for (int i = 0; i < 10; i++) {
97 for (int j = 0; j < 10; j++) {
98 byte[] value = new byte[128 * 1024];
99 ThreadLocalRandom.current().nextBytes(value);
100 table.put(new Put(Bytes.toBytes(i * 10 + j)).addColumn(family, qualifier, value));
101 }
102 admin.flush(tableName);
103 edge.increment(1001);
104 }
105 return getStoreWithName(tableName);
106 }
107
108 @BeforeClass
109 public static void setEnvironmentEdge() throws Exception {
110 EnvironmentEdge ee = new TimeOffsetEnvironmentEdge();
111 EnvironmentEdgeManager.injectEdge(ee);
112 Configuration conf = TEST_UTIL.getConfiguration();
113 conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000);
114 TEST_UTIL.startMiniCluster(1);
115 }
116
117 @AfterClass
118 public static void resetEnvironmentEdge() throws Exception {
119 TEST_UTIL.shutdownMiniCluster();
120 EnvironmentEdgeManager.reset();
121 }
122
123 @Test
124 public void testPurgeExpiredFiles() throws Exception {
125 TableName tableName = TableName.valueOf(getClass().getSimpleName());
126 final Store store = prepareData(tableName);
127 assertEquals(10, store.getStorefilesCount());
128 TEST_UTIL.getHBaseAdmin().majorCompact(tableName);
129 TEST_UTIL.waitFor(30000, new ExplainingPredicate<Exception>() {
130
131 @Override
132 public boolean evaluate() throws Exception {
133 return store.getStorefilesCount() == 1;
134 }
135
136 @Override
137 public String explainFailure() throws Exception {
138 return "The store file count " + store.getStorefilesCount() + " is still greater than 1";
139 }
140 });
141 }
142
143 @Test
144 public void testSanityCheckTTL() throws IOException {
145 error.expect(DoNotRetryIOException.class);
146 error.expectMessage("Default TTL is not supported");
147 TableName tableName = TableName.valueOf(getClass().getSimpleName() + "-TTL");
148 HTableDescriptor desc = new HTableDescriptor(tableName);
149 desc.setConfiguration(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
150 FIFOCompactionPolicy.class.getName());
151 desc.setConfiguration(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
152 DisabledRegionSplitPolicy.class.getName());
153 HColumnDescriptor colDesc = new HColumnDescriptor(family);
154 desc.addFamily(colDesc);
155 TEST_UTIL.getHBaseAdmin().createTable(desc);
156 }
157
158 @Test
159 public void testSanityCheckMinVersion() throws IOException {
160 error.expect(DoNotRetryIOException.class);
161 error.expectMessage("MIN_VERSION > 0 is not supported for FIFO compaction");
162 TableName tableName = TableName.valueOf(getClass().getSimpleName() + "-MinVersion");
163 HTableDescriptor desc = new HTableDescriptor(tableName);
164 desc.setConfiguration(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
165 FIFOCompactionPolicy.class.getName());
166 desc.setConfiguration(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
167 DisabledRegionSplitPolicy.class.getName());
168 HColumnDescriptor colDesc = new HColumnDescriptor(family);
169 colDesc.setTimeToLive(1);
170 colDesc.setMinVersions(1);
171 desc.addFamily(colDesc);
172 TEST_UTIL.getHBaseAdmin().createTable(desc);
173 }
174
175 @Test
176 public void testSanityCheckBlockingStoreFiles() throws IOException {
177 error.expect(DoNotRetryIOException.class);
178 error.expectMessage("blocking file count 'hbase.hstore.blockingStoreFiles'");
179 error.expectMessage("is below recommended minimum of 1000");
180 TableName tableName = TableName.valueOf(getClass().getSimpleName() + "-BlockingStoreFiles");
181 HTableDescriptor desc = new HTableDescriptor(tableName);
182 desc.setConfiguration(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
183 FIFOCompactionPolicy.class.getName());
184 desc.setConfiguration(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
185 DisabledRegionSplitPolicy.class.getName());
186 desc.setConfiguration(HStore.BLOCKING_STOREFILES_KEY, "10");
187 HColumnDescriptor colDesc = new HColumnDescriptor(family);
188 colDesc.setTimeToLive(1);
189 desc.addFamily(colDesc);
190 TEST_UTIL.getHBaseAdmin().createTable(desc);
191 }
192 }