1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.mapreduce;
20
21 import org.apache.hadoop.conf.Configuration;
22 import org.apache.hadoop.fs.FileStatus;
23 import org.apache.hadoop.fs.FileSystem;
24 import org.apache.hadoop.fs.Path;
25 import org.apache.hadoop.hbase.CategoryBasedTimeout;
26 import org.apache.hadoop.hbase.Cell;
27 import org.apache.hadoop.hbase.CellScanner;
28 import org.apache.hadoop.hbase.HBaseTestingUtility;
29 import org.apache.hadoop.hbase.TableName;
30 import org.apache.hadoop.hbase.client.Admin;
31 import org.apache.hadoop.hbase.client.HTable;
32 import org.apache.hadoop.hbase.client.Result;
33 import org.apache.hadoop.hbase.io.HFileLink;
34 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
35 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
36 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
37 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.apache.hadoop.hbase.util.FSUtils;
40 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
41 import org.junit.Assert;
42 import org.junit.Rule;
43 import org.junit.Test;
44 import org.junit.rules.TestRule;
45
46 import static org.junit.Assert.assertFalse;
47
48 import java.io.IOException;
49 import java.util.Arrays;
50
51 public abstract class TableSnapshotInputFormatTestBase {
52 @Rule public final TestRule timeout = CategoryBasedTimeout.builder().
53 withTimeout(this.getClass()).withLookingForStuckThread(true).build();
54 protected final HBaseTestingUtility UTIL = new HBaseTestingUtility();
55 protected static final int NUM_REGION_SERVERS = 2;
56 protected static final byte[][] FAMILIES = {Bytes.toBytes("f1"), Bytes.toBytes("f2")};
57
58 protected FileSystem fs;
59 protected Path rootDir;
60
61 public void setupCluster() throws Exception {
62 setupConf(UTIL.getConfiguration());
63 UTIL.setJobWithoutMRCluster();
64 UTIL.startMiniCluster(NUM_REGION_SERVERS, true);
65 rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
66 fs = rootDir.getFileSystem(UTIL.getConfiguration());
67 }
68
69 public void tearDownCluster() throws Exception {
70 UTIL.shutdownMiniCluster();
71 }
72
73 private static void setupConf(Configuration conf) {
74
75 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
76 }
77
78 protected abstract void testWithMockedMapReduce(HBaseTestingUtility util, String snapshotName,
79 int numRegions, int numSplitsPerRegion, int expectedNumSplits) throws Exception;
80
81 protected abstract void testWithMapReduceImpl(HBaseTestingUtility util, TableName tableName,
82 String snapshotName, Path tableDir, int numRegions, int numSplitsPerRegion, int expectedNumSplits,
83 boolean shutdownCluster) throws Exception;
84
85 protected abstract byte[] getStartRow();
86
87 protected abstract byte[] getEndRow();
88
89 @Test
90 public void testWithMockedMapReduceSingleRegion() throws Exception {
91 testWithMockedMapReduce(UTIL, "testWithMockedMapReduceSingleRegion", 1, 1, 1);
92 }
93
94 @Test
95 public void testWithMockedMapReduceMultiRegion() throws Exception {
96 testWithMockedMapReduce(UTIL, "testWithMockedMapReduceMultiRegion", 10, 1, 8);
97 }
98
99 @Test
100 public void testWithMapReduceSingleRegion() throws Exception {
101 testWithMapReduce(UTIL, "testWithMapReduceSingleRegion", 1, 1, 1, false);
102 }
103
104 @Test
105 public void testWithMapReduceMultiRegion() throws Exception {
106 testWithMapReduce(UTIL, "testWithMapReduceMultiRegion", 10, 1, 8, false);
107 }
108
109 @Test
110 public void testWithMapReduceMultipleMappersPerRegion() throws Exception {
111 testWithMapReduce(UTIL, "testWithMapReduceMultiRegion", 10, 5, 50, false);
112 }
113
114 @Test
115
116 public void testWithMapReduceAndOfflineHBaseMultiRegion() throws Exception {
117 testWithMapReduce(UTIL, "testWithMapReduceAndOfflineHBaseMultiRegion", 10, 1, 8, true);
118 }
119
120
121 @Test
122 public void testRestoreSnapshotDoesNotCreateBackRefLinks() throws Exception {
123 setupCluster();
124 TableName tableName = TableName.valueOf("testRestoreSnapshotDoesNotCreateBackRefLinks");
125 String snapshotName = "foo";
126
127 try {
128 createTableAndSnapshot(UTIL, tableName, snapshotName, getStartRow(), getEndRow(), 1);
129
130 Path tmpTableDir = UTIL.getRandomDir();
131
132 testRestoreSnapshotDoesNotCreateBackRefLinksInit(tableName, snapshotName,tmpTableDir);
133
134 Path rootDir = FSUtils.getRootDir(UTIL.getConfiguration());
135 for (Path regionDir : FSUtils.getRegionDirs(fs, FSUtils.getTableDir(rootDir, tableName))) {
136 for (Path storeDir : FSUtils.getFamilyDirs(fs, regionDir)) {
137 for (FileStatus status : fs.listStatus(storeDir)) {
138 System.out.println(status.getPath());
139 if (StoreFileInfo.isValid(status)) {
140 Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(UTIL.getConfiguration(),
141 tableName, regionDir.getName(), storeDir.getName());
142
143 Path path = HFileLink.getBackReferencesDir(storeDir, status.getPath().getName());
144
145 assertFalse("There is a back reference in " + path, fs.exists(path));
146
147 path = HFileLink.getBackReferencesDir(archiveStoreDir, status.getPath().getName());
148
149 assertFalse("There is a back reference in " + path, fs.exists(path));
150 }
151 }
152 }
153 }
154 } finally {
155 UTIL.getHBaseAdmin().deleteSnapshot(snapshotName);
156 UTIL.deleteTable(tableName);
157 tearDownCluster();
158 }
159 }
160
161 public abstract void testRestoreSnapshotDoesNotCreateBackRefLinksInit(TableName tableName,
162 String snapshotName, Path tmpTableDir) throws Exception;
163
164 protected void testWithMapReduce(HBaseTestingUtility util, String snapshotName,
165 int numRegions, int numSplitsPerRegion, int expectedNumSplits, boolean shutdownCluster) throws Exception {
166 setupCluster();
167 try {
168 Path tableDir = util.getRandomDir();
169 TableName tableName = TableName.valueOf("testWithMapReduce");
170 testWithMapReduceImpl(util, tableName, snapshotName, tableDir, numRegions,
171 numSplitsPerRegion, expectedNumSplits, shutdownCluster);
172 } finally {
173 tearDownCluster();
174 }
175 }
176
177 protected static void verifyRowFromMap(ImmutableBytesWritable key, Result result)
178 throws IOException {
179 byte[] row = key.get();
180 CellScanner scanner = result.cellScanner();
181 while (scanner.advance()) {
182 Cell cell = scanner.current();
183
184
185 Assert.assertEquals(0, Bytes.compareTo(row, 0, row.length,
186 cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
187 }
188
189 for (int j = 0; j < FAMILIES.length; j++) {
190 byte[] actual = result.getValue(FAMILIES[j], FAMILIES[j]);
191 Assert.assertArrayEquals("Row in snapshot does not match, expected:" + Bytes.toString(row)
192 + " ,actual:" + Bytes.toString(actual), row, actual);
193 }
194 }
195
196 protected static void createTableAndSnapshot(HBaseTestingUtility util, TableName tableName,
197 String snapshotName, byte[] startRow, byte[] endRow, int numRegions)
198 throws Exception {
199 try {
200 util.deleteTable(tableName);
201 } catch(Exception ex) {
202
203 }
204
205 if (numRegions > 1) {
206 util.createTable(tableName, FAMILIES, 1, startRow, endRow, numRegions);
207 } else {
208 util.createTable(tableName, FAMILIES);
209 }
210 Admin admin = util.getHBaseAdmin();
211
212
213 HTable table = new HTable(util.getConfiguration(), tableName);
214 util.loadTable(table, FAMILIES);
215
216 Path rootDir = FSUtils.getRootDir(util.getConfiguration());
217 FileSystem fs = rootDir.getFileSystem(util.getConfiguration());
218
219 SnapshotTestingUtils.createSnapshotAndValidate(admin, tableName,
220 Arrays.asList(FAMILIES), null, snapshotName, rootDir, fs, true);
221
222
223 byte[] value = Bytes.toBytes("after_snapshot_value");
224 util.loadTable(table, FAMILIES, value);
225
226
227 admin.flush(tableName);
228 table.close();
229 }
230
231 }